diff --git a/clickhouse/Dockerfile b/clickhouse/Dockerfile new file mode 100644 index 0000000..4e841c0 --- /dev/null +++ b/clickhouse/Dockerfile @@ -0,0 +1,2 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} diff --git a/clickhouse/config.xml b/clickhouse/config.xml index 55cdbbd..58ab862 100644 --- a/clickhouse/config.xml +++ b/clickhouse/config.xml @@ -1,10 +1,15 @@ - + + + + information 1 1 diff --git a/docker-compose.yml b/docker-compose.yml index b725bc9..c35dc3a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,391 +1,398 @@ x-restart-policy: &restart_policy restart: unless-stopped x-depends_on-healthy: &depends_on-healthy condition: service_healthy x-depends_on-default: &depends_on-default condition: service_started x-healthcheck-defaults: &healthcheck_defaults # Avoid setting the interval too small, as docker uses much more CPU than one would expect. # Related issues: # https://github.com/moby/moby/issues/39102 # https://github.com/moby/moby/issues/39388 # https://github.com/getsentry/self-hosted/issues/1000 interval: "$HEALTHCHECK_INTERVAL" timeout: "$HEALTHCHECK_TIMEOUT" retries: $HEALTHCHECK_RETRIES start_period: 10s x-sentry-defaults: &sentry_defaults <<: *restart_policy image: sentry-self-hosted-local + # Set the platform to build for linux/arm64 when needed on Apple silicon Macs. + platform: ${DOCKER_PLATFORM:-} build: context: ./sentry args: - SENTRY_IMAGE depends_on: redis: <<: *depends_on-healthy kafka: <<: *depends_on-healthy postgres: <<: *depends_on-healthy memcached: <<: *depends_on-default smtp: <<: *depends_on-default snuba-api: <<: *depends_on-default snuba-consumer: <<: *depends_on-default snuba-outcomes-consumer: <<: *depends_on-default snuba-sessions-consumer: <<: *depends_on-default snuba-transactions-consumer: <<: *depends_on-default snuba-subscription-consumer-events: <<: *depends_on-default snuba-subscription-consumer-transactions: <<: *depends_on-default snuba-replacer: <<: *depends_on-default symbolicator: <<: *depends_on-default entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: PYTHONUSERBASE: "/data/custom-packages" SENTRY_CONF: "/etc/sentry" SNUBA: "http://snuba-api:1218" # Force everything to use the system CA bundle # This is mostly needed to support installing custom CA certs # This one is used by botocore DEFAULT_CA_BUNDLE: &ca_bundle "/etc/ssl/certs/ca-certificates.crt" # This one is used by requests REQUESTS_CA_BUNDLE: *ca_bundle # This one is used by grpc/google modules GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR: *ca_bundle # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: SENTRY_MAIL_HOST: volumes: - "sentry-data:/data" - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" - "./certificates:/usr/local/share/ca-certificates:ro" x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: clickhouse: <<: *depends_on-healthy kafka: <<: *depends_on-healthy redis: <<: *depends_on-healthy image: "$SNUBA_IMAGE" environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse DEFAULT_BROKERS: "kafka:9092" REDIS_HOST: redis UWSGI_MAX_REQUESTS: "10000" UWSGI_DISABLE_LOGGING: "true" # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: services: smtp: <<: *restart_policy image: tianon/exim4 hostname: "${SENTRY_MAIL_HOST:-}" volumes: - "sentry-smtp:/var/spool/exim4" - "sentry-smtp-log:/var/log/exim4" memcached: <<: *restart_policy image: "memcached:1.6.9-alpine" healthcheck: <<: *healthcheck_defaults # From: https://stackoverflow.com/a/31877626/5155484 test: echo stats | nc 127.0.0.1 11211 redis: <<: *restart_policy image: "redis:6.2.4-alpine" healthcheck: <<: *healthcheck_defaults test: redis-cli ping volumes: - "sentry-redis:/data" ulimits: nofile: soft: 10032 hard: 10032 postgres: <<: *restart_policy image: "postgres:9.6" healthcheck: <<: *healthcheck_defaults # Using default user "postgres" from sentry/sentry.conf.example.py or value of POSTGRES_USER if provided test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres}"] command: [ "postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=1", "-c", "max_wal_senders=1", ] environment: POSTGRES_HOST_AUTH_METHOD: "trust" entrypoint: /opt/sentry/postgres-entrypoint.sh volumes: - "sentry-postgres:/var/lib/postgresql/data" - type: bind read_only: true source: ./postgres/ target: /opt/sentry/ zookeeper: <<: *restart_policy image: "confluentinc/cp-zookeeper:5.5.0" environment: ZOOKEEPER_CLIENT_PORT: "2181" CONFLUENT_SUPPORT_METRICS_ENABLE: "false" ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN" ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN" KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok" volumes: - "sentry-zookeeper:/var/lib/zookeeper/data" - "sentry-zookeeper-log:/var/lib/zookeeper/log" - "sentry-secrets:/etc/zookeeper/secrets" healthcheck: <<: *healthcheck_defaults test: ["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] kafka: <<: *restart_policy depends_on: zookeeper: <<: *depends_on-healthy image: "confluentinc/cp-kafka:5.5.0" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1" KAFKA_LOG_RETENTION_HOURS: "24" KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too CONFLUENT_SUPPORT_METRICS_ENABLE: "false" KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN" KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN" KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" volumes: - "sentry-kafka:/var/lib/kafka/data" - "sentry-kafka-log:/var/lib/kafka/log" - "sentry-secrets:/etc/kafka/secrets" healthcheck: <<: *healthcheck_defaults test: ["CMD-SHELL", "nc -z localhost 9092"] clickhouse: <<: *restart_policy - image: "yandex/clickhouse-server:20.3.9.70" + image: clickhouse-self-hosted-local + build: + context: + ./clickhouse + args: + BASE_IMAGE: "${CLICKHOUSE_IMAGE:-}" ulimits: nofile: soft: 262144 hard: 262144 volumes: - "sentry-clickhouse:/var/lib/clickhouse" - "sentry-clickhouse-log:/var/log/clickhouse-server" - type: bind read_only: true source: ./clickhouse/config.xml target: /etc/clickhouse-server/config.d/sentry.xml environment: # This limits Clickhouse's memory to 30% of the host memory # If you have high volume and your search return incomplete results # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 healthcheck: test: [ "CMD-SHELL", # Manually override any http_proxy envvar that might be set, because # this wget does not support no_proxy. See: # https://github.com/getsentry/self-hosted/issues/1537 "http_proxy='' wget -nv -t1 --spider 'http://localhost:8123/' || exit 1", ] interval: 3s timeout: 600s retries: 200 geoipupdate: image: "maxmindinc/geoipupdate:v4.7.1" # Override the entrypoint in order to avoid using envvars for config. # Futz with settings so we can keep mmdb and conf in same dir on host # (image looks for them in separate dirs by default). entrypoint: ["/usr/bin/geoipupdate", "-d", "/sentry", "-f", "/sentry/GeoIP.conf"] volumes: - "./geoip:/sentry" snuba-api: <<: *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse snuba-consumer: <<: *snuba_defaults command: consumer --storage errors --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration snuba-outcomes-consumer: <<: *snuba_defaults command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750 # Kafka consumer responsible for feeding session data into Clickhouse snuba-sessions-consumer: <<: *snuba_defaults command: consumer --storage sessions_raw --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: <<: *snuba_defaults command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --commit-log-topic=snuba-commit-log snuba-replacer: <<: *snuba_defaults command: replacer --storage errors --auto-offset-reset=latest --max-batch-size 3 snuba-subscription-consumer-events: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60 --stale-threshold-seconds=900 snuba-subscription-consumer-transactions: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --delay-seconds=60 --schedule-ttl=60 --stale-threshold-seconds=900 snuba-cleanup: <<: *snuba_defaults image: snuba-cleanup-self-hosted-local build: context: ./cron args: BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * snuba cleanup --storage errors --dry-run False"' snuba-transactions-cleanup: <<: *snuba_defaults image: snuba-cleanup-self-hosted-local build: context: ./cron args: BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * snuba cleanup --storage transactions --dry-run False"' symbolicator: <<: *restart_policy image: "$SYMBOLICATOR_IMAGE" volumes: - "sentry-symbolicator:/data" - type: bind read_only: true source: ./symbolicator target: /etc/symbolicator command: run -c /etc/symbolicator/config.yml symbolicator-cleanup: <<: *restart_policy image: symbolicator-cleanup-self-hosted-local build: context: ./cron args: BASE_IMAGE: "$SYMBOLICATOR_IMAGE" command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' volumes: - "sentry-symbolicator:/data" web: <<: *sentry_defaults healthcheck: <<: *healthcheck_defaults test: - "CMD" - "/bin/bash" - "-c" # Courtesy of https://unix.stackexchange.com/a/234089/108960 - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' cron: <<: *sentry_defaults command: run cron worker: <<: *sentry_defaults command: run worker ingest-consumer: <<: *sentry_defaults command: run ingest-consumer --all-consumer-types post-process-forwarder: <<: *sentry_defaults # Increase `--commit-batch-size 1` below to deal with high-load environments. command: run post-process-forwarder --commit-batch-size 1 subscription-consumer-events: <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic events-subscription-results subscription-consumer-transactions: <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic transactions-subscription-results sentry-cleanup: <<: *sentry_defaults image: sentry-cleanup-self-hosted-local build: context: ./cron args: BASE_IMAGE: sentry-self-hosted-local entrypoint: "/entrypoint.sh" command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: <<: *restart_policy ports: - "$SENTRY_BIND:80/tcp" image: "nginx:1.22.0-alpine" volumes: - type: bind read_only: true source: ./nginx target: /etc/nginx - sentry-nginx-cache:/var/cache/nginx depends_on: - web - relay relay: <<: *restart_policy image: "$RELAY_IMAGE" volumes: - type: bind read_only: true source: ./relay target: /work/.relay - type: bind read_only: true source: ./geoip target: /geoip depends_on: kafka: <<: *depends_on-healthy redis: <<: *depends_on-healthy web: <<: *depends_on-healthy volumes: # These store application data that should persist across restarts. sentry-data: external: true sentry-postgres: external: true sentry-redis: external: true sentry-zookeeper: external: true sentry-kafka: external: true sentry-clickhouse: external: true sentry-symbolicator: external: true # These store ephemeral data that needn't persist across restarts. sentry-secrets: sentry-smtp: sentry-nginx-cache: sentry-zookeeper-log: sentry-kafka-log: sentry-smtp-log: sentry-clickhouse-log: diff --git a/install.sh b/install.sh index 8b4f063..efee0cd 100755 --- a/install.sh +++ b/install.sh @@ -1,36 +1,37 @@ #!/usr/bin/env bash set -e # Pre-pre-flight? 🤷 if [[ -n "$MSYSTEM" ]]; then echo "Seems like you are using an MSYS2-based system (such as Git Bash) which is not supported. Please use WSL instead."; exit 1 fi source "$(dirname $0)/install/_lib.sh" # does a `cd .../install/`, among other things # Pre-flight. No impact yet. source parse-cli.sh +source detect-platform.sh source dc-detect-version.sh source error-handling.sh source check-latest-commit.sh source check-minimum-requirements.sh # Let's go! Start impacting things. source turn-things-off.sh source create-docker-volumes.sh source ensure-files-from-examples.sh source ensure-relay-credentials.sh source generate-secret-key.sh source replace-tsdb.sh source update-docker-images.sh source build-docker-images.sh source set-up-zookeeper.sh source install-wal2json.sh source bootstrap-snuba.sh source create-kafka-topics.sh source upgrade-postgres.sh source set-up-and-migrate-database.sh source migrate-file-storage.sh source geoip.sh source wrap-up.sh diff --git a/install/check-minimum-requirements.sh b/install/check-minimum-requirements.sh index dae17bd..d91cd04 100644 --- a/install/check-minimum-requirements.sh +++ b/install/check-minimum-requirements.sh @@ -1,49 +1,55 @@ echo "${_group}Checking minimum requirements ..." source "$(dirname $0)/_min-requirements.sh" # Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') + +if [[ -z "$DOCKER_VERSION" ]]; then + echo "FAIL: Unable to get docker version, is the docker daemon running?" + exit 1 +fi + if [[ "$(ver $DOCKER_VERSION)" -lt "$(ver $MIN_DOCKER_VERSION)" ]]; then echo "FAIL: Expected minimum docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" exit 1 fi echo "Found Docker version $DOCKER_VERSION" COMPOSE_VERSION=$($dc_base version --short) if [[ "$(ver ${COMPOSE_VERSION//v})" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then echo "FAIL: Expected minimum $dc_base version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" exit 1 fi echo "Found Docker Compose version $COMPOSE_VERSION" CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all); if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER" exit 1 elif [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT, found $CPU_AVAILABLE_IN_DOCKER" fi RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then echo "FAIL: Required minimum RAM available to Docker is $MIN_RAM_HARD MB, found $RAM_AVAILABLE_IN_DOCKER MB" exit 1 elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_SOFT" ]]; then echo "WARN: Recommended minimum RAM available to Docker is $MIN_RAM_SOFT MB, found $RAM_AVAILABLE_IN_DOCKER MB" fi #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) # On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297 IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :) -if [[ "$IS_KVM" -eq 0 ]]; then +if [[ "$IS_KVM" -eq 0 && "$DOCKER_ARCH" = "x86_64" ]]; then SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :) if [[ "$SUPPORTS_SSE42" -eq 0 ]]; then echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://github.com/getsentry/self-hosted/issues/340 for more info." exit 1 fi fi echo "${_endgroup}" diff --git a/install/detect-platform.sh b/install/detect-platform.sh new file mode 100755 index 0000000..23ddd4a --- /dev/null +++ b/install/detect-platform.sh @@ -0,0 +1,30 @@ +echo "${_group}Detecting Docker platform" + + +# Sentry SaaS uses stock Yandex ClickHouse, but they don't provide images that +# support ARM, which is relevant especially for Apple M1 laptops, Sentry's +# standard developer environment. As a workaround, we use an altinity image +# targeting ARM. +# +# See https://github.com/getsentry/self-hosted/issues/1385#issuecomment-1101824274 +# +# Images built on ARM also need to be tagged to use linux/arm64 on Apple +# silicon Macs to work around an issue where they are built for +# linux/amd64 by default due to virtualization. +# See https://github.com/docker/cli/issues/3286 for the Docker bug. + +export DOCKER_ARCH=$(docker info --format '{{.Architecture}}') + +if [[ "$DOCKER_ARCH" = "x86_64" ]]; then + export DOCKER_PLATFORM="linux/amd64" + export CLICKHOUSE_IMAGE="yandex/clickhouse-server:20.3.9.70" +elif [[ "$DOCKER_ARCH" = "aarch64" ]]; then + export DOCKER_PLATFORM="linux/arm64" + export CLICKHOUSE_IMAGE="altinity/clickhouse-server:21.6.1.6734-testing-arm" +else + echo "FAIL: Unsupported docker architecture $DOCKER_ARCH." + exit 1 +fi +echo "Detected Docker platform is $DOCKER_PLATFORM" + +echo "${_endgroup}"