diff --git a/.env b/.env index 3848c8c..d49dae9 100644 --- a/.env +++ b/.env @@ -1,9 +1,10 @@ COMPOSE_PROJECT_NAME=sentry_onpremise SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 SENTRY_IMAGE=getsentry/sentry:nightly SNUBA_IMAGE=getsentry/snuba:nightly RELAY_IMAGE=getsentry/relay:nightly SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +WAL2JSON_VERSION=latest diff --git a/.gitignore b/.gitignore index 707622f..8a16904 100644 --- a/.gitignore +++ b/.gitignore @@ -1,86 +1,89 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt sentry_install_log*.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Sphinx documentation docs/_build/ # PyBuilder target/ # Ipython Notebook .ipynb_checkpoints # pyenv .python-version # https://docs.docker.com/compose/extends/ docker-compose.override.yml *.tar data/ .vscode/tags # custom Sentry config sentry/sentry.conf.py sentry/config.yml sentry/*.bak sentry/requirements.txt relay/credentials.json relay/config.yml symbolicator/config.yml geoip/GeoIP.conf geoip/*.mmdb geoip/.geoipupdate.lock + +# wal2json download +postgres/wal2json diff --git a/docker-compose.yml b/docker-compose.yml index 5415f5d..9a78d96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,294 +1,300 @@ version: "3.4" x-restart-policy: &restart_policy restart: unless-stopped x-sentry-defaults: &sentry_defaults <<: *restart_policy image: "$SENTRY_IMAGE" depends_on: - redis - postgres - memcached - smtp - snuba-api - snuba-consumer - snuba-outcomes-consumer - snuba-sessions-consumer - snuba-transactions-consumer - snuba-subscription-consumer-events - snuba-subscription-consumer-transactions - snuba-replacer - symbolicator - kafka entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: PYTHONUSERBASE: "/data/custom-packages" SENTRY_CONF: "/etc/sentry" SNUBA: "http://snuba-api:1218" # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: volumes: - "sentry-data:/data" - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: - redis - clickhouse - kafka image: "$SNUBA_IMAGE" environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse DEFAULT_BROKERS: "kafka:9092" REDIS_HOST: redis UWSGI_MAX_REQUESTS: "10000" UWSGI_DISABLE_LOGGING: "true" # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: services: smtp: <<: *restart_policy image: tianon/exim4 volumes: - "sentry-smtp:/var/spool/exim4" - "sentry-smtp-log:/var/log/exim4" memcached: <<: *restart_policy image: "memcached:1.5-alpine" redis: <<: *restart_policy image: "redis:5.0-alpine" volumes: - "sentry-redis:/data" ulimits: nofile: soft: 10032 hard: 10032 postgres: <<: *restart_policy image: "postgres:9.6" + command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=1", "-c", "max_wal_senders=1"] environment: POSTGRES_HOST_AUTH_METHOD: "trust" + entrypoint: /opt/sentry/postgres-entrypoint.sh volumes: - "sentry-postgres:/var/lib/postgresql/data" + - type: bind + read_only: true + source: ./postgres/ + target: /opt/sentry/ zookeeper: <<: *restart_policy image: "confluentinc/cp-zookeeper:5.5.0" environment: ZOOKEEPER_CLIENT_PORT: "2181" CONFLUENT_SUPPORT_METRICS_ENABLE: "false" ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN" ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN" KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok" volumes: - "sentry-zookeeper:/var/lib/zookeeper/data" - "sentry-zookeeper-log:/var/lib/zookeeper/log" - "sentry-secrets:/etc/zookeeper/secrets" healthcheck: test: ["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] interval: 10s timeout: 5s retries: 6 kafka: <<: *restart_policy depends_on: - zookeeper image: "confluentinc/cp-kafka:5.5.0" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1" KAFKA_LOG_RETENTION_HOURS: "24" KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too CONFLUENT_SUPPORT_METRICS_ENABLE: "false" KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN" KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN" KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" volumes: - "sentry-kafka:/var/lib/kafka/data" - "sentry-kafka-log:/var/lib/kafka/log" - "sentry-secrets:/etc/kafka/secrets" healthcheck: test: ["CMD-SHELL", 'nc -z localhost 9092'] interval: 10s timeout: 5s retries: 6 clickhouse: <<: *restart_policy image: "yandex/clickhouse-server:20.3.9.70" ulimits: nofile: soft: 262144 hard: 262144 volumes: - "sentry-clickhouse:/var/lib/clickhouse" - "sentry-clickhouse-log:/var/log/clickhouse-server" - type: bind read_only: true source: ./clickhouse/config.xml target: /etc/clickhouse-server/config.d/sentry.xml environment: # This limits Clickhouse's memory to 30% of the host memory # If you have high volume and your search return incomplete results # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 geoipupdate: image: "maxmindinc/geoipupdate:latest" # Override the entrypoint in order to avoid using envvars for config. # Futz with settings so we can keep mmdb and conf in same dir on host # (image looks for them in separate dirs by default). entrypoint: ["/usr/bin/geoipupdate", "-d", "/sentry", "-f", "/sentry/GeoIP.conf"] volumes: - "./geoip:/sentry" snuba-api: <<: *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse snuba-consumer: <<: *snuba_defaults command: consumer --storage errors --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration snuba-outcomes-consumer: <<: *snuba_defaults command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750 # Kafka consumer responsible for feeding session data into Clickhouse snuba-sessions-consumer: <<: *snuba_defaults command: consumer --storage sessions_raw --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: <<: *snuba_defaults command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --commit-log-topic=snuba-commit-log snuba-replacer: <<: *snuba_defaults command: replacer --storage errors --auto-offset-reset=latest --max-batch-size 3 snuba-subscription-consumer-events: <<: *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-events-subscriptions-consumers --topic=events --result-topic=events-subscription-results --dataset=events --commit-log-topic=snuba-commit-log --commit-log-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60 snuba-subscription-consumer-transactions: <<: *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-transactions-subscriptions-consumers --topic=events --result-topic=transactions-subscription-results --dataset=transactions --commit-log-topic=snuba-commit-log --commit-log-group=transactions_group --delay-seconds=60 --schedule-ttl=60 snuba-cleanup: <<: *snuba_defaults image: snuba-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * gosu snuba snuba cleanup --storage errors --dry-run False"' snuba-transactions-cleanup: <<: *snuba_defaults image: snuba-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * gosu snuba snuba cleanup --storage transactions --dry-run False"' symbolicator: <<: *restart_policy image: "$SYMBOLICATOR_IMAGE" volumes: - "sentry-symbolicator:/data" - type: bind read_only: true source: ./symbolicator target: /etc/symbolicator command: run -c /etc/symbolicator/config.yml symbolicator-cleanup: <<: *restart_policy image: symbolicator-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: "$SYMBOLICATOR_IMAGE" command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' volumes: - "sentry-symbolicator:/data" web: <<: *sentry_defaults cron: <<: *sentry_defaults command: run cron worker: <<: *sentry_defaults command: run worker ingest-consumer: <<: *sentry_defaults command: run ingest-consumer --all-consumer-types post-process-forwarder: <<: *sentry_defaults # Increase `--commit-batch-size 1` below to deal with high-load environments. command: run post-process-forwarder --commit-batch-size 1 subscription-consumer-events: <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic events-subscription-results subscription-consumer-transactions: <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic transactions-subscription-results sentry-cleanup: <<: *sentry_defaults image: sentry-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: "$SENTRY_IMAGE" entrypoint: "/entrypoint.sh" command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: <<: *restart_policy ports: - "$SENTRY_BIND:80/tcp" image: "nginx:1.16" volumes: - type: bind read_only: true source: ./nginx target: /etc/nginx depends_on: - web - relay relay: <<: *restart_policy image: "$RELAY_IMAGE" volumes: - type: bind read_only: true source: ./relay target: /work/.relay - type: bind read_only: true source: ./geoip target: /geoip depends_on: - kafka - redis - web volumes: sentry-data: external: true sentry-postgres: external: true sentry-redis: external: true sentry-zookeeper: external: true sentry-kafka: external: true sentry-clickhouse: external: true sentry-symbolicator: external: true sentry-secrets: sentry-smtp: sentry-zookeeper-log: sentry-kafka-log: sentry-smtp-log: sentry-clickhouse-log: diff --git a/install.sh b/install.sh index 171851b..3886463 100755 --- a/install.sh +++ b/install.sh @@ -1,28 +1,29 @@ #!/usr/bin/env bash set -e if [[ -n "$MSYSTEM" ]]; then echo "Seems like you are using an MSYS2-based system (such as Git Bash) which is not supported. Please use WSL instead."; exit 1 fi source "$(dirname $0)/install/_lib.sh" # does a `cd .../install/`, among other things source parse-cli.sh source error-handling.sh source check-minimum-requirements.sh source create-docker-volumes.sh source ensure-files-from-examples.sh source generate-secret-key.sh source replace-tsdb.sh source update-docker-images.sh source build-docker-images.sh source turn-things-off.sh source set-up-zookeeper.sh +source install-wal2json.sh source bootstrap-snuba.sh source create-kafka-topics.sh source upgrade-postgres.sh source set-up-and-migrate-database.sh source migrate-file-storage.sh source relay-credentials.sh source geoip.sh source wrap-up.sh diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh new file mode 100644 index 0000000..5ed5800 --- /dev/null +++ b/install/install-wal2json.sh @@ -0,0 +1,34 @@ +echo "${_group}Downloading and installing wal2json ..." + +FILE_TO_USE="../postgres/wal2json/wal2json.so" +ARCH=$(uname -m) +FILE_NAME="wal2json-Linux-$ARCH-glibc.so" + +DOCKER_CURL="docker run --rm curlimages/curl" + +if [[ $WAL2JSON_VERSION == "latest" ]]; then + VERSION=$( + $DOCKER_CURL https://api.github.com/repos/getsentry/wal2json/releases/latest | + grep '"tag_name":' | + sed -E 's/.*"([^"]+)".*/\1/' + ) + + if [[ ! $VERSION ]]; then + echo "Cannot find wal2json latest version" + exit 1 + fi +else + VERSION=$WAL2JSON_VERSION +fi + +mkdir -p ../postgres/wal2json +if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then + mkdir -p "../postgres/wal2json/$VERSION" + $DOCKER_CURL -L \ + "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ + > "../postgres/wal2json/$VERSION/$FILE_NAME" + + cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" +fi + +echo "${_endgroup}" diff --git a/postgres/init_hba.sh b/postgres/init_hba.sh new file mode 100755 index 0000000..f4b332a --- /dev/null +++ b/postgres/init_hba.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Initializes the pg_hba file with access permissions to the replication +# slots. + +set -e + +{ echo "host replication all all trust"; } >> "$PGDATA/pg_hba.conf" diff --git a/postgres/postgres-entrypoint.sh b/postgres/postgres-entrypoint.sh new file mode 100755 index 0000000..0b0d98a --- /dev/null +++ b/postgres/postgres-entrypoint.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# This script replaces the default docker entrypoint for postgres in the +# development environment. +# Its job is to ensure postgres is properly configured to support the +# Change Data Capture pipeline (by setting access permissions and installing +# the replication plugin we use for CDC). Unfortunately the default +# Postgres image does not allow this level of configurability so we need +# to do it this way in order not to have to publish and maintain our own +# Postgres image. +# +# This then, at the end, transfers control to the default entrypoint. + +set -e + +prep_init_db() { + cp /opt/sentry/init_hba.sh /docker-entrypoint-initdb.d/init_hba.sh +} + +cdc_setup_hba_conf() { + # Ensure pg-hba is properly configured to allow connections + # to the replication slots. + + PG_HBA="$PGDATA/pg_hba.conf" + if [ ! -f "$PG_HBA" ]; then + echo "DB not initialized. Postgres will take care of pg_hba" + elif [ "$(grep -c -E "^host\s+replication" "$PGDATA"/pg_hba.conf)" != 0 ]; then + echo "Replication config already present in pg_hba. Not changing anything." + else + # Execute the same script we run on DB initialization + /opt/sentry/init_hba.sh + fi +} + +bind_wal2json() { + # Copy the file in the right place + cp /opt/sentry/wal2json/wal2json.so `pg_config --pkglibdir`/wal2json.so +} + +echo "Setting up Change Data Capture" + +prep_init_db +if [ "$1" = 'postgres' ]; then + cdc_setup_hba_conf + bind_wal2json +fi +exec /docker-entrypoint.sh "$@" diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 4c8bb5a..b5d4586 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -1,18 +1,21 @@ #!/bin/bash set -eu SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $SCRIPT_DIR/.. OLD_VERSION="$1" NEW_VERSION="$2" SYMBOLICATOR_VERSION=${SYMBOLICATOR_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/symbolicator/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} +WAL2JSON_VERSION=${WAL2JSON_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/wal2json/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} sed -i -e "s/^SYMBOLICATOR_IMAGE=\([^:]\+\):.\+\$/SYMBOLICATOR_IMAGE=\1:$SYMBOLICATOR_VERSION/" .env +sed -i -e "s/^WAL2JSON_VERSION=\([^:]\+\):.\+\$/WAL2JSON_VERSION=\1:$WAL2JSON_VERSION/" .env sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE echo "New version: $NEW_VERSION" echo "New Symbolicator version: $SYMBOLICATOR_VERSION" +echo "New wal2json version: $WAL2JSON_VERSION"