diff --git a/docker-compose.yml b/docker-compose.yml index 4cfafb2..2d78266 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,115 +1,117 @@ version: '3.4' x-restart-policy: &restart_policy restart: unless-stopped x-sentry-defaults: &sentry_defaults << : *restart_policy build: context: ./sentry args: - SENTRY_IMAGE image: sentry-onpremise-local depends_on: - redis - memcached - snuba-api - snuba-consumer - snuba-replacer - symbolicator environment: SNUBA: 'http://snuba-api:1218' volumes: - - 'sentry-data:/var/lib/sentry/files' + - 'sentry-data:/data' x-snuba-defaults: &snuba_defaults << : *restart_policy depends_on: - redis - clickhouse image: 'getsentry/snuba:latest' environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse - DEFAULT_BROKERS: 'kafka:9093' + DEFAULT_BROKERS: 'kafka:9092' REDIS_HOST: redis # TODO: Remove these after getsentry/snuba#353 UWSGI_MAX_REQUESTS: '10000' UWSGI_DISABLE_LOGGING: 'true' UWSGI_ENABLE_THREADS: 'true' UWSGI_DIE_ON_TERM: 'true' UWSGI_NEED_APP: 'true' UWSGI_IGNORE_SIGPIPE: 'true' UWSGI_IGNORE_WRITE_ERRORS: 'true' UWSGI_DISABLE_WRITE_EXCEPTION: 'true' services: memcached: << : *restart_policy image: 'memcached:1.5-alpine' redis: << : *restart_policy image: 'redis:5.0-alpine' volumes: - 'sentry-redis:/data' clickhouse: << : *restart_policy image: 'yandex/clickhouse-server:19.4' ulimits: nofile: soft: 262144 hard: 262144 volumes: - 'sentry-clickhouse:/var/lib/clickhouse' snuba-api: << : *snuba_defaults snuba-consumer: << : *snuba_defaults command: consumer --auto-offset-reset=latest --max-batch-time-ms 750 snuba-replacer: << : *snuba_defaults command: replacer --auto-offset-reset=latest --max-batch-size 3 snuba-cleanup: << : *snuba_defaults image: snuba-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: 'getsentry/snuba:latest' command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"' symbolicator: << : *restart_policy image: 'getsentry/symbolicator:latest' volumes: - 'sentry-symbolicator:/data' command: run symbolicator-cleanup: image: symbolicator-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: 'getsentry/symbolicator:latest' command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' + volumes: + - 'sentry-symbolicator:/data' web: << : *sentry_defaults ports: - '9000:9000/tcp' cron: << : *sentry_defaults command: run cron worker: << : *sentry_defaults command: run worker sentry-cleanup: << : *sentry_defaults image: sentry-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: 'sentry-onpremise-local' command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' volumes: sentry-data: external: true sentry-redis: external: true sentry-clickhouse: external: true sentry-symbolicator: external: true diff --git a/install.sh b/install.sh index 8801c02..1ddf916 100755 --- a/install.sh +++ b/install.sh @@ -1,110 +1,114 @@ #!/usr/bin/env bash set -e MIN_DOCKER_VERSION='17.05.0' MIN_COMPOSE_VERSION='1.19.0' MIN_RAM=2400 # MB SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' DID_CLEAN_UP=0 # the cleanup function will be the exit point cleanup () { if [ "$DID_CLEAN_UP" -eq 1 ]; then return 0; fi echo "Cleaning up..." docker-compose down &> /dev/null DID_CLEAN_UP=1 } trap cleanup ERR INT TERM echo "Checking minimum requirements..." DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') COMPOSE_VERSION=$(docker-compose --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); # Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } # Thanks to https://stackoverflow.com/a/25123013/90297 for the quick `sed` pattern function ensure_file_from_example { if [ -f "$1" ]; then echo "$1 already exists, skipped creation." else echo "Creating $1..." cp -n $(echo "$1" | sed 's/\.[^.]*$/.example&/') "$1" fi } if [ $(ver $DOCKER_VERSION) -lt $(ver $MIN_DOCKER_VERSION) ]; then echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" exit -1 fi if [ $(ver $COMPOSE_VERSION) -lt $(ver $MIN_COMPOSE_VERSION) ]; then echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" exit -1 fi if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" exit -1 fi echo "" ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS echo "" echo "Creating volumes for persistent storage..." echo "Created $(docker volume create --name=sentry-data)." echo "Created $(docker volume create --name=sentry-redis)." echo "Created $(docker volume create --name=sentry-clickhouse)." echo "Created $(docker volume create --name=sentry-symbolicator)." echo "" echo "Building and tagging Docker images..." echo "" # Build the sentry onpremise image first as it is needed for the cron image docker-compose build --force-rm web docker-compose build --force-rm echo "" echo "Docker images built." echo "" echo "Setting up database..." if [ $CI ]; then docker-compose run --rm web upgrade --noinput echo "" echo "Did not prompt for user creation due to non-interactive shell." echo "Run the following command to create one yourself (recommended):" echo "" echo " docker-compose run --rm web createuser" echo "" else docker-compose run --rm web upgrade fi echo "Boostrapping Snuba..." docker-compose up -d redis clickhouse until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do # `bootstrap` is for fresh installs, and `migrate` is for existing installs # Running them both for both cases is harmless so we blindly run them docker-compose run --rm snuba-api bootstrap --force || true; docker-compose run --rm snuba-api migrate || true; done; echo "" +echo "Migrating file storage..." +docker run --rm -it -v sentry-data:/data alpine ash -c \ + "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" + cleanup echo "" echo "----------------" echo "You're all done! Run the following command to get Sentry running:" echo "" echo " docker-compose up -d" echo "" diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 5cff314..d31cf2e 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -1,71 +1,73 @@ # While a lot of configuration in Sentry can be changed via the UI, for all # new-style config (as of 8.0) you can also declare values here in this file # to enforce defaults or to ensure they cannot be changed via the UI. For more # information see the Sentry documentation. ############### # Mail Server # ############### # mail.backend: 'smtp' # Use dummy if you want to disable email entirely mail.host: 'smtp' # mail.port: 25 # mail.username: '' # mail.password: '' # mail.use-tls: false # The email address to send on behalf of # mail.from: 'root@localhost' # If you'd like to configure email replies, enable this. # mail.enable-replies: true # When email-replies are enabled, this value is used in the Reply-To header # mail.reply-hostname: '' # If you're using mailgun for inbound mail, set your API key and configure a # route to forward to /api/hooks/mailgun/inbound/ # Also don't forget to set `mail.enable-replies: true` above. # mail.mailgun-api-key: '' ################### # System Settings # ################### # If this file ever becomes compromised, it's important to regenerate your a new key # Changing this value will result in all current sessions being invalidated. # A new key can be generated with `$ sentry config generate-secret-key` system.secret-key: '!!changeme!!' # The ``redis.clusters`` setting is used, unsurprisingly, to configure Redis # clusters. These clusters can be then referred to by name when configuring # backends such as the cache, digests, or TSDB backend. # redis.clusters: # default: # hosts: # 0: # host: 127.0.0.1 # port: 6379 ################ # File storage # ################ # Uploaded media uses these `filestore` settings. The available # backends are either `filesystem` or `s3`. filestore.backend: 'filesystem' filestore.options: - location: '/var/lib/sentry/files' + location: '/data/files' +dsym.cache-path: '/data/dsym-cache' +releasefile.cache-path: '/data/releasefile-cache' # filestore.backend: 's3' # filestore.options: # access_key: 'AKIXXXXXX' # secret_key: 'XXXXXXX' # bucket_name: 's3-bucket-name' system.internal-url-prefix: 'http://web:9000' symbolicator.enabled: true symbolicator.options: url: "http://symbolicator:3021" transaction-events.force-disable-internal-project: true