diff --git a/.gitignore b/.gitignore index b8ee807..707622f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,83 +1,86 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt sentry_install_log*.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Sphinx documentation docs/_build/ # PyBuilder target/ # Ipython Notebook .ipynb_checkpoints # pyenv .python-version # https://docs.docker.com/compose/extends/ docker-compose.override.yml *.tar data/ .vscode/tags # custom Sentry config sentry/sentry.conf.py sentry/config.yml sentry/*.bak sentry/requirements.txt relay/credentials.json relay/config.yml symbolicator/config.yml +geoip/GeoIP.conf +geoip/*.mmdb +geoip/.geoipupdate.lock diff --git a/docker-compose.yml b/docker-compose.yml index 5909226..160ae62 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,259 +1,272 @@ version: '3.4' x-restart-policy: &restart_policy restart: unless-stopped x-sentry-defaults: &sentry_defaults << : *restart_policy build: context: ./sentry args: - SENTRY_IMAGE - SENTRY_PYTHON2 image: sentry-onpremise-local depends_on: - redis - postgres - memcached - smtp - snuba-api - snuba-consumer - snuba-outcomes-consumer - snuba-sessions-consumer - snuba-transactions-consumer - snuba-replacer - symbolicator - kafka environment: SENTRY_CONF: '/etc/sentry' SNUBA: 'http://snuba-api:1218' # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: volumes: - 'sentry-data:/data' - './sentry:/etc/sentry' + - './geoip:/geoip:ro' x-snuba-defaults: &snuba_defaults << : *restart_policy depends_on: - redis - clickhouse - kafka image: '$SNUBA_IMAGE' environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse DEFAULT_BROKERS: 'kafka:9092' REDIS_HOST: redis UWSGI_MAX_REQUESTS: '10000' UWSGI_DISABLE_LOGGING: 'true' # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: services: smtp: << : *restart_policy image: tianon/exim4 volumes: - 'sentry-smtp:/var/spool/exim4' - 'sentry-smtp-log:/var/log/exim4' memcached: << : *restart_policy image: 'memcached:1.5-alpine' redis: << : *restart_policy image: 'redis:5.0-alpine' volumes: - 'sentry-redis:/data' ulimits: nofile: soft: 10032 hard: 10032 postgres: << : *restart_policy image: 'postgres:9.6' environment: POSTGRES_HOST_AUTH_METHOD: 'trust' volumes: - 'sentry-postgres:/var/lib/postgresql/data' zookeeper: << : *restart_policy image: 'confluentinc/cp-zookeeper:5.5.0' environment: ZOOKEEPER_CLIENT_PORT: '2181' CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: 'WARN' ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: 'WARN' volumes: - 'sentry-zookeeper:/var/lib/zookeeper/data' - 'sentry-zookeeper-log:/var/lib/zookeeper/log' - 'sentry-secrets:/etc/zookeeper/secrets' kafka: << : *restart_policy depends_on: - zookeeper image: 'confluentinc/cp-kafka:5.5.0' environment: KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: '1' KAFKA_LOG_RETENTION_HOURS: '24' KAFKA_MESSAGE_MAX_BYTES: '50000000' #50MB or bust KAFKA_MAX_REQUEST_SIZE: '50000000' #50MB on requests apparently too CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' KAFKA_LOG4J_LOGGERS: 'kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN' KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN' KAFKA_TOOLS_LOG4J_LOGLEVEL: 'WARN' volumes: - 'sentry-kafka:/var/lib/kafka/data' - 'sentry-kafka-log:/var/lib/kafka/log' - 'sentry-secrets:/etc/kafka/secrets' clickhouse: << : *restart_policy image: 'yandex/clickhouse-server:20.3.9.70' ulimits: nofile: soft: 262144 hard: 262144 volumes: - 'sentry-clickhouse:/var/lib/clickhouse' - 'sentry-clickhouse-log:/var/log/clickhouse-server' - type: bind read_only: true source: ./clickhouse/config.xml target: /etc/clickhouse-server/config.d/sentry.xml environment: # This limits Clickhouse's memory to 30% of the host memory # If you have high volume and your search return incomplete results # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 + geoipupdate: + image: 'maxmindinc/geoipupdate:latest' + # Override the entrypoint in order to avoid using envvars for config. + # Futz with settings so we can keep mmdb and conf in same dir on host + # (image looks for them in separate dirs by default). + entrypoint: ['/usr/bin/geoipupdate', '-d', '/sentry', '-f', '/sentry/GeoIP.conf'] + volumes: + - './geoip:/sentry' snuba-api: << : *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse snuba-consumer: << : *snuba_defaults command: consumer --storage events --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration snuba-outcomes-consumer: << : *snuba_defaults command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750 # Kafka consumer responsible for feeding session data into Clickhouse snuba-sessions-consumer: << : *snuba_defaults command: consumer --storage sessions_raw --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: << : *snuba_defaults command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --commit-log-topic=snuba-commit-log snuba-replacer: << : *snuba_defaults command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3 snuba-subscription-consumer-events: << : *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-events-subscriptions-consumers --topic=events --result-topic=events-subscription-results --dataset=events --commit-log-topic=snuba-commit-log --commit-log-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60 snuba-subscription-consumer-transactions: << : *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-transactions-subscriptions-consumers --topic=events --result-topic=transactions-subscription-results --dataset=transactions --commit-log-topic=snuba-commit-log --commit-log-group=transactions_group --delay-seconds=60 --schedule-ttl=60 snuba-cleanup: << : *snuba_defaults image: snuba-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: '$SNUBA_IMAGE' command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"' symbolicator: << : *restart_policy image: '$SYMBOLICATOR_IMAGE' volumes: - 'sentry-symbolicator:/data' - type: bind read_only: true source: ./symbolicator target: /etc/symbolicator command: run -c /etc/symbolicator/config.yml symbolicator-cleanup: << : *restart_policy image: symbolicator-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: '$SYMBOLICATOR_IMAGE' command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' volumes: - 'sentry-symbolicator:/data' web: << : *sentry_defaults cron: << : *sentry_defaults command: run cron worker: << : *sentry_defaults command: run worker ingest-consumer: << : *sentry_defaults command: run ingest-consumer --all-consumer-types post-process-forwarder: << : *sentry_defaults # Increase `--commit-batch-size 1` below to deal with high-load environments. command: run post-process-forwarder --commit-batch-size 1 subscription-consumer-events: << : *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic events-subscription-results subscription-consumer-transactions: << : *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic transactions-subscription-results sentry-cleanup: << : *sentry_defaults image: sentry-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: 'sentry-onpremise-local' command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: << : *restart_policy ports: - '$SENTRY_BIND:80/tcp' image: 'nginx:1.16' volumes: - type: bind read_only: true source: ./nginx target: /etc/nginx depends_on: - web - relay relay: << : *restart_policy image: '$RELAY_IMAGE' volumes: - type: bind read_only: true source: ./relay target: /work/.relay + - type: bind + read_only: true + source: ./geoip + target: /geoip depends_on: - kafka - redis volumes: sentry-data: external: true sentry-postgres: external: true sentry-redis: external: true sentry-zookeeper: external: true sentry-kafka: external: true sentry-clickhouse: external: true sentry-symbolicator: external: true sentry-secrets: sentry-smtp: sentry-zookeeper-log: sentry-kafka-log: sentry-smtp-log: sentry-clickhouse-log: diff --git a/geoip/GeoLite2-City.mmdb.empty b/geoip/GeoLite2-City.mmdb.empty new file mode 100644 index 0000000..94f6921 Binary files /dev/null and b/geoip/GeoLite2-City.mmdb.empty differ diff --git a/install.sh b/install.sh index 2ec1832..44c4f3e 100755 --- a/install.sh +++ b/install.sh @@ -1,346 +1,350 @@ #!/usr/bin/env bash set -e if [[ -n "$MSYSTEM" ]]; then echo "Seems like you are using an MSYS2-based system (such as Git Bash) which is not supported. Please use WSL instead."; exit 1 fi # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 t=$(mktemp) && export -p > "$t" && set -a && . ./.env && set +a && . "$t" && rm "$t" && unset t dc="docker-compose --no-ansi" dcr="$dc run --rm" # Thanks to https://unix.stackexchange.com/a/145654/108960 log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") MIN_DOCKER_VERSION='19.03.6' MIN_COMPOSE_VERSION='1.24.1' MIN_RAM=2400 # MB # Increase the default 10 second SIGTERM timeout # to ensure celery queues are properly drained # between upgrades as task signatures may change across # versions STOP_TIMEOUT=60 # seconds SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' SYMBOLICATOR_CONFIG_YML='symbolicator/config.yml' RELAY_CONFIG_YML='relay/config.yml' RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' MINIMIZE_DOWNTIME= show_help() { cat < /dev/null fi } trap_with_arg cleanup ERR INT TERM EXIT echo "Checking minimum requirements..." DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') COMPOSE_VERSION=$($dc --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); # Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } # Thanks to https://stackoverflow.com/a/25123013/90297 for the quick `sed` pattern function ensure_file_from_example { if [[ -f "$1" ]]; then echo "$1 already exists, skipped creation." else echo "Creating $1..." cp -n $(echo "$1" | sed 's/\.[^.]*$/.example&/') "$1" fi } if [[ "$(ver $DOCKER_VERSION)" -lt "$(ver $MIN_DOCKER_VERSION)" ]]; then echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" exit 1 fi if [[ "$(ver $COMPOSE_VERSION)" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" exit 1 fi if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]]; then echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" exit 1 fi #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) # On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297 IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :) if [[ "$IS_KVM" -eq 0 ]]; then SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :) if [[ "$SUPPORTS_SSE42" -eq 0 ]]; then echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." exit 1 fi fi echo "" echo "Creating volumes for persistent storage..." echo "Created $(docker volume create --name=sentry-data)." echo "Created $(docker volume create --name=sentry-postgres)." echo "Created $(docker volume create --name=sentry-redis)." echo "Created $(docker volume create --name=sentry-zookeeper)." echo "Created $(docker volume create --name=sentry-kafka)." echo "Created $(docker volume create --name=sentry-clickhouse)." echo "Created $(docker volume create --name=sentry-symbolicator)." echo "" ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS ensure_file_from_example $SYMBOLICATOR_CONFIG_YML ensure_file_from_example $RELAY_CONFIG_YML if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then echo "" echo "Generating secret key..." # This is to escape the secret key to be used in sed below # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML echo "Secret key written to $SENTRY_CONFIG_YML" fi replace_tsdb() { if ( [[ -f "$SENTRY_CONFIG_PY" ]] && ! grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY" ); then # Do NOT indent the following string as it would be reflected in the end result, # breaking the final config file. See getsentry/onpremise#624. tsdb_settings="\ SENTRY_TSDB = \"sentry.tsdb.redissnuba.RedisSnubaTSDB\" # Automatic switchover 90 days after $(date). Can be removed afterwards. SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)}\ " if grep -q 'SENTRY_TSDB_OPTIONS = ' "$SENTRY_CONFIG_PY"; then echo "Not attempting automatic TSDB migration due to presence of SENTRY_TSDB_OPTIONS" else echo "Attempting to automatically migrate to new TSDB" # Escape newlines for sed tsdb_settings="${tsdb_settings//$'\n'/\\n}" cp "$SENTRY_CONFIG_PY" "$SENTRY_CONFIG_PY.bak" sed -i -e "s/^SENTRY_TSDB = .*$/${tsdb_settings}/g" "$SENTRY_CONFIG_PY" || true if grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY"; then echo "Migrated TSDB to Snuba. Old configuration file backed up to $SENTRY_CONFIG_PY.bak" return fi echo "Failed to automatically migrate TSDB. Reverting..." mv "$SENTRY_CONFIG_PY.bak" "$SENTRY_CONFIG_PY" echo "$SENTRY_CONFIG_PY restored from backup." fi echo "WARN: Your Sentry configuration uses a legacy data store for time-series data. Remove the options SENTRY_TSDB and SENTRY_TSDB_OPTIONS from $SENTRY_CONFIG_PY and add:" echo "" echo "$tsdb_settings" echo "" echo "For more information please refer to https://github.com/getsentry/onpremise/pull/430" fi } replace_tsdb echo "" echo "Fetching and updating Docker images..." echo "" # We tag locally built images with an '-onpremise-local' suffix. docker-compose pull tries to pull these too and # shows a 404 error on the console which is confusing and unnecessary. To overcome this, we add the stderr>stdout # redirection below and pass it through grep, ignoring all lines having this '-onpremise-local' suffix. $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true # We may not have the set image on the repo (local images) so allow fails docker pull ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2} || true; echo "" echo "Building and tagging Docker images..." echo "" # Build the sentry onpremise image first as it is needed for the cron image $dc build --force-rm web $dc build --force-rm --parallel echo "" echo "Docker images built." if [[ -n "$MINIMIZE_DOWNTIME" ]]; then # Stop everything but relay and nginx $dc rm -fsv $($dc config --services | grep -v -E '^(nginx|relay)$') else # Clean up old stuff and ensure nothing is working while we install/update # This is for older versions of on-premise: $dc -p onpremise down -t $STOP_TIMEOUT --rmi local --remove-orphans # This is for newer versions $dc down -t $STOP_TIMEOUT --rmi local --remove-orphans fi ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') if [[ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq 1 ]]; then ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') # This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 if [[ "$ZOOKEEPER_LOG_FILE_COUNT" -gt 0 ]] && [[ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq 0 ]]; then $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper fi fi echo "Bootstrapping and migrating Snuba..." $dcr snuba-api bootstrap --no-migrate --force $dcr snuba-api migrations migrate --force echo "" # NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step # XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable) EXISTING_KAFKA_TOPICS=$($dcr kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events" for topic in $NEEDED_KAFKA_TOPICS; do if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq $topic; then echo "Creating additional Kafka topics..." $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092 echo "" fi done # Very naively check whether there's an existing sentry-postgres volume and the PG version in it if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null)" == "9.5" ]]; then docker volume rm sentry-postgres-new || true # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume docker run --rm \ -v sentry-postgres:/var/lib/postgresql/9.5/data \ -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ tianon/postgres-upgrade:9.5-to-9.6 # Get rid of the old volume as we'll rename the new one to that docker volume rm sentry-postgres docker volume create --name sentry-postgres # There's no rename volume in Docker so copy the contents from old to new name # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` # doesn't do that automatically. docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" # Finally, remove the new old volume as we are all in sentry-postgres now docker volume rm sentry-postgres-new fi echo "" echo "Setting up database..." if [[ -n "$CI" || "$SKIP_USER_PROMPT" == 1 ]]; then $dcr web upgrade --noinput echo "" echo "Did not prompt for user creation due to non-interactive shell." echo "Run the following command to create one yourself (recommended):" echo "" echo " docker-compose run --rm web createuser" echo "" else $dcr web upgrade fi SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l || true") if [[ -n "$SENTRY_DATA_NEEDS_MIGRATION" ]]; then echo "Migrating file storage..." # Use the web (Sentry) image so the file owners are kept as sentry:sentry # The `\"` escape pattern is to make this compatible w/ Git Bash on Windows. See #329. $dcr --entrypoint \"/bin/bash\" web -c \ "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" fi if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then echo "" echo "Generating Relay credentials..." # We need the ugly hack below as `relay generate credentials` tries to read the config and the credentials # even with the `--stdout` and `--overwrite` flags and then errors out when the credentials file exists but # not valid JSON. We hit this case as we redirect output to the same config folder, creating an empty # credentials file before relay runs. $dcr --no-deps -v $(pwd)/$RELAY_CONFIG_YML:/tmp/config.yml relay --config /tmp credentials generate --stdout > "$RELAY_CREDENTIALS_JSON" echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" fi + +./install/geoip.sh + + if [[ "$MINIMIZE_DOWNTIME" ]]; then # Start the whole setup, except nginx and relay. $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') $dc exec -T nginx service nginx reload echo "Waiting for Sentry to start..." docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' # Make sure everything is up. This should only touch relay and nginx $dc up -d else echo "" echo "----------------" echo "You're all done! Run the following command to get Sentry running:" echo "" echo " docker-compose up -d" echo "" fi diff --git a/install/geoip.sh b/install/geoip.sh new file mode 100755 index 0000000..ec37b21 --- /dev/null +++ b/install/geoip.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +if [ ! -f 'install.sh' ]; then echo 'Where are you?'; exit 1; fi + +dc="docker-compose --no-ansi" +dcr="$dc run --rm" + + +install_geoip() { + local mmdb='geoip/GeoLite2-City.mmdb' + local conf='geoip/GeoIP.conf' + local result='Done' + + echo "Setting up IP address geolocation ..." + if [[ ! -f "$mmdb" ]]; then + echo -n "Installing (empty) IP address geolocation database ... " + cp "$mmdb.empty" "$mmdb" + echo "done." + else + echo "IP address geolocation database already exists." + fi + + if [[ ! -f "$conf" ]]; then + echo "IP address geolocation is not configured for updates." + echo "See https://develop.sentry.dev/self-hosted/geolocation/ for instructions." + result='Error' + else + echo "IP address geolocation is configured for updates." + echo "Updating IP address geolocation database ... " + $dcr geoipupdate + if [ $? -gt 0 ]; then + result='Error' + fi + echo "$result updating IP address geolocation database." + fi + echo "$result setting up IP address geolocation." +} + +install_geoip diff --git a/nginx/nginx.conf b/nginx/nginx.conf index bd81a5e..e8d5644 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,75 +1,75 @@ user nginx; worker_processes 1; error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; events { worker_connections 1024; } http { default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log /var/log/nginx/access.log main; sendfile on; tcp_nopush on; tcp_nodelay on; reset_timedout_connection on; keepalive_timeout 75s; gzip off; server_tokens off; server_names_hash_bucket_size 64; types_hash_max_size 2048; types_hash_bucket_size 64; client_max_body_size 100m; proxy_http_version 1.1; proxy_redirect off; proxy_buffering off; proxy_next_upstream error timeout invalid_header http_502 http_503 non_idempotent; proxy_next_upstream_tries 2; # Remove the Connection header if the client sends it, # it could be "close" to close a keepalive connection proxy_set_header Connection ''; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Request-Id $request_id; proxy_read_timeout 30s; proxy_send_timeout 5s; upstream relay { server relay:3000; } upstream sentry { server web:9000; } server { listen 80; location /api/store/ { proxy_pass http://relay; } location ~ ^/api/[1-9]\d*/ { proxy_pass http://relay; } location / { proxy_pass http://sentry; } } } diff --git a/relay/config.example.yml b/relay/config.example.yml index 0488ba9..8538bd7 100644 --- a/relay/config.example.yml +++ b/relay/config.example.yml @@ -1,12 +1,13 @@ relay: upstream: "http://web:9000/" host: 0.0.0.0 port: 3000 logging: level: WARN processing: enabled: true kafka_config: - {name: "bootstrap.servers", value: "kafka:9092"} - {name: "message.max.bytes", value: 50000000} #50MB or bust redis: redis://redis:6379 + geoip_path: "/geoip/GeoLite2-City.mmdb" diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index ca217d1..2d89b56 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -1,268 +1,274 @@ # This file is just Python, with a touch of Django which means # you can inherit and tweak settings to your hearts content. from sentry.conf.server import * # NOQA # Generously adapted from pynetlinux: https://git.io/JJmga def get_internal_network(): import ctypes import fcntl import math import socket import struct iface = b"eth0" sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) ifreq = struct.pack(b"16sH14s", iface, socket.AF_INET, b"\x00" * 14) try: ip = struct.unpack( b"!I", struct.unpack(b"16sH2x4s8x", fcntl.ioctl(sockfd, 0x8915, ifreq))[2] )[0] netmask = socket.ntohl( struct.unpack(b"16sH2xI8x", fcntl.ioctl(sockfd, 0x891B, ifreq))[2] ) except IOError: return () base = socket.inet_ntoa(struct.pack(b"!I", ip & netmask)) netmask_bits = 32 - int(round(math.log(ctypes.c_uint32(~netmask).value + 1, 2), 1)) return "{0:s}/{1:d}".format(base, netmask_bits) INTERNAL_SYSTEM_IPS = (get_internal_network(),) DATABASES = { "default": { "ENGINE": "sentry.db.postgres", "NAME": "postgres", "USER": "postgres", "PASSWORD": "", "HOST": "postgres", "PORT": "", } } # You should not change this setting after your database has been created # unless you have altered all schemas first SENTRY_USE_BIG_INTS = True # If you're expecting any kind of real traffic on Sentry, we highly recommend # configuring the CACHES and Redis settings ########### # General # ########### # Instruct Sentry that this install intends to be run by a single organization # and thus various UI optimizations should be enabled. SENTRY_SINGLE_ORGANIZATION = True SENTRY_OPTIONS["system.event-retention-days"] = int( env("SENTRY_EVENT_RETENTION_DAYS", "90") ) ######### # Redis # ######### # Generic Redis configuration used as defaults for various things including: # Buffers, Quotas, TSDB SENTRY_OPTIONS["redis.clusters"] = { "default": { "hosts": {0: {"host": "redis", "password": "", "port": "6379", "db": "0"}} } } ######### # Queue # ######### # See https://develop.sentry.dev/services/queue/ for more # information on configuring your queue broker and workers. Sentry relies # on a Python framework called Celery to manage queues. rabbitmq_host = None if rabbitmq_host: BROKER_URL = "amqp://{username}:{password}@{host}/{vhost}".format( username="guest", password="guest", host=rabbitmq_host, vhost="/" ) else: BROKER_URL = "redis://:{password}@{host}:{port}/{db}".format( **SENTRY_OPTIONS["redis.clusters"]["default"]["hosts"][0] ) ######### # Cache # ######### # Sentry currently utilizes two separate mechanisms. While CACHES is not a # requirement, it will optimize several high throughput patterns. CACHES = { "default": { "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", "LOCATION": ["memcached:11211"], "TIMEOUT": 3600, } } # A primary cache is required for things such as processing events SENTRY_CACHE = "sentry.cache.redis.RedisCache" DEFAULT_KAFKA_OPTIONS = { "bootstrap.servers": "kafka:9092", "message.max.bytes": 50000000, "socket.timeout.ms": 1000, } SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" SENTRY_EVENTSTREAM_OPTIONS = {"producer_configuration": DEFAULT_KAFKA_OPTIONS} KAFKA_CLUSTERS["default"] = DEFAULT_KAFKA_OPTIONS ############### # Rate Limits # ############### # Rate limits apply to notification handlers and are enforced per-project # automatically. SENTRY_RATELIMITER = "sentry.ratelimits.redis.RedisRateLimiter" ################## # Update Buffers # ################## # Buffers (combined with queueing) act as an intermediate layer between the # database and the storage API. They will greatly improve efficiency on large # numbers of the same events being sent to the API in a short amount of time. # (read: if you send any kind of real data to Sentry, you should enable buffers) SENTRY_BUFFER = "sentry.buffer.redis.RedisBuffer" ########## # Quotas # ########## # Quotas allow you to rate limit individual projects or the Sentry install as # a whole. SENTRY_QUOTAS = "sentry.quotas.redis.RedisQuota" ######## # TSDB # ######## # The TSDB is used for building charts as well as making things like per-rate # alerts possible. SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB" ######### # SNUBA # ######### SENTRY_SEARCH = "sentry.search.snuba.EventsDatasetSnubaSearchBackend" SENTRY_SEARCH_OPTIONS = {} SENTRY_TAGSTORE_OPTIONS = {} ########### # Digests # ########### # The digest backend powers notification summaries. SENTRY_DIGESTS = "sentry.digests.backends.redis.RedisBackend" ############## # Web Server # ############## SENTRY_WEB_HOST = "0.0.0.0" SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), "protocol": "uwsgi", # This is needed in order to prevent https://git.io/fj7Lw "uwsgi-socket": None, "so-keepalive": True, # Keep this between 15s-75s as that's what Relay supports "http-keepalive": 15, "http-chunked-input": True, # the number of web workers "workers": 3, "threads": 4, "memory-report": False, # Some stuff so uwsgi will cycle workers sensibly "max-requests": 100000, "max-requests-delta": 500, "max-worker-lifetime": 86400, # Duplicate options from sentry default just so we don't get # bit by sentry changing a default value that we depend on. "thunder-lock": True, "log-x-forwarded-for": False, "buffer-size": 32768, "limit-post": 209715200, "disable-logging": True, "reload-on-rss": 600, "ignore-sigpipe": True, "ignore-write-errors": True, "disable-write-exception": True, } ########### # SSL/TLS # ########### # If you're using a reverse SSL proxy, you should enable the X-Forwarded-Proto # header and enable the settings below # SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') # SESSION_COOKIE_SECURE = True # CSRF_COOKIE_SECURE = True # SOCIAL_AUTH_REDIRECT_IS_HTTPS = True # End of SSL/TLS settings ############ # Features # ############ SENTRY_FEATURES["projects:sample-events"] = False SENTRY_FEATURES.update( { feature: True for feature in ( "organizations:discover", "organizations:events", "organizations:global-views", "organizations:incidents", "organizations:integrations-issue-basic", "organizations:integrations-issue-sync", "organizations:invite-members", "organizations:metric-alert-builder-aggregate", "organizations:sso-basic", "organizations:sso-rippling", "organizations:sso-saml2", "organizations:performance-view", "organizations:advanced-search", "projects:custom-inbound-filters", "projects:data-forwarding", "projects:discard-groups", "projects:plugins", "projects:rate-limits", "projects:servicehooks", ) } ) +####################### +# MaxMind Integration # +####################### + +GEOIP_PATH_MMDB = '/geoip/GeoLite2-City.mmdb' + ######################### # Bitbucket Integration # -######################## +######################### # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY' # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET'