diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60807a5..e2f024b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,75 +1,67 @@ name: Test on: # Run CI on all pushes to the master and release/** branches, and on all new # pull requests, and on all pushes to pull requests (even if a pull request # is not against master). push: branches: - "master" - "release/**" pull_request: schedule: - cron: '1 0 * * *' defaults: run: shell: bash jobs: unit-test: runs-on: ubuntu-20.04 name: "unit tests" steps: - name: Checkout uses: actions/checkout@v2 - name: Unit Tests working-directory: install run: find ./ -type f -name "*-test.sh" -exec "./{}" \; integration-test: runs-on: ubuntu-20.04 name: "integration test" strategy: fail-fast: false matrix: include: # Disabled due to https://github.com/getsentry/self-hosted/issues/1415 # - compose_version: "1.28.0" # compose_path: "/usr/local/bin" # - compose_version: "1.29.2" # compose_path: "/usr/local/bin" - compose_version: "v2.0.1" compose_path: "/usr/local/lib/docker/cli-plugins" - compose_version: "v2.2.3" compose_path: "/usr/local/lib/docker/cli-plugins" env: COMPOSE_PROJECT_NAME: self-hosted-${{ strategy.job-index }} steps: - name: Checkout uses: actions/checkout@v2 - name: Get Compose run: | # Always remove `docker compose` support as that's the newer version # and comes installed by default nowadays. sudo rm -f "/usr/local/lib/docker/cli-plugins/docker-compose" sudo rm -f "${{ matrix.compose_path }}/docker-compose" sudo mkdir -p "${{ matrix.compose_path }}" sudo curl -L https://github.com/docker/compose/releases/download/${{ matrix.compose_version }}/docker-compose-`uname -s`-`uname -m` -o "${{ matrix.compose_path }}/docker-compose" sudo chmod +x "${{ matrix.compose_path }}/docker-compose" - name: Integration Test - run: | - echo "Testing initial install" - ./install.sh - ./_integration-test/run.sh - echo "Testing in-place upgrade" - # Also test plugin installation here - echo "sentry-auth-oidc" >> sentry/requirements.txt - ./install.sh --minimize-downtime - ./_integration-test/run.sh + run: ./integration-test.sh - name: Inspect failure if: failure() run: | docker compose ps docker compose logs diff --git a/.gitignore b/.gitignore index e1dbcd6..abc4940 100644 --- a/.gitignore +++ b/.gitignore @@ -1,96 +1,97 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt sentry_install_log*.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Sphinx documentation docs/_build/ # PyBuilder target/ # Ipython Notebook .ipynb_checkpoints # pyenv .python-version # https://docs.docker.com/compose/extends/ docker-compose.override.yml # https://docs.docker.com/compose/environment-variables/#using-the---env-file--option .env.custom *.tar data/ .vscode/tags # custom Sentry config sentry/sentry.conf.py sentry/config.yml sentry/*.bak +sentry/enhance-image.sh sentry/requirements.txt relay/credentials.json relay/config.yml symbolicator/config.yml geoip/GeoIP.conf geoip/*.mmdb geoip/.geoipupdate.lock # wal2json download postgres/wal2json # integration testing _integration-test/custom-ca-roots/nginx/* sentry/test-custom-ca-roots.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..49120af --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,7 @@ +## Testing + +Validate changes to the setup by running the integration test: + +```shell +./integration-test.sh +``` diff --git a/README.md b/README.md index 722d99f..b63a246 100644 --- a/README.md +++ b/README.md @@ -1,63 +1,67 @@ # Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). ## Requirements * Docker 19.03.6+ * Compose 1.28.0+ * 4 CPU Cores * 8 GB RAM * 20 GB Free Disk Space ## Setup +### Installation + +To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020 and Sentry 21.1.0 is the last version to support Python 2. + +During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --skip-user-prompt`. + +Thinking of not managing this yourself? Check out the [SaaS migration docs](https://docs.sentry.io/product/sentry-basics/migration/) or [contact us](https://sentry.io/from/self-hosted) for help. + +Please visit [our documentation](https://develop.sentry.dev/self-hosted/) for everything else. + ### Customize DotEnv (.env) file Environment specific configurations can be done in the `.env.custom` file. It will be located in the root directory of the Sentry installation. By default, there exists no `.env.custom` file. In this case, you can manually add this file by copying the `.env` file to a new `.env.custom` file and adjust your settings in the `.env.custom` file. Please keep in mind to check the `.env` file for changes, when you perform an upgrade of Sentry, so that you can adjust your `.env.custom` accordingly, if required. -### Installation +### Enhance Sentry image -To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020 and Sentry 21.1.0 is the last version to support Python 2. +To install plugins and their dependencies or make other modifications to the Sentry base image, +copy `sentry/enhance-image.example.sh` to `sentry/enhance-image.sh` and add necessary steps there. +For example, you can use `apt-get` to install dependencies and use `pip` to install plugins. -During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --skip-user-prompt`. - -Thinking of not managing this yourself? Check out the [SaaS migration docs](https://docs.sentry.io/product/sentry-basics/migration/) or [contact us](https://sentry.io/from/self-hosted) for help. - -Please visit [our documentation](https://develop.sentry.dev/self-hosted/) for everything else. +After making modifications to `sentry/enhance-image.sh`, run `./install.sh` again to apply them. ## Tips & Tricks ### Event Retention Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file. ### Installing a specific SHA If you want to install a specific release of Sentry, use the tags/releases on this repo. -We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: +We continuously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: ```shell SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh ``` Note that this may not work for all commit SHAs as this repository evolves with Sentry and its satellite projects. It is highly recommended to check out a version of this repository that is close to the timestamp of the Sentry commit you are installing. ### Using Linux If you are using Linux and you need to use `sudo` when running `./install.sh`, make sure to place the environment variable *after* `sudo`: ```shell sudo SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 ./install.sh ``` Where you replace `83b1380` with the sha you want to use. - -[build-status-image]: https://github.com/getsentry/self-hosted/workflows/test/badge.svg -[build-status-url]: https://github.com/getsentry/self-hosted/actions?query=workflow%3Atest+branch%3Amaster+event%3Apush - diff --git a/_integration-test/ensure-customizations-not-present.sh b/_integration-test/ensure-customizations-not-present.sh new file mode 100755 index 0000000..99ae6ba --- /dev/null +++ b/_integration-test/ensure-customizations-not-present.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -ex + +source "$(dirname $0)/../install/_lib.sh" + +source ../install/dc-detect-version.sh + +# Negated version of ensure-customizations-work.sh, make changes in sync +echo "${_group}Ensure customizations not present" +! $dcr --no-deps web bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi" +! $dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi" +! $dcr --no-deps web python -c "import ldap" +! $dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup python -c "import ldap" +echo "${_endgroup}" diff --git a/_integration-test/ensure-customizations-work.sh b/_integration-test/ensure-customizations-work.sh new file mode 100755 index 0000000..8c92c62 --- /dev/null +++ b/_integration-test/ensure-customizations-work.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -ex + +source "$(dirname $0)/../install/_lib.sh" + +source ../install/dc-detect-version.sh + +# Negated version of ensure-customizations-not-present.sh, make changes in sync +echo "${_group}Ensure customizations work" +$dcr --no-deps web bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi" +$dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi" +$dcr --no-deps web python -c "import ldap" +$dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup python -c "import ldap" +echo "${_endgroup}" diff --git a/docker-compose.yml b/docker-compose.yml index b01e8e3..06734ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,382 +1,386 @@ x-restart-policy: &restart_policy restart: unless-stopped x-depends_on-healthy: &depends_on-healthy condition: service_healthy x-depends_on-default: &depends_on-default condition: service_started x-healthcheck-defaults: &healthcheck_defaults # Avoid setting the interval too small, as docker uses much more CPU than one would expect. # Related issues: # https://github.com/moby/moby/issues/39102 # https://github.com/moby/moby/issues/39388 # https://github.com/getsentry/self-hosted/issues/1000 interval: "$HEALTHCHECK_INTERVAL" timeout: "$HEALTHCHECK_TIMEOUT" retries: "$HEALTHCHECK_RETRIES" start_period: 10s x-sentry-defaults: &sentry_defaults <<: *restart_policy - image: "$SENTRY_IMAGE" + image: sentry-self-hosted-local + build: + context: ./sentry + args: + - SENTRY_IMAGE depends_on: redis: <<: *depends_on-healthy kafka: <<: *depends_on-healthy postgres: <<: *depends_on-healthy memcached: <<: *depends_on-default smtp: <<: *depends_on-default snuba-api: <<: *depends_on-default snuba-consumer: <<: *depends_on-default snuba-outcomes-consumer: <<: *depends_on-default snuba-sessions-consumer: <<: *depends_on-default snuba-transactions-consumer: <<: *depends_on-default snuba-subscription-consumer-events: <<: *depends_on-default snuba-subscription-consumer-transactions: <<: *depends_on-default snuba-replacer: <<: *depends_on-default symbolicator: <<: *depends_on-default entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: PYTHONUSERBASE: "/data/custom-packages" SENTRY_CONF: "/etc/sentry" SNUBA: "http://snuba-api:1218" # Force everything to use the system CA bundle # This is mostly needed to support installing custom CA certs # This one is used by botocore DEFAULT_CA_BUNDLE: &ca_bundle "/etc/ssl/certs/ca-certificates.crt" # This one is used by requests REQUESTS_CA_BUNDLE: *ca_bundle # This one is used by grpc/google modules GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR: *ca_bundle # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: SENTRY_MAIL_HOST: volumes: - "sentry-data:/data" - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" - "./certificates:/usr/local/share/ca-certificates:ro" x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: clickhouse: <<: *depends_on-healthy kafka: <<: *depends_on-healthy redis: <<: *depends_on-healthy image: "$SNUBA_IMAGE" environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse DEFAULT_BROKERS: "kafka:9092" REDIS_HOST: redis UWSGI_MAX_REQUESTS: "10000" UWSGI_DISABLE_LOGGING: "true" # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: services: smtp: <<: *restart_policy image: tianon/exim4 hostname: ${SENTRY_MAIL_HOST:-} volumes: - "sentry-smtp:/var/spool/exim4" - "sentry-smtp-log:/var/log/exim4" memcached: <<: *restart_policy image: "memcached:1.6.9-alpine" healthcheck: <<: *healthcheck_defaults # From: https://stackoverflow.com/a/31877626/5155484 test: echo stats | nc 127.0.0.1 11211 redis: <<: *restart_policy image: "redis:6.2.4-alpine" healthcheck: <<: *healthcheck_defaults test: redis-cli ping volumes: - "sentry-redis:/data" ulimits: nofile: soft: 10032 hard: 10032 postgres: <<: *restart_policy image: "postgres:9.6" healthcheck: <<: *healthcheck_defaults # Using default user "postgres" from sentry/sentry.conf.example.py or value of POSTGRES_USER if provided test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres}"] command: [ "postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=1", "-c", "max_wal_senders=1", ] environment: POSTGRES_HOST_AUTH_METHOD: "trust" entrypoint: /opt/sentry/postgres-entrypoint.sh volumes: - "sentry-postgres:/var/lib/postgresql/data" - type: bind read_only: true source: ./postgres/ target: /opt/sentry/ zookeeper: <<: *restart_policy image: "confluentinc/cp-zookeeper:5.5.0" environment: ZOOKEEPER_CLIENT_PORT: "2181" CONFLUENT_SUPPORT_METRICS_ENABLE: "false" ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN" ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN" KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok" volumes: - "sentry-zookeeper:/var/lib/zookeeper/data" - "sentry-zookeeper-log:/var/lib/zookeeper/log" - "sentry-secrets:/etc/zookeeper/secrets" healthcheck: <<: *healthcheck_defaults test: ["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] kafka: <<: *restart_policy depends_on: zookeeper: <<: *depends_on-healthy image: "confluentinc/cp-kafka:5.5.0" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1" KAFKA_LOG_RETENTION_HOURS: "24" KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too CONFLUENT_SUPPORT_METRICS_ENABLE: "false" KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN" KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN" KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" volumes: - "sentry-kafka:/var/lib/kafka/data" - "sentry-kafka-log:/var/lib/kafka/log" - "sentry-secrets:/etc/kafka/secrets" healthcheck: <<: *healthcheck_defaults test: ["CMD-SHELL", "nc -z localhost 9092"] clickhouse: <<: *restart_policy image: "yandex/clickhouse-server:20.3.9.70" ulimits: nofile: soft: 262144 hard: 262144 volumes: - "sentry-clickhouse:/var/lib/clickhouse" - "sentry-clickhouse-log:/var/log/clickhouse-server" - type: bind read_only: true source: ./clickhouse/config.xml target: /etc/clickhouse-server/config.d/sentry.xml environment: # This limits Clickhouse's memory to 30% of the host memory # If you have high volume and your search return incomplete results # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 healthcheck: test: [ "CMD-SHELL", "wget -nv -t1 --spider 'http://localhost:8123/' || exit 1", ] interval: 3s timeout: 600s retries: 200 geoipupdate: image: "maxmindinc/geoipupdate:v4.7.1" # Override the entrypoint in order to avoid using envvars for config. # Futz with settings so we can keep mmdb and conf in same dir on host # (image looks for them in separate dirs by default). entrypoint: ["/usr/bin/geoipupdate", "-d", "/sentry", "-f", "/sentry/GeoIP.conf"] volumes: - "./geoip:/sentry" snuba-api: <<: *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse snuba-consumer: <<: *snuba_defaults command: consumer --storage errors --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration snuba-outcomes-consumer: <<: *snuba_defaults command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750 # Kafka consumer responsible for feeding session data into Clickhouse snuba-sessions-consumer: <<: *snuba_defaults command: consumer --storage sessions_raw --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: <<: *snuba_defaults command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --commit-log-topic=snuba-commit-log snuba-replacer: <<: *snuba_defaults command: replacer --storage errors --auto-offset-reset=latest --max-batch-size 3 snuba-subscription-consumer-events: <<: *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-events-subscriptions-consumers --topic=events --result-topic=events-subscription-results --dataset=events --commit-log-topic=snuba-commit-log --commit-log-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60 snuba-subscription-consumer-transactions: <<: *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-transactions-subscriptions-consumers --topic=events --result-topic=transactions-subscription-results --dataset=transactions --commit-log-topic=snuba-commit-log --commit-log-group=transactions_group --delay-seconds=60 --schedule-ttl=60 snuba-cleanup: <<: *snuba_defaults image: snuba-cleanup-self-hosted-local build: context: ./cron args: BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * gosu snuba snuba cleanup --storage errors --dry-run False"' snuba-transactions-cleanup: <<: *snuba_defaults image: snuba-cleanup-self-hosted-local build: context: ./cron args: BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * gosu snuba snuba cleanup --storage transactions --dry-run False"' symbolicator: <<: *restart_policy image: "$SYMBOLICATOR_IMAGE" volumes: - "sentry-symbolicator:/data" - type: bind read_only: true source: ./symbolicator target: /etc/symbolicator command: run -c /etc/symbolicator/config.yml symbolicator-cleanup: <<: *restart_policy image: symbolicator-cleanup-self-hosted-local build: context: ./cron args: BASE_IMAGE: "$SYMBOLICATOR_IMAGE" command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' volumes: - "sentry-symbolicator:/data" web: <<: *sentry_defaults healthcheck: <<: *healthcheck_defaults test: - "CMD" - "/bin/bash" - '-c' # Courtesy of https://unix.stackexchange.com/a/234089/108960 - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' cron: <<: *sentry_defaults command: run cron worker: <<: *sentry_defaults command: run worker ingest-consumer: <<: *sentry_defaults command: run ingest-consumer --all-consumer-types post-process-forwarder: <<: *sentry_defaults # Increase `--commit-batch-size 1` below to deal with high-load environments. command: run post-process-forwarder --commit-batch-size 1 subscription-consumer-events: <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic events-subscription-results subscription-consumer-transactions: <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic transactions-subscription-results sentry-cleanup: <<: *sentry_defaults image: sentry-cleanup-self-hosted-local build: context: ./cron args: - BASE_IMAGE: "$SENTRY_IMAGE" + BASE_IMAGE: sentry-self-hosted-local entrypoint: "/entrypoint.sh" command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: <<: *restart_policy ports: - "$SENTRY_BIND:80/tcp" image: "nginx:1.21.6-alpine" volumes: - type: bind read_only: true source: ./nginx target: /etc/nginx depends_on: - web - relay relay: <<: *restart_policy image: "$RELAY_IMAGE" volumes: - type: bind read_only: true source: ./relay target: /work/.relay - type: bind read_only: true source: ./geoip target: /geoip depends_on: kafka: <<: *depends_on-healthy redis: <<: *depends_on-healthy web: <<: *depends_on-healthy volumes: # These store application data that should persist across restarts. sentry-data: external: true sentry-postgres: external: true sentry-redis: external: true sentry-zookeeper: external: true sentry-kafka: external: true sentry-clickhouse: external: true sentry-symbolicator: external: true # These store ephemeral data that needn't persist across restarts. sentry-secrets: sentry-smtp: sentry-zookeeper-log: sentry-kafka-log: sentry-smtp-log: sentry-clickhouse-log: diff --git a/install/build-docker-images.sh b/install/build-docker-images.sh index 4bb96b5..63bb0da 100644 --- a/install/build-docker-images.sh +++ b/install/build-docker-images.sh @@ -1,8 +1,11 @@ echo "${_group}Building and tagging Docker images ..." echo "" +# Build any service that provides the image sentry-self-hosted-local first, +# as it is used as the base image for sentry-cleanup-self-hosted-local. +$dc build --force-rm web $dc build --force-rm echo "" echo "Docker images built." echo "${_endgroup}" diff --git a/install/ensure-files-from-examples.sh b/install/ensure-files-from-examples.sh index 17958a0..f96f7ca 100644 --- a/install/ensure-files-from-examples.sh +++ b/install/ensure-files-from-examples.sh @@ -1,8 +1,7 @@ echo "${_group}Ensuring files from examples ..." -ensure_file_from_example $SENTRY_CONFIG_PY -ensure_file_from_example $SENTRY_CONFIG_YML +ensure_file_from_example "$SENTRY_CONFIG_PY" +ensure_file_from_example "$SENTRY_CONFIG_YML" ensure_file_from_example '../symbolicator/config.yml' -ensure_file_from_example '../sentry/requirements.txt' echo "${_endgroup}" diff --git a/integration-test.sh b/integration-test.sh new file mode 100755 index 0000000..52421e8 --- /dev/null +++ b/integration-test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -ex + +echo "Reset customizations" +rm -f sentry/enhance-image.sh +rm -f sentry/requirements.txt + +echo "Testing initial install" +./install.sh +./_integration-test/run.sh +./_integration-test/ensure-customizations-not-present.sh + +echo "Make customizations" +cat < sentry/enhance-image.sh +#!/bin/bash +touch /created-by-enhance-image +apt-get update +apt-get install -y gcc libsasl2-dev python-dev libldap2-dev libssl-dev +EOT +chmod +x sentry/enhance-image.sh +printf "python-ldap" > sentry/requirements.txt + +echo "Testing in-place upgrade and customizations" +./install.sh --minimize-downtime +./_integration-test/run.sh +./_integration-test/ensure-customizations-work.sh diff --git a/sentry/Dockerfile b/sentry/Dockerfile new file mode 100644 index 0000000..5afe9c2 --- /dev/null +++ b/sentry/Dockerfile @@ -0,0 +1,13 @@ +ARG SENTRY_IMAGE +FROM ${SENTRY_IMAGE} + +COPY . /usr/src/sentry + +RUN if [ -s /usr/src/sentry/enhance-image.sh ]; then \ + /usr/src/sentry/enhance-image.sh; \ +fi + +RUN if [ -s /usr/src/sentry/requirements.txt ]; then \ + echo "sentry/requirements.txt is deprecated, use sentry/enhance-image.sh - see https://github.com/getsentry/self-hosted#enhance-sentry-image"; \ + pip install -r /usr/src/sentry/requirements.txt; \ +fi diff --git a/sentry/enhance-image.example.sh b/sentry/enhance-image.example.sh new file mode 100755 index 0000000..e5b3182 --- /dev/null +++ b/sentry/enhance-image.example.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Enhance the base $SENTRY_IMAGE with additional dependencies, plugins - see https://github.com/getsentry/self-hosted#enhance-sentry-image +# For example: +# apt-get update +# apt-get install -y gcc libsasl2-dev python-dev libldap2-dev libssl-dev +# pip install python-ldap diff --git a/sentry/entrypoint.sh b/sentry/entrypoint.sh index 2f2614a..552de05 100755 --- a/sentry/entrypoint.sh +++ b/sentry/entrypoint.sh @@ -1,20 +1,12 @@ #!/bin/bash set -e if [ "$(ls -A /usr/local/share/ca-certificates/)" ]; then update-ca-certificates fi -req_file="/etc/sentry/requirements.txt" -plugins_dir="/data/custom-packages" -checksum_file="$plugins_dir/.checksum" - -if [[ -s "$req_file" ]] && ! cat "$req_file" | grep '^[^#[:space:]]' | shasum -s -c "$checksum_file" 2>/dev/null; then - echo "Installing additional dependencies..." - mkdir -p "$plugins_dir" - pip install --user -r "$req_file" - cat "$req_file" | grep '^[^#[:space:]]' | shasum > "$checksum_file" - echo "" +if [ -e /etc/sentry/requirements.txt ]; then + echo "sentry/requirements.txt is deprecated, use sentry/enhance-image.sh - see https://github.com/getsentry/self-hosted#enhance-sentry-image" fi source /docker-entrypoint.sh diff --git a/sentry/requirements.example.txt b/sentry/requirements.example.txt index b4659b6..e7b63dc 100644 --- a/sentry/requirements.example.txt +++ b/sentry/requirements.example.txt @@ -1 +1 @@ -# Add plugins here +# sentry/requirements.txt is deprecated, use sentry/enhance-image.sh - see https://github.com/getsentry/self-hosted#enhance-sentry-image diff --git a/test.sh b/test.sh index 85945dc..0f90fd5 100755 --- a/test.sh +++ b/test.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash set -e -./_integration-test/run.sh \ No newline at end of file +# This file runs in https://github.com/getsentry/sentry/blob/fe4795f5eae9e0d7c33e0ecb736c9d1369535eca/docker/cloudbuild.yaml#L59 + +./_integration-test/run.sh