Page MenuHomeSoftware Heritage

No OneTemporary

diff --git a/.env b/.env
index 2a80a47..0af9c2d 100644
--- a/.env
+++ b/.env
@@ -1,15 +1,15 @@
COMPOSE_PROJECT_NAME=sentry-self-hosted
SENTRY_EVENT_RETENTION_DAYS=90
# You can either use a port number or an IP:PORT combo for SENTRY_BIND
# See https://docs.docker.com/compose/compose-file/#ports for more
SENTRY_BIND=9000
# Set SENTRY_MAIL_HOST to a valid FQDN (host/domain name) to be able to send emails!
# SENTRY_MAIL_HOST=example.com
-SENTRY_IMAGE=getsentry/sentry:22.1.0
-SNUBA_IMAGE=getsentry/snuba:22.1.0
-RELAY_IMAGE=getsentry/relay:22.1.0
-SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.4.1
+SENTRY_IMAGE=getsentry/sentry:22.2.0
+SNUBA_IMAGE=getsentry/snuba:22.2.0
+RELAY_IMAGE=getsentry/relay:22.2.0
+SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.4.2
WAL2JSON_VERSION=latest
HEALTHCHECK_INTERVAL=30s
HEALTHCHECK_TIMEOUT=60s
HEALTHCHECK_RETRIES=5
diff --git a/.github/workflows/issue-routing-helper.yml b/.github/workflows/issue-routing-helper.yml
deleted file mode 100644
index a361795..0000000
--- a/.github/workflows/issue-routing-helper.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-name: Issue Routing Helper
-on:
- issues:
- types: [labeled]
-env:
- # Use GH_RELEASE_PAT as github-actions bot is not allowed to ping teams
- GH_TOKEN: ${{ secrets.GH_RELEASE_PAT }}
- GH_REPO: ${{ github.repository }}
-jobs:
- route:
- runs-on: ubuntu-latest
- if: >-
- github.event.issue.state == 'open'
- &&
- startsWith(github.event.label.name, 'Team: ')
- &&
- !contains(github.event.issue.labels.*.name, 'Status: Backlog')
- &&
- !contains(github.event.issue.labels.*.name, 'Status: In Progress')
- steps:
- - name: "Ensure a single 'Team: *' label with 'Status: Untriaged'"
- run: |
- labels_to_remove=$(gh api --paginate "/repos/$GH_REPO/labels" -q '[.[].name | select((startswith("Team: ") or startswith("Status: ")) and . != "${{ github.event.label.name }}" and . != "Status: Untriaged")] | join(",")')
- gh issue edit ${{ github.event.issue.number }} --remove-label "$labels_to_remove" --add-label '${{ github.event.label.name }},Status: Untriaged'
- - name: "Mention/ping assigned team for triage"
- run: |
- # Get team label mention name:
- team_label='${{ github.event.label.name }}'
- team_name="${team_label:6}" # Strip the first 6 chars, which is the 'Team: ' part
- team_slug="${team_name// /-}" # Replace spaces with hyphens for url/slug friendliness
- mention_slug=$(gh api "/orgs/getsentry/teams/$team_slug" -q .slug || true)
-
- if [[ -z "$mention_slug" ]]; then
- echo "Couldn't find team mention from slug, trying the label description"
- team_slug=$(gh api "/repos/$GH_REPO/labels/$team_label" -q '.description')
- mention_slug=$(gh api "/orgs/getsentry/teams/$team_slug" -q .slug || true)
- fi
-
- if [[ -n "$mention_slug" ]]; then
- echo "Routing to @getsentry/$mention_slug for [triage](https://develop.sentry.dev/processing-tickets/#3-triage). ⏲️" > comment_body
- else
- echo "[Failed]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID) to route to \`${{ github.event.label.name }}\`. 😕" > comment_body
- echo "" >> comment_body
- echo "Defaulting to @getsentry/open-source for [triage](https://develop.sentry.dev/processing-tickets/#3-triage). ⏲️" >> comment_body
- fi
- gh issue comment ${{ github.event.issue.number }} --body-file comment_body
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5890960..700a860 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,66 +1,70 @@
name: Test
on:
# Run CI on all pushes to the master and release/** branches, and on all new
# pull requests, and on all pushes to pull requests (even if a pull request
# is not against master).
push:
branches:
- "master"
- "release/**"
pull_request:
defaults:
run:
shell: bash
jobs:
unit-test:
runs-on: ubuntu-20.04
name: "unit tests"
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Unit Tests
working-directory: install
run: find ./ -type f -name "*-test.sh" -exec "./{}" \;
integration-test:
runs-on: ubuntu-20.04
name: "integration test"
strategy:
fail-fast: false
matrix:
include:
- compose_version: '1.28.0'
compose_path: '/usr/local/bin'
+ - compose_version: '1.29.2'
+ compose_path: '/usr/local/bin'
- compose_version: 'v2.0.1'
compose_path: '/usr/local/lib/docker/cli-plugins'
+ - compose_version: 'v2.2.3'
+ compose_path: '/usr/local/lib/docker/cli-plugins'
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get Compose
run: |
# Always remove `docker compose` support as that's the newer version
# and comes installed by default nowadays.
sudo rm -f "/usr/local/lib/docker/cli-plugins/docker-compose"
sudo rm -f "${{ matrix.compose_path }}/docker-compose"
sudo mkdir -p "${{ matrix.compose_path }}"
sudo curl -L https://github.com/docker/compose/releases/download/${{ matrix.compose_version }}/docker-compose-`uname -s`-`uname -m` -o "${{ matrix.compose_path }}/docker-compose"
sudo chmod +x "${{ matrix.compose_path }}/docker-compose"
- name: Integration Test
run: |
echo "Testing initial install"
./install.sh
./_integration-test/run.sh
echo "Testing in-place upgrade"
# Also test plugin installation here
echo "sentry-auth-oidc" >> sentry/requirements.txt
./install.sh --minimize-downtime
./_integration-test/run.sh
- name: Inspect failure
if: failure()
run: |
docker compose ps
docker compose logs
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2930c1e..13aee22 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,115 +1,132 @@
# Changelog
+## 22.2.0
+
+### Various fixes & improvements
+
+- fix: unbound variable _group in reset/dc-detect-version script (#1283) (#1284) by @lovetodream
+- Remove routing helper (#1323) by @chadwhitacre
+- Bump nginx:1.21.6-alpine (#1319) by @aminvakil
+- Add a cloudbuild.yaml for GCB (#1315) by @chadwhitacre
+- Update set-up-and-migrate-database.sh (#1308) by @drmrbrewer
+- Pull relay explicitly to avoid garbage in creds (#1301) by @chadwhitacre
+- Improve logging of docker versions and relay creds (#1298) by @chadwhitacre
+- Remove file again (#1299) by @chadwhitacre
+- Clean up relay credentials generation (#1289) by @chadwhitacre
+- Add CI compose version 1.29.2 / 2.0.1 / 2.2.3 (#1290) by @chadwhitacre
+- Revert "Add CI compose version 1.29.2 / 2.0.1 / 2.2.3 (#1251)" (#1272) by @chadwhitacre
+- Add CI compose version 1.29.2 / 2.0.1 / 2.2.3 (#1251) by @aminvakil
+
## 22.1.0
### Various fixes & improvements
- Make healthcheck variables configurable in .env (#1248) by @aminvakil
- Take some actions to avoid unhealthy containers (#1241) by @chadwhitacre
- Install: setup umask (#1222) by @glensc
- Deprecated /docker-entrypoint.sh call (#1218) by @marcinroman
- Bump nginx:1.21.5-alpine (#1230) by @aminvakil
- Fix reset.sh docker-compose call (#1215) by @aminvakil
- Set worker_processes to auto (#1207) by @aminvakil
## 21.12.0
### Support Docker Compose v2 (ongoing)
Self-hosted Sentry mostly works with Docker Compose v2 (in addition to v1 >= 1.28.0). There is [one more bug](https://github.com/getsentry/self-hosted/issues/1133) we are trying to squash.
By: @chadwhitacre (#1179)
### Prevent Component Drift
When a user runs the `install.sh` script, they get the latest version of the Sentry, Snuba, Relay and Symbolicator projects. However there is no guarantee they have pulled the latest `self-hosted` version first, and running an old one may cause problems. To mitigate this, we now perform a check during installation that the user is on the latest commit if they are on the `master` branch. You can disable this check with `--skip-commit-check`.
By: @chadwhitacre (#1191), @aminvakil (#1186)
### React to log4shell
Self-hosted Sentry is [not vulnerable](https://github.com/getsentry/self-hosted/issues/1196) to the [log4shell](https://log4shell.com/) vulnerability.
By: @chadwhitacre (#1203)
### Forum → Issues
In the interest of reducing sources of truth, providing better support, and restarting the fire of the self-hosted Sentry community, we [deprecated the Discourse forum in favor of GitHub Issues](https://github.com/getsentry/self-hosted/issues/1151).
By: @chadwhitacre (#1167, #1160, #1159)
### Rename onpremise to self-hosted (ongoing)
In the beginning we used the term "on-premise" and over time we introduced the term "self-hosted." In an effort to regain some consistency for both branding and developer mental overhead purposes, we are standardizing on the term "self-hosted." This release includes a fair portion of the work towards this across multiple repos, hopefully a future release will include the remainder. Some orphaned containers / volumes / networks are [expected](https://github.com/getsentry/self-hosted/pull/1169#discussion_r756401917). You may clean them up with `docker-compose down --remove-orphans`.
By: @chadwhitacre (#1169)
### Add support for custom DotEnv file
There are several ways to [configure self-hosted Sentry](https://develop.sentry.dev/self-hosted/#configuration) and one of them is the `.env` file. In this release we add support for a `.env.custom` file that is git-ignored to make it easier for you to override keys configured this way with custom values. Thanks to @Sebi94nbg for the contribution!
By: @Sebi94nbg (#1113)
### Various fixes & improvements
- Revert "Rename onpremise to self-hosted" (5495fe2e) by @chadwhitacre
- Rename onpremise to self-hosted (9ad05d87) by @chadwhitacre
## 21.11.0
### Various fixes & improvements
- Fix #1079 - bug in reset.sh (#1134) by @chadwhitacre
- ci: Enable parallel tests again, increase timeouts (#1125) by @BYK
- fix: Hide compose errors during version check (#1124) by @BYK
- build: Omit nightly bump commit from changelog (#1120) by @BYK
- build: Set master version to nightly (d3e77857)
## 21.10.0
### Support for Docker Compose v2 (ongoing)
You asked for it and you did it! Sentry self-hosted now can work with Docker Compose v2 thanks to our community's contributions.
PRs: #1116
### Various fixes & improvements
- docs: simplify Linux `sudo` instructions in README (#1096)
- build: Set master version to nightly (58874cf9)
## 21.9.0
- fix(healthcheck): Increase retries to 5 (#1072)
- fix(requirements): Make compose version check bw-compatible (#1068)
- ci: Test with the required minimum docker-compose (#1066)
Run tests using docker-compose `1.28.0` instead of latest
- fix(clickhouse): Use correct HTTP port for healthcheck (#1069)
Fixes the regular `Unexpected packet` errors in Clickhouse
## 21.8.0
- feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details.
- fix: Fix `curl` image to version 7.77.0
- upgrade: docker-compose version to 1.29.2
- feat: Leverage health checks for depends_on
## 21.7.0
- No documented changes.
## 21.6.3
- No documented changes.
## 21.6.2
- BREAKING CHANGE: The frontend bundle will be loaded asynchronously (via [#25744](https://github.com/getsentry/sentry/pull/25744)). This is a breaking change that can affect custom plugins that access certain globals in the django template. Please see https://forum.sentry.io/t/breaking-frontend-changes-for-custom-plugins/14184 for more information.
## 21.6.1
- No documented changes.
## 21.6.0
- feat: Add healthchecks for redis, memcached and postgres (#975)
diff --git a/LICENSE b/LICENSE
index be0984a..b3e4c08 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,104 +1,104 @@
Business Source License 1.1
Parameters
Licensor: Functional Software, Inc.
Licensed Work: Sentry
The Licensed Work is (c) 2019 Functional Software, Inc.
Additional Use Grant: You may make use of the Licensed Work, provided that you do
not use the Licensed Work for an Application Monitoring
Service.
An "Application Monitoring Service" is a commercial offering
that allows third parties (other than your employees and
contractors) to access the functionality of the Licensed
Work so that such third parties directly benefit from the
error-reporting or application monitoring features of the
Licensed Work.
-Change Date: 2025-01-15
+Change Date: 2025-02-15
Change License: Apache License, Version 2.0
For information about alternative licensing arrangements for the Software,
please visit: https://sentry.io/pricing/
Notice
The Business Source License (this document, or the "License") is not an Open
Source license. However, the Licensed Work will eventually be made available
under an Open Source License, as stated in this License.
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
"Business Source License" is a trademark of MariaDB Corporation Ab.
-----------------------------------------------------------------------------
Business Source License 1.1
Terms
The Licensor hereby grants you the right to copy, modify, create derivative
works, redistribute, and make non-production use of the Licensed Work. The
Licensor may make an Additional Use Grant, above, permitting limited
production use.
Effective on the Change Date, or the fourth anniversary of the first publicly
available distribution of a specific version of the Licensed Work under this
License, whichever comes first, the Licensor hereby grants you rights under
the terms of the Change License, and the rights granted in the paragraph
above terminate.
If your use of the Licensed Work does not comply with the requirements
currently in effect as described in this License, you must purchase a
commercial license from the Licensor, its affiliated entities, or authorized
resellers, or you must refrain from using the Licensed Work.
All copies of the original and modified Licensed Work, and derivative works
of the Licensed Work, are subject to this License. This License applies
separately for each version of the Licensed Work and the Change Date may vary
for each version of the Licensed Work released by Licensor.
You must conspicuously display this License on each original or modified copy
of the Licensed Work. If you receive the Licensed Work in original or
modified form from a third party, the terms and conditions set forth in this
License apply to your use of that work.
Any use of the Licensed Work in violation of this License will automatically
terminate your rights under this License for the current and all other
versions of the Licensed Work.
This License does not grant you any right in any trademark or logo of
Licensor or its affiliates (provided that you may use a trademark or logo of
Licensor as expressly required by this License).
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
TITLE.
MariaDB hereby grants you permission to use this License’s text to license
your works, and to refer to it using the trademark "Business Source License",
as long as you comply with the Covenants of Licensor below.
Covenants of Licensor
In consideration of the right to use this License’s text and the "Business
Source License" name and trademark, Licensor covenants to MariaDB, and to all
other recipients of the licensed work to be provided by Licensor:
1. To specify as the Change License the GPL Version 2.0 or any later version,
or a license that is compatible with GPL Version 2.0 or a later version,
where "compatible" means that software provided under the Change License can
be included in a program with software provided under GPL Version 2.0 or a
later version. Licensor may specify additional Change Licenses without
limitation.
2. To either: (a) specify an additional grant of rights to use that does not
impose any additional restriction on the right granted in this License, as
the Additional Use Grant; or (b) insert the text "None".
3. To specify a Change Date.
4. Not to modify this License in any other way.
diff --git a/README.md b/README.md
index 5a9525f..c9ff7f2 100644
--- a/README.md
+++ b/README.md
@@ -1,61 +1,61 @@
-# Self-Hosted Sentry 22.1.0
+# Self-Hosted Sentry 22.2.0
Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/).
## Requirements
* Docker 19.03.6+
* Compose 1.28.0+
* 4 CPU Cores
* 8 GB RAM
* 20 GB Free Disk Space
## Setup
### Customize DotEnv (.env) file
Environment specific configurations can be done in the `.env.custom` file. It will be located in the root directory of the Sentry installation.
By default, there exists no `.env.custom` file. In this case, you can manually add this file by copying the `.env` file to a new `.env.custom` file and adjust your settings in the `.env.custom` file.
Please keep in mind to check the `.env` file for changes, when you perform an upgrade of Sentry, so that you can adjust your `.env.custom` accordingly, if required.
### Installation
To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020 and Sentry 21.1.0 is the last version to support Python 2.
During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --skip-user-prompt`.
Please visit [our documentation](https://develop.sentry.dev/self-hosted/) for everything else.
## Tips & Tricks
### Event Retention
Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file.
### Installing a specific SHA
If you want to install a specific release of Sentry, use the tags/releases on this repo.
We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`:
```shell
SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh
```
Note that this may not work for all commit SHAs as this repository evolves with Sentry and its satellite projects. It is highly recommended to check out a version of this repository that is close to the timestamp of the Sentry commit you are installing.
### Using Linux
If you are using Linux and you need to use `sudo` when running `./install.sh`, make sure to place the environment variable *after* `sudo`:
```shell
sudo SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 ./install.sh
```
Where you replace `83b1380` with the sha you want to use.
[build-status-image]: https://github.com/getsentry/self-hosted/workflows/test/badge.svg
[build-status-url]: https://git.io/JUYkh
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
new file mode 100644
index 0000000..5ec784b
--- /dev/null
+++ b/cloudbuild.yaml
@@ -0,0 +1,57 @@
+steps:
+ - name: "gcr.io/$PROJECT_ID/docker-compose"
+ id: clone-and-configure
+ entrypoint: "bash"
+ args:
+ - "-e"
+ - "-c"
+ - |
+ git clone https://github.com/getsentry/self-hosted.git
+ echo '{"version": "3.4", "networks":{"default":{"external":{"name":"cloudbuild"}}}}' > self-hosted/docker-compose.override.yml
+ timeout: 60s
+ - name: "gcr.io/$PROJECT_ID/docker-compose"
+ id: install
+ waitFor:
+ - clone-and-configure
+ dir: self-hosted
+ entrypoint: "bash"
+ args:
+ - "-e"
+ - "-c"
+ - |
+ ./install.sh
+ timeout: 600s
+
+ # The point is to run the tests that get run in sentry and snuba, to avoid
+ # accidentally breaking CI over there (not that this has ever happened;
+ # it's purely theoretical, you understand). Unfortunately this is a bit
+ # duplicatory. Meh.
+
+ - name: "gcr.io/$PROJECT_ID/docker-compose"
+ id: e2e-test
+ waitFor:
+ - install
+ dir: self-hosted
+ entrypoint: "bash"
+ args:
+ - "-e"
+ - "-c"
+ - |
+ set +e
+ ./test.sh
+ test_return=$?
+ set -e
+ if [[ $test_return -ne 0 ]]; then
+ echo "Test failed.";
+ docker-compose ps;
+ docker-compose logs;
+ exit $test_return;
+ fi
+ timeout: 600s
+timeout: 2640s
+options:
+ # We need more memory for Webpack builds & e2e self-hosted tests
+ machineType: "N1_HIGHCPU_8"
+ env:
+ - "CI=1"
+ - "SENTRY_TEST_HOST=http://nginx"
diff --git a/docker-compose.yml b/docker-compose.yml
index 87dfda1..1427971 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,341 +1,341 @@
x-restart-policy: &restart_policy
restart: unless-stopped
x-depends_on-healthy: &depends_on-healthy
condition: service_healthy
x-depends_on-default: &depends_on-default
condition: service_started
x-healthcheck-defaults: &healthcheck_defaults
# Avoid setting the interval too small, as docker uses much more CPU than one would expect.
# Related issues:
# https://github.com/moby/moby/issues/39102
# https://github.com/moby/moby/issues/39388
# https://github.com/getsentry/self-hosted/issues/1000
interval: "$HEALTHCHECK_INTERVAL"
timeout: "$HEALTHCHECK_TIMEOUT"
retries: "$HEALTHCHECK_RETRIES"
start_period: 10s
x-sentry-defaults: &sentry_defaults
<<: *restart_policy
image: "$SENTRY_IMAGE"
depends_on:
redis:
<<: *depends_on-healthy
kafka:
<<: *depends_on-healthy
memcached:
<<: *depends_on-default
snuba-api:
<<: *depends_on-default
snuba-consumer:
<<: *depends_on-default
snuba-outcomes-consumer:
<<: *depends_on-default
snuba-sessions-consumer:
<<: *depends_on-default
snuba-transactions-consumer:
<<: *depends_on-default
snuba-subscription-consumer-events:
<<: *depends_on-default
snuba-subscription-consumer-transactions:
<<: *depends_on-default
snuba-replacer:
<<: *depends_on-default
symbolicator:
<<: *depends_on-default
entrypoint: "/etc/sentry/entrypoint.sh"
command: ["run", "web"]
environment:
PYTHONUSERBASE: "/data/custom-packages"
SENTRY_CONF: "/etc/sentry"
SNUBA: "http://snuba-api:1218"
# Force everything to use the system CA bundle
# This is mostly needed to support installing custom CA certs
# This one is used by botocore
DEFAULT_CA_BUNDLE: &ca_bundle "/etc/ssl/certs/ca-certificates.crt"
# This one is used by requests
REQUESTS_CA_BUNDLE: *ca_bundle
# This one is used by grpc/google modules
GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR: *ca_bundle
# Leaving the value empty to just pass whatever is set
# on the host system (or in the .env file)
SENTRY_EVENT_RETENTION_DAYS:
SENTRY_MAIL_HOST:
volumes:
- "sentry-data:/data"
- "./sentry:/etc/sentry"
- "./geoip:/geoip:ro"
- "./certificates:/usr/local/share/ca-certificates:ro"
x-snuba-defaults: &snuba_defaults
<<: *restart_policy
depends_on:
clickhouse:
<<: *depends_on-healthy
kafka:
<<: *depends_on-healthy
redis:
<<: *depends_on-healthy
image: "$SNUBA_IMAGE"
environment:
SNUBA_SETTINGS: docker
CLICKHOUSE_HOST: clickhouse
DEFAULT_BROKERS: "kafka:9092"
REDIS_HOST: redis
UWSGI_MAX_REQUESTS: "10000"
UWSGI_DISABLE_LOGGING: "true"
# Leaving the value empty to just pass whatever is set
# on the host system (or in the .env file)
SENTRY_EVENT_RETENTION_DAYS:
services:
memcached:
<<: *restart_policy
image: "memcached:1.6.9-alpine"
healthcheck:
<<: *healthcheck_defaults
# From: https://stackoverflow.com/a/31877626/5155484
test: echo stats | nc 127.0.0.1 11211
redis:
<<: *restart_policy
image: "redis:6.2.4-alpine"
healthcheck:
<<: *healthcheck_defaults
test: redis-cli ping
volumes:
- "sentry-redis:/data"
ulimits:
nofile:
soft: 10032
hard: 10032
zookeeper:
<<: *restart_policy
image: "confluentinc/cp-zookeeper:5.5.0"
environment:
ZOOKEEPER_CLIENT_PORT: "2181"
CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN"
ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN"
KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok"
volumes:
- "sentry-zookeeper:/var/lib/zookeeper/data"
- "sentry-zookeeper-log:/var/lib/zookeeper/log"
- "sentry-secrets:/etc/zookeeper/secrets"
healthcheck:
<<: *healthcheck_defaults
test:
["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok']
kafka:
<<: *restart_policy
depends_on:
zookeeper:
<<: *depends_on-healthy
image: "confluentinc/cp-kafka:5.5.0"
environment:
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092"
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1"
KAFKA_LOG_RETENTION_HOURS: "24"
KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust
KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too
CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN"
KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN"
KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN"
volumes:
- "sentry-kafka:/var/lib/kafka/data"
- "sentry-kafka-log:/var/lib/kafka/log"
- "sentry-secrets:/etc/kafka/secrets"
healthcheck:
<<: *healthcheck_defaults
test: ["CMD-SHELL", "nc -z localhost 9092"]
clickhouse:
<<: *restart_policy
image: "yandex/clickhouse-server:20.3.9.70"
ulimits:
nofile:
soft: 262144
hard: 262144
volumes:
- "sentry-clickhouse:/var/lib/clickhouse"
- "sentry-clickhouse-log:/var/log/clickhouse-server"
- type: bind
read_only: true
source: ./clickhouse/config.xml
target: /etc/clickhouse-server/config.d/sentry.xml
environment:
# This limits Clickhouse's memory to 30% of the host memory
# If you have high volume and your search return incomplete results
# You might want to change this to a higher value (and ensure your host has enough memory)
MAX_MEMORY_USAGE_RATIO: 0.3
healthcheck:
test:
[
"CMD-SHELL",
"wget -nv -t1 --spider 'http://localhost:8123/' || exit 1",
]
interval: 3s
timeout: 600s
retries: 200
geoipupdate:
image: "maxmindinc/geoipupdate:v4.7.1"
# Override the entrypoint in order to avoid using envvars for config.
# Futz with settings so we can keep mmdb and conf in same dir on host
# (image looks for them in separate dirs by default).
entrypoint:
["/usr/bin/geoipupdate", "-d", "/sentry", "-f", "/sentry/GeoIP.conf"]
volumes:
- "./geoip:/sentry"
snuba-api:
<<: *snuba_defaults
# Kafka consumer responsible for feeding events into Clickhouse
snuba-consumer:
<<: *snuba_defaults
command: consumer --storage errors --auto-offset-reset=latest --max-batch-time-ms 750
# Kafka consumer responsible for feeding outcomes into Clickhouse
# Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data
# since we did not do a proper migration
snuba-outcomes-consumer:
<<: *snuba_defaults
command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750
# Kafka consumer responsible for feeding session data into Clickhouse
snuba-sessions-consumer:
<<: *snuba_defaults
command: consumer --storage sessions_raw --auto-offset-reset=latest --max-batch-time-ms 750
# Kafka consumer responsible for feeding transactions data into Clickhouse
snuba-transactions-consumer:
<<: *snuba_defaults
command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --commit-log-topic=snuba-commit-log
snuba-replacer:
<<: *snuba_defaults
command: replacer --storage errors --auto-offset-reset=latest --max-batch-size 3
snuba-subscription-consumer-events:
<<: *snuba_defaults
command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-events-subscriptions-consumers --topic=events --result-topic=events-subscription-results --dataset=events --commit-log-topic=snuba-commit-log --commit-log-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60
snuba-subscription-consumer-transactions:
<<: *snuba_defaults
command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-transactions-subscriptions-consumers --topic=events --result-topic=transactions-subscription-results --dataset=transactions --commit-log-topic=snuba-commit-log --commit-log-group=transactions_group --delay-seconds=60 --schedule-ttl=60
snuba-cleanup:
<<: *snuba_defaults
image: snuba-cleanup-self-hosted-local
build:
context: ./cron
args:
BASE_IMAGE: "$SNUBA_IMAGE"
command: '"*/5 * * * * gosu snuba snuba cleanup --storage errors --dry-run False"'
snuba-transactions-cleanup:
<<: *snuba_defaults
image: snuba-cleanup-self-hosted-local
build:
context: ./cron
args:
BASE_IMAGE: "$SNUBA_IMAGE"
command: '"*/5 * * * * gosu snuba snuba cleanup --storage transactions --dry-run False"'
symbolicator:
<<: *restart_policy
image: "$SYMBOLICATOR_IMAGE"
volumes:
- "sentry-symbolicator:/data"
- type: bind
read_only: true
source: ./symbolicator
target: /etc/symbolicator
command: run -c /etc/symbolicator/config.yml
symbolicator-cleanup:
<<: *restart_policy
image: symbolicator-cleanup-self-hosted-local
build:
context: ./cron
args:
BASE_IMAGE: "$SYMBOLICATOR_IMAGE"
command: '"55 23 * * * gosu symbolicator symbolicator cleanup"'
volumes:
- "sentry-symbolicator:/data"
web:
<<: *sentry_defaults
healthcheck:
<<: *healthcheck_defaults
test:
- "CMD"
- "/bin/bash"
- '-c'
# Courtesy of https://unix.stackexchange.com/a/234089/108960
- 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3'
cron:
<<: *sentry_defaults
command: run cron
worker:
<<: *sentry_defaults
command: run worker
ingest-consumer:
<<: *sentry_defaults
command: run ingest-consumer --all-consumer-types
post-process-forwarder:
<<: *sentry_defaults
# Increase `--commit-batch-size 1` below to deal with high-load environments.
command: run post-process-forwarder --commit-batch-size 1
subscription-consumer-events:
<<: *sentry_defaults
command: run query-subscription-consumer --commit-batch-size 1 --topic events-subscription-results
subscription-consumer-transactions:
<<: *sentry_defaults
command: run query-subscription-consumer --commit-batch-size 1 --topic transactions-subscription-results
sentry-cleanup:
<<: *sentry_defaults
image: sentry-cleanup-self-hosted-local
build:
context: ./cron
args:
BASE_IMAGE: "$SENTRY_IMAGE"
entrypoint: "/entrypoint.sh"
command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"'
nginx:
<<: *restart_policy
ports:
- "$SENTRY_BIND:80/tcp"
- image: "nginx:1.21.5-alpine"
+ image: "nginx:1.21.6-alpine"
volumes:
- type: bind
read_only: true
source: ./nginx
target: /etc/nginx
depends_on:
- web
- relay
relay:
<<: *restart_policy
image: "$RELAY_IMAGE"
volumes:
- type: bind
read_only: true
source: ./relay
target: /work/.relay
- type: bind
read_only: true
source: ./geoip
target: /geoip
depends_on:
kafka:
<<: *depends_on-healthy
redis:
<<: *depends_on-healthy
web:
<<: *depends_on-healthy
volumes:
# These store application data that should persist across restarts.
sentry-data:
external: true
sentry-redis:
external: true
sentry-zookeeper:
external: true
sentry-kafka:
external: true
sentry-clickhouse:
external: true
sentry-symbolicator:
external: true
# These store ephemeral data that needn't persist across restarts.
sentry-secrets:
sentry-zookeeper-log:
sentry-kafka-log:
sentry-clickhouse-log:
diff --git a/install.sh b/install.sh
index 714856e..8b4f063 100755
--- a/install.sh
+++ b/install.sh
@@ -1,36 +1,36 @@
#!/usr/bin/env bash
set -e
# Pre-pre-flight? 🤷
if [[ -n "$MSYSTEM" ]]; then
echo "Seems like you are using an MSYS2-based system (such as Git Bash) which is not supported. Please use WSL instead.";
exit 1
fi
source "$(dirname $0)/install/_lib.sh" # does a `cd .../install/`, among other things
# Pre-flight. No impact yet.
source parse-cli.sh
source dc-detect-version.sh
source error-handling.sh
source check-latest-commit.sh
source check-minimum-requirements.sh
# Let's go! Start impacting things.
source turn-things-off.sh
source create-docker-volumes.sh
source ensure-files-from-examples.sh
+source ensure-relay-credentials.sh
source generate-secret-key.sh
source replace-tsdb.sh
source update-docker-images.sh
source build-docker-images.sh
source set-up-zookeeper.sh
source install-wal2json.sh
source bootstrap-snuba.sh
source create-kafka-topics.sh
source upgrade-postgres.sh
source set-up-and-migrate-database.sh
source migrate-file-storage.sh
-source relay-credentials.sh
source geoip.sh
source wrap-up.sh
diff --git a/install/check-minimum-requirements.sh b/install/check-minimum-requirements.sh
index 42689bf..b05952a 100644
--- a/install/check-minimum-requirements.sh
+++ b/install/check-minimum-requirements.sh
@@ -1,54 +1,50 @@
echo "${_group}Checking minimum requirements ..."
source "$(dirname $0)/_min-requirements.sh"
# Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368
function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; }
DOCKER_VERSION=$(docker version --format '{{.Server.Version}}')
if [[ "$(ver $DOCKER_VERSION)" -lt "$(ver $MIN_DOCKER_VERSION)" ]]; then
- echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION"
+ echo "FAIL: Expected minimum docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION"
exit 1
fi
+echo "Found Docker version $DOCKER_VERSION"
-if docker compose version &>/dev/null; then
- # If `docker compose` exists then it's guaranteed to be Docker Compose v2, which is good enough for us.
- true
-else
- # If we have `docker-compose` instead then it could be either v1 or v2 (also, use portable sed).
- # See https://github.com/getsentry/self-hosted/issues/1132#issuecomment-982823712 ff. for regex testing.
- COMPOSE_VERSION=$(docker-compose version | head -n1 | sed -E 's/^.* version:? v?([0-9.]+),?.*$/\1/')
- if [[ "$(ver $COMPOSE_VERSION)" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then
- echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION"
- exit 1
- fi
+# See https://github.com/getsentry/self-hosted/issues/1132#issuecomment-982823712 ff. for regex testing.
+COMPOSE_VERSION=$($dc_base version | head -n1 | sed -E 's/^.* version:? v?([0-9.]+),?.*$/\1/')
+if [[ "$(ver $COMPOSE_VERSION)" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then
+ echo "FAIL: Expected minimum $dc_base version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION"
+ exit 1
fi
+echo "Found Docker Compose version $COMPOSE_VERSION"
CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all);
if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then
echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER"
exit 1
elif [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then
echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT, found $CPU_AVAILABLE_IN_DOCKER"
fi
RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}');
if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then
echo "FAIL: Required minimum RAM available to Docker is $MIN_RAM_HARD MB, found $RAM_AVAILABLE_IN_DOCKER MB"
exit 1
elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_SOFT" ]]; then
echo "WARN: Recommended minimum RAM available to Docker is $MIN_RAM_SOFT MB, found $RAM_AVAILABLE_IN_DOCKER MB"
fi
#SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/)
# On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297
IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :)
if [[ "$IS_KVM" -eq 0 ]]; then
SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :)
if [[ "$SUPPORTS_SSE42" -eq 0 ]]; then
echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info."
exit 1
fi
fi
echo "${_endgroup}"
diff --git a/install/create-kafka-topics.sh b/install/create-kafka-topics.sh
index a542cb5..3c5b845 100644
--- a/install/create-kafka-topics.sh
+++ b/install/create-kafka-topics.sh
@@ -1,14 +1,14 @@
echo "${_group}Creating additional Kafka topics ..."
# NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step
# XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable)
-EXISTING_KAFKA_TOPICS=$($dcr kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null)
+EXISTING_KAFKA_TOPICS=$($dcr -T kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null)
NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events"
for topic in $NEEDED_KAFKA_TOPICS; do
if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq $topic; then
$dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092
echo ""
fi
done
echo "${_endgroup}"
diff --git a/install/dc-detect-version.sh b/install/dc-detect-version.sh
index c277400..2c92d6b 100644
--- a/install/dc-detect-version.sh
+++ b/install/dc-detect-version.sh
@@ -1,12 +1,20 @@
+if [ "${GITHUB_ACTIONS:-}" = "true" ]; then
+ _group="::group::"
+ _endgroup="::endgroup::"
+else
+ _group="▶ "
+ _endgroup=""
+fi
+
echo "${_group}Initializing Docker Compose ..."
# Some environments still use `docker-compose` even for Docker Compose v2.
dc_base="$(docker compose version &> /dev/null && echo 'docker compose' || echo 'docker-compose')"
if [[ "$(basename $0)" = "install.sh" ]]; then
dc="$dc_base --ansi never --env-file ${_ENV}"
else
dc="$dc_base --ansi never"
fi
dcr="$dc run --rm"
echo "${_endgroup}"
diff --git a/install/relay-credentials-test.sh b/install/ensure-relay-credentials-test.sh
similarity index 87%
rename from install/relay-credentials-test.sh
rename to install/ensure-relay-credentials-test.sh
index ea740f3..b153d63 100755
--- a/install/relay-credentials-test.sh
+++ b/install/ensure-relay-credentials-test.sh
@@ -1,24 +1,24 @@
#!/usr/bin/env bash
source "$(dirname $0)/_test_setup.sh"
cfg="../relay/config.yml"
creds="../relay/credentials.json"
# Relay files don't exist in a clean clone.
test ! -f $cfg
test ! -f $creds
# Running the install script adds them.
-source relay-credentials.sh
+source ensure-relay-credentials.sh
test -f $cfg
test -f $creds
test "$(jq -r 'keys[2]' $creds)" = "secret_key"
# If the files exist we don't touch it.
echo GARBAGE > $cfg
echo MOAR GARBAGE > $creds
-source relay-credentials.sh
+source ensure-relay-credentials.sh
test "$(cat $cfg)" = "GARBAGE"
test "$(cat $creds)" = "MOAR GARBAGE"
report_success
diff --git a/install/ensure-relay-credentials.sh b/install/ensure-relay-credentials.sh
new file mode 100644
index 0000000..a6cee64
--- /dev/null
+++ b/install/ensure-relay-credentials.sh
@@ -0,0 +1,42 @@
+echo "${_group}Ensuring Relay credentials ..."
+
+RELAY_CONFIG_YML="../relay/config.yml"
+RELAY_CREDENTIALS_JSON="../relay/credentials.json"
+
+ensure_file_from_example $RELAY_CONFIG_YML
+
+if [[ -f "$RELAY_CREDENTIALS_JSON" ]]; then
+ echo "$RELAY_CREDENTIALS_JSON already exists, skipped creation."
+else
+
+ # There are a couple gotchas here:
+ #
+ # 1. We need to use a tmp file because if we redirect output directly to
+ # credentials.json, then the shell will create an empty file that relay
+ # will then try to read from (regardless of options such as --stdout or
+ # --overwrite) and fail because it is empty.
+ #
+ # 2. We pull relay:nightly before invoking `run relay credentials generate`
+ # because an implicit pull under the run causes extra stdout that results
+ # in a garbage credentials.json.
+ #
+ # 3. We need to use -T to ensure that we receive output on Docker Compose
+ # 1.x and 2.2.3+ (funny story about that ... ;). Note that the long opt
+ # --no-tty doesn't exist in Docker Compose 1.
+
+ $dc pull relay
+ creds="$dcr --no-deps -T relay credentials"
+ $creds generate --stdout > "$RELAY_CREDENTIALS_JSON".tmp
+ mv "$RELAY_CREDENTIALS_JSON".tmp "$RELAY_CREDENTIALS_JSON"
+ if ! grep -q Credentials <($creds show); then
+ # Let's fail early if creds failed, to make debugging easier.
+ echo "Failed to create relay credentials in $RELAY_CREDENTIALS_JSON."
+ echo "--- credentials.json v ---------------------------------------"
+ cat -v "$RELAY_CREDENTIALS_JSON" || true
+ echo "--- credentials.json ^ ---------------------------------------"
+ exit 1
+ fi
+ echo "Relay credentials written to $RELAY_CREDENTIALS_JSON."
+fi
+
+echo "${_endgroup}"
diff --git a/install/relay-credentials.sh b/install/relay-credentials.sh
deleted file mode 100644
index 2d62e2b..0000000
--- a/install/relay-credentials.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-echo "${_group}Generating Relay credentials ..."
-
-RELAY_CONFIG_YML="../relay/config.yml"
-RELAY_CREDENTIALS_JSON="../relay/credentials.json"
-
-ensure_file_from_example $RELAY_CONFIG_YML
-
-if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then
-
- # We need the ugly hack below as `relay generate credentials` tries to read
- # the config and the credentials even with the `--stdout` and `--overwrite`
- # flags and then errors out when the credentials file exists but not valid
- # JSON. We hit this case as we redirect output to the same config folder,
- # creating an empty credentials file before relay runs.
-
- $dcr \
- --no-deps \
- --volume "$(pwd)/$RELAY_CONFIG_YML:/tmp/config.yml" \
- relay --config /tmp credentials generate --stdout \
- > "$RELAY_CREDENTIALS_JSON"
-
- echo "Relay credentials written to $RELAY_CREDENTIALS_JSON"
-fi
-
-echo "${_endgroup}"
diff --git a/install/set-up-and-migrate-database.sh b/install/set-up-and-migrate-database.sh
index 4cb9b70..e1f2412 100644
--- a/install/set-up-and-migrate-database.sh
+++ b/install/set-up-and-migrate-database.sh
@@ -1,15 +1,15 @@
echo "${_group}Setting up / migrating database ..."
if [[ -n "${CI:-}" || "${SKIP_USER_PROMPT:-0}" == 1 ]]; then
$dcr web upgrade --noinput
echo ""
echo "Did not prompt for user creation due to non-interactive shell."
echo "Run the following command to create one yourself (recommended):"
echo ""
- echo " docker compose run --rm web createuser"
+ echo " $dc_base run --rm web createuser"
echo ""
else
$dcr web upgrade
fi
echo "${_endgroup}"

File Metadata

Mime Type
text/x-diff
Expires
Fri, Jul 4, 3:42 PM (1 w, 6 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3308163

Event Timeline