diff --git a/.craft.yml b/.craft.yml index fcc29e7..4524aff 100644 --- a/.craft.yml +++ b/.craft.yml @@ -1,12 +1,6 @@ -minVersion: "0.14.0" -github: - owner: getsentry - repo: onpremise -releaseBranchPrefix: releases -changelogPolicy: none +minVersion: "0.23.1" +changelogPolicy: auto artifactProvider: name: none -statusProvider: - name: github targets: - name: github diff --git a/.env b/.env index 8014f1a..f2e5130 100644 --- a/.env +++ b/.env @@ -1,9 +1,10 @@ COMPOSE_PROJECT_NAME=sentry_onpremise SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.5.1 -SNUBA_IMAGE=getsentry/snuba:21.5.1 -RELAY_IMAGE=getsentry/relay:21.5.1 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 +SENTRY_IMAGE=getsentry/sentry:21.6.3 +SNUBA_IMAGE=getsentry/snuba:21.6.3 +RELAY_IMAGE=getsentry/relay:21.6.3 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 +WAL2JSON_VERSION=latest diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0beb4bb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +/.gitattributes export-ignore +/.gitignore export-ignore +/.github export-ignore +/.editorconfig export-ignore +/.craft.yml export-ignore diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 0000000..b1ec358 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,46 @@ +name: 🐞 Bug Report +description: Tell us about something that's not working the way we (probably) intend. +body: + - type: input + id: version + attributes: + label: Version + placeholder: 21.7.0 ← should look like this (check the footer) + description: What version of self-hosted Sentry are you running? + validations: + required: true + - type: textarea + id: repro + attributes: + label: Steps to Reproduce + description: How can we see what you're seeing? Specific is terrific. + placeholder: |- + 1. foo + 2. bar + 3. baz + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected Result + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual Result + description: Logs? Screenshots? Yes, please. + placeholder: |- + e.g.: + - latest install logs: `ls -1 sentry_install_log-*.txt | tail -1 | xargs cat` + - `docker-compose logs` output + validations: + required: true + - type: markdown + attributes: + value: |- + ## Thanks πŸ™ + Check our [triage docs](https://open.sentry.io/triage/) for what to expect next. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 359d812..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -name: 🐞 Bug Report -about: Report a bug in Self-Hosted Sentry ---- - -### Version Information - -Version: *VERSION HERE* - -### Steps to Reproduce - -1. What -2. you -3. did. - -### Expected Result - -What you thought would happen. - -### Actual Result - -What actually happened. Maybe a screenshot/recording? - -### Logs - -What you saw along the way, e.g.: - -- latest install logs: `ls -1 sentry_install_log-*.txt | tail -1 | xargs cat` -- `docker-compose logs` output diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 0000000..49cfa3e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,28 @@ +name: πŸ’‘ Feature Request +description: Tell us about a problem our software could solve but doesn't. +body: + - type: textarea + id: problem + attributes: + label: Problem Statement + description: What problem could `onpremise` solve that it doesn't? + placeholder: |- + I want to make whirled peas, but `onpremise` doesn't blend. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Solution Brainstorm + description: We know you have bright ideas to share ... share away, friend. + placeholder: |- + Add a blender to `onpremise`. + validations: + required: false + - type: markdown + attributes: + value: |- + ## Thanks πŸ™ + Check our [triage docs](https://open.sentry.io/triage/) for what to expect next. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 32e32f2..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: 🧠 Feature Request -about: Suggest an idea for improving Self-Hosted Sentry ---- - -### Summary - -One paragraph description of the feature. - -### Motivation - -Why should this be worked on? What problems or use cases does it solve or -improve? - -### Additional Context - -Any other context or screenshots or API request payload/responses that -pertain to the feature. diff --git a/.github/workflows/issue-routing-helper.yml b/.github/workflows/issue-routing-helper.yml index 0648d90..22e381c 100644 --- a/.github/workflows/issue-routing-helper.yml +++ b/.github/workflows/issue-routing-helper.yml @@ -1,39 +1,44 @@ name: Issue Routing Helper on: issues: types: [labeled] env: # Use GH_RELEASE_PAT as github-actions bot is not allowed to ping teams GH_TOKEN: ${{ secrets.GH_RELEASE_PAT }} GH_REPO: ${{ github.repository }} jobs: route: runs-on: ubuntu-latest - if: "startsWith(github.event.label.name, 'Team: ')" + if: >- + startsWith(github.event.label.name, 'Team: ') + && + !contains(github.event.issue.labels.*.name, 'Status: Backlog') + && + !contains(github.event.issue.labels.*.name, 'Status: In Progress') steps: - name: "Ensure a single 'Team: *' label with 'Status: Untriaged'" run: | labels_to_remove=$(gh api --paginate "/repos/$GH_REPO/labels" -q '[.[].name | select((startswith("Team: ") or startswith("Status: ")) and . != "${{ github.event.label.name }}" and . != "Status: Untriaged")] | join(",")') gh issue edit ${{ github.event.issue.number }} --remove-label "$labels_to_remove" --add-label '${{ github.event.label.name }},Status: Untriaged' - name: "Mention/ping assigned team for triage" run: | # Get team label mention name: team_label='${{ github.event.label.name }}' team_name="${team_label:6}" # Strip the first 6 chars, which is the 'Team: ' part team_slug="${team_name// /-}" # Replace spaces with hyphens for url/slug friendliness mention_slug=$(gh api "/orgs/getsentry/teams/$team_slug" -q .slug || true) if [[ -z "$mention_slug" ]]; then echo "Couldn't find team mention from slug, trying the label description" team_slug=$(gh api "/repos/$GH_REPO/labels/$team_label" -q '.description') mention_slug=$(gh api "/orgs/getsentry/teams/$team_slug" -q .slug || true) fi if [[ -n "$mention_slug" ]]; then echo "Routing to @getsentry/$mention_slug for [triage](https://develop.sentry.dev/processing-tickets/#3-triage). ⏲️" > comment_body else echo "[Failed]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID) to route to \`${{ github.event.label.name }}\`. πŸ˜•" > comment_body echo "" >> comment_body echo "Defaulting to @getsentry/open-source for [triage](https://develop.sentry.dev/processing-tickets/#3-triage). ⏲️" >> comment_body fi gh issue comment ${{ github.event.issue.number }} --body-file comment_body diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b01e351..5c239db 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,57 +1,57 @@ name: Test on: # Run CI on all pushes to the master and release/** branches, and on all new # pull requests, and on all pushes to pull requests (even if a pull request # is not against master). push: branches: - "master" - - "releases/**" + - "release/**" pull_request: env: DOCKER_COMPOSE_VERSION: 1.24.1 defaults: run: shell: bash jobs: unit-test: runs-on: ubuntu-18.04 name: "unit tests" steps: - name: Checkout uses: actions/checkout@v2 - name: Unit Tests working-directory: install run: find ./ -type f -name "*-test.sh" -exec "./{}" \; integration-test: runs-on: ubuntu-18.04 name: "test" steps: - name: Pin docker-compose run: | sudo rm /usr/local/bin/docker-compose curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose chmod +x docker-compose sudo mv docker-compose /usr/local/bin - name: Checkout uses: actions/checkout@v2 - name: Integration Test run: | echo "Testing initial install" ./install.sh ./test.sh echo "Testing in-place upgrade" # Also test plugin installation here echo "sentry-auth-oidc" >> sentry/requirements.txt ./install.sh --minimize-downtime ./test.sh - name: Inspect failure if: failure() run: | docker-compose ps docker-compose logs diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml deleted file mode 100644 index 328e2e2..0000000 --- a/.github/workflows/validate-new-issue.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: Validate new issue -on: - issues: - types: ["opened"] -jobs: - validate-new-issue: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: "Validate new issue" - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - issue_number=${{ github.event.issue.number }} - echo "Validating issue #${issue_number}." - - # Trust users who belong to the getsentry org. - if gh api "https://api.github.com/orgs/getsentry/members/${{ github.actor }}" >/dev/null 2>&1; then - echo "Skipping validation, because ${{ github.actor }} is a member of the getsentry org." - exit 0 - else - echo "${{ github.actor }} is not a member of the getsentry org. 🧐" - fi - - # Helper - function gh-issue-label() { - gh api "/repos/:owner/:repo/issues/${1}/labels" \ - -X POST \ - --input <(echo "{\"labels\":[\"$2\"]}") - } - - # Prep reasons for error message comment. - REASON="your issue does not properly use one of this repo's available issue templates" - REASON_EXACT_MATCH="you created an issue from a template without filling in anything" - REASON_EMPTY="you created an empty issue" - - BASE_CASE_TITLE="validation bot is confused" - - # Definition of valid: - # - is a report about buggy validation πŸ˜… or ... - # - not empty (ignoring whitespace) - # - matches a template - # - at least one of the headings are also in this issue - # - extra headings in the issue are fine - # - order doesn't matter - # - case-sensitive tho - # - not an *exact* match for a template (ignoring whitespace) - - jq -r .issue.title "$GITHUB_EVENT_PATH" > issue-title - if diff issue-title <(echo "$BASE_CASE_TITLE") > /dev/null; then - echo "Infinite recursion avoided." - exit 0 - fi - - function extract-headings { { sed 's/\r$//' "$1" | grep '^#' || echo -n ''; } | sort; } - jq -r .issue.body "$GITHUB_EVENT_PATH" > issue - if ! grep -q '[^[:space:]]' issue; then - REASON="${REASON_EMPTY}" - else - extract-headings <(cat issue) > headings-in-issue - for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do - # Strip front matter. https://stackoverflow.com/a/29292490/14946704 - sed -i'' '1{/^---$/!q;};1,/^---$/d' "$template" - extract-headings "$template" > headings-in-template - echo -n "$(basename $template)? " - if [ ! -s headings-in-template ]; then - echo "No headers in template. 🀷" - elif [ "$(comm -12 headings-in-template headings-in-issue)" ]; then - echo "Match! πŸ‘ πŸ’ƒ" - if diff -Bw "$template" issue > /dev/null; then - echo "... like, an /exact/ match. πŸ˜–" - REASON="${REASON_EXACT_MATCH}" - break - else - gh-issue-label "${issue_number}" "Status: Unrouted" - exit 0 - fi - else - echo "No match. πŸ‘Ž" - fi - done - fi - - # Failed validation! Close the issue with a comment and a label. - cat << EOF > comment - Sorry, friend. As far as this ol' bot can tell, ${REASON}. Please [try again](https://github.com/${{ github.repository }}/issues/new/choose), if you like. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=$(echo "$BASE_CASE_TITLE" | tr ' ' '+')&body=${{ github.event.issue.html_url }}). 😬) - - ---- - - [![Did you see the memo about this?](https://user-images.githubusercontent.com/134455/104515469-e04a9c80-55c0-11eb-8e15-ffe9c0b8dd7f.gif)](https://www.youtube.com/watch?v=Fy3rjQGc6lA) - - ([log](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})) - EOF - - echo -n "Commented: " - gh issue comment "${issue_number}" --body "$(cat comment)" - gh-issue-label "${issue_number}" "Status: Invalid" - gh issue close "${issue_number}" - echo "Closed with: \"${REASON}.\"" diff --git a/.gitignore b/.gitignore index 707622f..8a16904 100644 --- a/.gitignore +++ b/.gitignore @@ -1,86 +1,89 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt sentry_install_log*.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Sphinx documentation docs/_build/ # PyBuilder target/ # Ipython Notebook .ipynb_checkpoints # pyenv .python-version # https://docs.docker.com/compose/extends/ docker-compose.override.yml *.tar data/ .vscode/tags # custom Sentry config sentry/sentry.conf.py sentry/config.yml sentry/*.bak sentry/requirements.txt relay/credentials.json relay/config.yml symbolicator/config.yml geoip/GeoIP.conf geoip/*.mmdb geoip/.geoipupdate.lock + +# wal2json download +postgres/wal2json diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4929c52 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +## 21.6.3 + +- No documented changes. + +## 21.6.2 + +- BREAKING CHANGE: The frontend bundle will be loaded asynchronously (via [#25744](https://github.com/getsentry/sentry/pull/25744)). This is a breaking change that can affect custom plugins that access certain globals in the django template. Please see https://forum.sentry.io/t/breaking-frontend-changes-for-custom-plugins/14184 for more information. + +## 21.6.1 + +- No documented changes. + +## 21.6.0 + +- feat: Add healthchecks for redis, memcached and postgres (#975) + diff --git a/LICENSE b/LICENSE index 67c62a8..8004839 100644 --- a/LICENSE +++ b/LICENSE @@ -1,104 +1,104 @@ Business Source License 1.1 Parameters Licensor: Functional Software, Inc. Licensed Work: Sentry The Licensed Work is (c) 2019 Functional Software, Inc. Additional Use Grant: You may make use of the Licensed Work, provided that you do not use the Licensed Work for an Application Monitoring Service. An "Application Monitoring Service" is a commercial offering that allows third parties (other than your employees and contractors) to access the functionality of the Licensed Work so that such third parties directly benefit from the error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-05-19 +Change Date: 2024-07-08 Change License: Apache License, Version 2.0 For information about alternative licensing arrangements for the Software, please visit: https://sentry.io/pricing/ Notice The Business Source License (this document, or the "License") is not an Open Source license. However, the Licensed Work will eventually be made available under an Open Source License, as stated in this License. License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. "Business Source License" is a trademark of MariaDB Corporation Ab. ----------------------------------------------------------------------------- Business Source License 1.1 Terms The Licensor hereby grants you the right to copy, modify, create derivative works, redistribute, and make non-production use of the Licensed Work. The Licensor may make an Additional Use Grant, above, permitting limited production use. Effective on the Change Date, or the fourth anniversary of the first publicly available distribution of a specific version of the Licensed Work under this License, whichever comes first, the Licensor hereby grants you rights under the terms of the Change License, and the rights granted in the paragraph above terminate. If your use of the Licensed Work does not comply with the requirements currently in effect as described in this License, you must purchase a commercial license from the Licensor, its affiliated entities, or authorized resellers, or you must refrain from using the Licensed Work. All copies of the original and modified Licensed Work, and derivative works of the Licensed Work, are subject to this License. This License applies separately for each version of the Licensed Work and the Change Date may vary for each version of the Licensed Work released by Licensor. You must conspicuously display this License on each original or modified copy of the Licensed Work. If you receive the Licensed Work in original or modified form from a third party, the terms and conditions set forth in this License apply to your use of that work. Any use of the Licensed Work in violation of this License will automatically terminate your rights under this License for the current and all other versions of the Licensed Work. This License does not grant you any right in any trademark or logo of Licensor or its affiliates (provided that you may use a trademark or logo of Licensor as expressly required by this License). TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND TITLE. MariaDB hereby grants you permission to use this License’s text to license your works, and to refer to it using the trademark "Business Source License", as long as you comply with the Covenants of Licensor below. Covenants of Licensor In consideration of the right to use this License’s text and the "Business Source License" name and trademark, Licensor covenants to MariaDB, and to all other recipients of the licensed work to be provided by Licensor: 1. To specify as the Change License the GPL Version 2.0 or any later version, or a license that is compatible with GPL Version 2.0 or a later version, where "compatible" means that software provided under the Change License can be included in a program with software provided under GPL Version 2.0 or a later version. Licensor may specify additional Change Licenses without limitation. 2. To either: (a) specify an additional grant of rights to use that does not impose any additional restriction on the right granted in this License, as the Additional Use Grant; or (b) insert the text "None". 3. To specify a Change Date. 4. Not to modify this License in any other way. diff --git a/README.md b/README.md index 067f723..79f7552 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,63 @@ -# Self-Hosted Sentry 21.5.1 +# Self-Hosted Sentry 21.6.3 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). ## Requirements - * Docker 19.03.6+ - * Compose 1.24.1+ - * 4 CPU Cores - * 8 GB RAM - * 20 GB Free Disk Space +* Docker 19.03.6+ +* Compose 1.24.1+ +* 4 CPU Cores +* 8 GB RAM +* 20 GB Free Disk Space ## Setup To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020 and Sentry 21.1.0 is the last version to support Python 2. During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --no-user-prompt`. -There may need to be modifications to the included example config files (`sentry/config.example.yml` and `sentry/sentry.conf.example.py`) to accommodate your needs or your environment (such as adding GitHub credentials). If you want to perform these, do them before you run the install script and copy them without the `.example` extensions in the name (such as `sentry/sentry.conf.py`) before running the `install.sh` script. +Please visit [our documentation](https://develop.sentry.dev/self-hosted/) for everything else. -The recommended way to customize your configuration is using the files below, in that order: +## Tips & Tricks - * `config.yml` - * `sentry.conf.py` - * `.env` w/ environment variables +### Event Retention -We currently support a very minimal set of environment variables to promote other means of configuration. - -If you have any issues or questions, our [Community Forum](https://forum.sentry.io/c/on-premise) is at your service! Everytime you run the install script, it will generate a log file, `sentry_install_log-.txt` with the output. Sharing these logs would help people diagnose any issues you might be having. +Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file. -## Versioning +### Installing a specific SHA If you want to install a specific release of Sentry, use the tags/releases on this repo. We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: ```shell SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh ``` Note that this may not work for all commit SHAs as this repository evolves with Sentry and its satellite projects. It is highly recommended to check out a version of this repository that is close to the timestamp of the Sentry commit you are installing. ### Using Linux If you are using Linux and you need to use `sudo` when running `./install.sh`, modifying the version of Sentry is slightly different. First, run the following: + ```shell sudo visudo ``` + Then add the following line: + ```shell Defaults env_keep += "SENTRY_IMAGE" ``` + Save the file then in your terminal run the following ```shell export SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 sudo ./install.sh ``` -Where you replace `83b1380` with the sha you want to use. - -## Event Retention - -Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file. - -## Securing Sentry with SSL/TLS - -If you'd like to protect your Sentry install with SSL/TLS, there are -fantastic SSL/TLS proxies like [HAProxy](http://www.haproxy.org/) -and [Nginx](http://nginx.org/). Our recommendation is running an external Nginx instance or your choice of load balancer that does the TLS termination and more. Read more over at our [productionalizing self-hosted docs](https://develop.sentry.dev/self-hosted/#productionalizing). - -## Updating Sentry - -_You need to be on at least Sentry 9.1.2 to be able to upgrade automatically to the latest version. If you are not, upgrade to 9.1.2 first by checking out the [9.1.2 tag](https://github.com/getsentry/onpremise/tree/9.1.2) on this repo._ - -We recommend (and sometimes require) you to upgrade Sentry one version at a time. That means if you are running 20.6.0, instead of going directly to 20.8.0, first go through 20.7.0. Skipping versions would work most of the time, but there will be times that we require you to stop at specific versions to ensure essential data migrations along the way. - -Pull the version of the repository that you wish to upgrade to by checking out the tagged release of this repo. Make sure to check for any difference between the example config files and your current config files in use. There might be new configuration that has to be added to your adjusted files such as feature flags or server configuration. - -The included `install.sh` script is meant to be idempotent and to bring you to the latest version. What this means is you can and should run `install.sh` to upgrade to the latest version available. Remember that the output of the script will be stored in a log file, `sentry_install_log-.txt`, which you may share for diagnosis if anything goes wrong. - -For more information regarding updating your Sentry installation, please visit [our documentation](https://develop.sentry.dev/self-hosted/#upgrading). - -## Resources - - * [Documentation](https://develop.sentry.dev/self-hosted/) - * [Bug Tracker](https://github.com/getsentry/onpremise/issues) - * [Community Forums](https://forum.sentry.io/c/on-premise) +Where you replace `83b1380` with the sha you want to use. [build-status-image]: https://github.com/getsentry/onpremise/workflows/test/badge.svg [build-status-url]: https://git.io/JUYkh diff --git a/docker-compose.yml b/docker-compose.yml index 318ce64..a494f77 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,275 +1,289 @@ version: "3.4" x-restart-policy: &restart_policy restart: unless-stopped +x-healthcheck-defaults: &healthcheck_defaults + # Avoid setting the interval too small, as docker uses much more CPU than one would expect. + # Related issues: + # https://github.com/moby/moby/issues/39102 + # https://github.com/moby/moby/issues/39388 + # https://github.com/getsentry/onpremise/issues/1000 + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s x-sentry-defaults: &sentry_defaults <<: *restart_policy image: "$SENTRY_IMAGE" depends_on: - redis - memcached - snuba-api - snuba-consumer - snuba-outcomes-consumer - snuba-sessions-consumer - snuba-transactions-consumer - snuba-subscription-consumer-events - snuba-subscription-consumer-transactions - snuba-replacer - symbolicator - kafka entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: PYTHONUSERBASE: "/data/custom-packages" SENTRY_CONF: "/etc/sentry" SNUBA: "http://snuba-api:1218" # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: volumes: - "sentry-data:/data" - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: - redis - clickhouse - kafka image: "$SNUBA_IMAGE" environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse DEFAULT_BROKERS: "kafka:9092" REDIS_HOST: redis UWSGI_MAX_REQUESTS: "10000" UWSGI_DISABLE_LOGGING: "true" # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: services: memcached: <<: *restart_policy - image: "memcached:1.5-alpine" + image: "memcached:1.6.9-alpine" + healthcheck: + <<: *healthcheck_defaults + # From: https://stackoverflow.com/a/31877626/5155484 + test: echo stats | nc 127.0.0.1 11211 redis: <<: *restart_policy - image: "redis:5.0-alpine" + image: "redis:6.2.4-alpine" + healthcheck: + <<: *healthcheck_defaults + test: redis-cli ping volumes: - "sentry-redis:/data" ulimits: nofile: soft: 10032 hard: 10032 zookeeper: <<: *restart_policy image: "confluentinc/cp-zookeeper:5.5.0" environment: ZOOKEEPER_CLIENT_PORT: "2181" CONFLUENT_SUPPORT_METRICS_ENABLE: "false" ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN" ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN" KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok" volumes: - "sentry-zookeeper:/var/lib/zookeeper/data" - "sentry-zookeeper-log:/var/lib/zookeeper/log" - "sentry-secrets:/etc/zookeeper/secrets" healthcheck: - test: ["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] - interval: 10s - timeout: 5s - retries: 6 + <<: *healthcheck_defaults + test: + ["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] kafka: <<: *restart_policy depends_on: - zookeeper image: "confluentinc/cp-kafka:5.5.0" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1" KAFKA_LOG_RETENTION_HOURS: "24" KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too CONFLUENT_SUPPORT_METRICS_ENABLE: "false" KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN" KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN" KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" volumes: - "sentry-kafka:/var/lib/kafka/data" - "sentry-kafka-log:/var/lib/kafka/log" - "sentry-secrets:/etc/kafka/secrets" healthcheck: - test: ["CMD-SHELL", 'nc -z localhost 9092'] - interval: 10s - timeout: 5s - retries: 6 + <<: *healthcheck_defaults + test: ["CMD-SHELL", "nc -z localhost 9092"] clickhouse: <<: *restart_policy image: "yandex/clickhouse-server:20.3.9.70" ulimits: nofile: soft: 262144 hard: 262144 volumes: - "sentry-clickhouse:/var/lib/clickhouse" - "sentry-clickhouse-log:/var/log/clickhouse-server" - type: bind read_only: true source: ./clickhouse/config.xml target: /etc/clickhouse-server/config.d/sentry.xml environment: # This limits Clickhouse's memory to 30% of the host memory # If you have high volume and your search return incomplete results # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 geoipupdate: - image: "maxmindinc/geoipupdate:latest" + image: "maxmindinc/geoipupdate:v4.7.1" # Override the entrypoint in order to avoid using envvars for config. # Futz with settings so we can keep mmdb and conf in same dir on host # (image looks for them in separate dirs by default). entrypoint: ["/usr/bin/geoipupdate", "-d", "/sentry", "-f", "/sentry/GeoIP.conf"] volumes: - "./geoip:/sentry" snuba-api: <<: *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse snuba-consumer: <<: *snuba_defaults command: consumer --storage errors --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration snuba-outcomes-consumer: <<: *snuba_defaults command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750 # Kafka consumer responsible for feeding session data into Clickhouse snuba-sessions-consumer: <<: *snuba_defaults command: consumer --storage sessions_raw --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: <<: *snuba_defaults command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --commit-log-topic=snuba-commit-log snuba-replacer: <<: *snuba_defaults command: replacer --storage errors --auto-offset-reset=latest --max-batch-size 3 snuba-subscription-consumer-events: <<: *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-events-subscriptions-consumers --topic=events --result-topic=events-subscription-results --dataset=events --commit-log-topic=snuba-commit-log --commit-log-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60 snuba-subscription-consumer-transactions: <<: *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-transactions-subscriptions-consumers --topic=events --result-topic=transactions-subscription-results --dataset=transactions --commit-log-topic=snuba-commit-log --commit-log-group=transactions_group --delay-seconds=60 --schedule-ttl=60 snuba-cleanup: <<: *snuba_defaults image: snuba-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * gosu snuba snuba cleanup --storage errors --dry-run False"' snuba-transactions-cleanup: <<: *snuba_defaults image: snuba-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * gosu snuba snuba cleanup --storage transactions --dry-run False"' symbolicator: <<: *restart_policy image: "$SYMBOLICATOR_IMAGE" volumes: - "sentry-symbolicator:/data" - type: bind read_only: true source: ./symbolicator target: /etc/symbolicator command: run -c /etc/symbolicator/config.yml symbolicator-cleanup: <<: *restart_policy image: symbolicator-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: "$SYMBOLICATOR_IMAGE" command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' volumes: - "sentry-symbolicator:/data" web: <<: *sentry_defaults cron: <<: *sentry_defaults command: run cron worker: <<: *sentry_defaults command: run worker ingest-consumer: <<: *sentry_defaults command: run ingest-consumer --all-consumer-types post-process-forwarder: <<: *sentry_defaults # Increase `--commit-batch-size 1` below to deal with high-load environments. command: run post-process-forwarder --commit-batch-size 1 subscription-consumer-events: <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic events-subscription-results subscription-consumer-transactions: <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic transactions-subscription-results sentry-cleanup: <<: *sentry_defaults image: sentry-cleanup-onpremise-local build: context: ./cron args: BASE_IMAGE: "$SENTRY_IMAGE" entrypoint: "/entrypoint.sh" command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: <<: *restart_policy ports: - "$SENTRY_BIND:80/tcp" - image: "nginx:1.16" + image: "nginx:1.21.0-alpine" volumes: - type: bind read_only: true source: ./nginx target: /etc/nginx depends_on: - web - relay relay: <<: *restart_policy image: "$RELAY_IMAGE" volumes: - type: bind read_only: true source: ./relay target: /work/.relay - type: bind read_only: true source: ./geoip target: /geoip depends_on: - kafka - redis - web volumes: sentry-data: external: true sentry-redis: external: true sentry-zookeeper: external: true sentry-kafka: external: true sentry-clickhouse: external: true sentry-symbolicator: external: true sentry-secrets: sentry-zookeeper-log: sentry-kafka-log: sentry-clickhouse-log: diff --git a/install.sh b/install.sh index 171851b..3886463 100755 --- a/install.sh +++ b/install.sh @@ -1,28 +1,29 @@ #!/usr/bin/env bash set -e if [[ -n "$MSYSTEM" ]]; then echo "Seems like you are using an MSYS2-based system (such as Git Bash) which is not supported. Please use WSL instead."; exit 1 fi source "$(dirname $0)/install/_lib.sh" # does a `cd .../install/`, among other things source parse-cli.sh source error-handling.sh source check-minimum-requirements.sh source create-docker-volumes.sh source ensure-files-from-examples.sh source generate-secret-key.sh source replace-tsdb.sh source update-docker-images.sh source build-docker-images.sh source turn-things-off.sh source set-up-zookeeper.sh +source install-wal2json.sh source bootstrap-snuba.sh source create-kafka-topics.sh source upgrade-postgres.sh source set-up-and-migrate-database.sh source migrate-file-storage.sh source relay-credentials.sh source geoip.sh source wrap-up.sh diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh new file mode 100644 index 0000000..47e390e --- /dev/null +++ b/install/install-wal2json.sh @@ -0,0 +1,38 @@ +echo "${_group}Downloading and installing wal2json ..." + +FILE_TO_USE="../postgres/wal2json/wal2json.so" +ARCH=$(uname -m) +FILE_NAME="wal2json-Linux-$ARCH-glibc.so" + +docker_curl() { + # The environment variables can be specified in lower case or upper case. + # The lower case version has precedence. http_proxy is an exception as it is only available in lower case. + docker run --rm -e http_proxy -e https_proxy -e HTTPS_PROXY -e no_proxy -e NO_PROXY curlimages/curl "$@" +} + +if [[ $WAL2JSON_VERSION == "latest" ]]; then + VERSION=$( + docker_curl https://api.github.com/repos/getsentry/wal2json/releases/latest | + grep '"tag_name":' | + sed -E 's/.*"([^"]+)".*/\1/' + ) + + if [[ ! $VERSION ]]; then + echo "Cannot find wal2json latest version" + exit 1 + fi +else + VERSION=$WAL2JSON_VERSION +fi + +mkdir -p ../postgres/wal2json +if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then + mkdir -p "../postgres/wal2json/$VERSION" + docker_curl -L \ + "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ + > "../postgres/wal2json/$VERSION/$FILE_NAME" +fi +cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" + + +echo "${_endgroup}" diff --git a/install/wrap-up.sh b/install/wrap-up.sh index 2671a3a..bbe08fa 100644 --- a/install/wrap-up.sh +++ b/install/wrap-up.sh @@ -1,25 +1,25 @@ if [[ "$MINIMIZE_DOWNTIME" ]]; then echo "${_group}Waiting for Sentry to start ..." # Start the whole setup, except nginx and relay. $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') - $dc exec -T nginx service nginx reload + $dc exec -T nginx nginx -s reload docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' # Make sure everything is up. This should only touch relay and nginx $dc up -d echo "${_endgroup}" else echo "" echo "-----------------------------------------------------------------" echo "" echo "You're all done! Run the following command to get Sentry running:" echo "" echo " docker-compose up -d" echo "" echo "-----------------------------------------------------------------" echo "" fi diff --git a/postgres/init_hba.sh b/postgres/init_hba.sh new file mode 100755 index 0000000..f4b332a --- /dev/null +++ b/postgres/init_hba.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Initializes the pg_hba file with access permissions to the replication +# slots. + +set -e + +{ echo "host replication all all trust"; } >> "$PGDATA/pg_hba.conf" diff --git a/postgres/postgres-entrypoint.sh b/postgres/postgres-entrypoint.sh new file mode 100755 index 0000000..0b0d98a --- /dev/null +++ b/postgres/postgres-entrypoint.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# This script replaces the default docker entrypoint for postgres in the +# development environment. +# Its job is to ensure postgres is properly configured to support the +# Change Data Capture pipeline (by setting access permissions and installing +# the replication plugin we use for CDC). Unfortunately the default +# Postgres image does not allow this level of configurability so we need +# to do it this way in order not to have to publish and maintain our own +# Postgres image. +# +# This then, at the end, transfers control to the default entrypoint. + +set -e + +prep_init_db() { + cp /opt/sentry/init_hba.sh /docker-entrypoint-initdb.d/init_hba.sh +} + +cdc_setup_hba_conf() { + # Ensure pg-hba is properly configured to allow connections + # to the replication slots. + + PG_HBA="$PGDATA/pg_hba.conf" + if [ ! -f "$PG_HBA" ]; then + echo "DB not initialized. Postgres will take care of pg_hba" + elif [ "$(grep -c -E "^host\s+replication" "$PGDATA"/pg_hba.conf)" != 0 ]; then + echo "Replication config already present in pg_hba. Not changing anything." + else + # Execute the same script we run on DB initialization + /opt/sentry/init_hba.sh + fi +} + +bind_wal2json() { + # Copy the file in the right place + cp /opt/sentry/wal2json/wal2json.so `pg_config --pkglibdir`/wal2json.so +} + +echo "Setting up Change Data Capture" + +prep_init_db +if [ "$1" = 'postgres' ]; then + cdc_setup_hba_conf + bind_wal2json +fi +exec /docker-entrypoint.sh "$@" diff --git a/reset.sh b/reset.sh new file mode 100755 index 0000000..0e4db0e --- /dev/null +++ b/reset.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# The purpose of this script is to make it easy to reset a local onpremise +# install to a clean state, optionally targeting a particular version. + +set -euo pipefail + +if [ -n "${DEBUG:-}" ]; then + set -x +fi + +cd "$(dirname $0)" + + +function confirm () { + read -p "$1 [y/n] " confirmation + if [ "$confirmation" != "y" ]; then + echo "Canceled. πŸ˜…" + exit + fi +} + + +# If we have a version given, validate it. +# ---------------------------------------- +# Note that arbitrary git refs won't work, because the *_IMAGE variables in +# .env will almost certainly point to :latest. Tagged releases are generally +# the only refs where these component versions are pinned, so enforce that +# we're targeting a valid tag here. Do this early in order to fail fast. + +version="${1:-}" +if [ -n "$version" ]; then + set +e + git rev-parse --verify --quiet "refs/tags/$version" > /dev/null + if [ $? -gt 0 ]; then + echo "Bad version: $version" + exit + fi + set -e +fi + +# Make sure they mean it. +confirm "☠️ Warning! 😳 This is highly destructive! 😱 Are you sure you wish to proceed?" +echo "Okay ... good luck! 😰" + +# Hit the reset button. +docker-compose down --volumes --remove-orphans --rmi local + +# Remove any remaining (likely external) volumes with name matching 'sentry-.*'. +for volume in $(docker volume list --format '{{ .Name }}' | grep '^sentry-'); do + docker volume remove $volume > /dev/null \ + && echo "Removed volume: $volume" \ + || echo "Skipped volume: $volume" +done + +# If we have a version given, switch to it. +if [ -n "$version" ]; then + git checkout "$version" +fi + +# Install. +exec ./install.sh diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 4c8bb5a..b5d4586 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -1,18 +1,21 @@ #!/bin/bash set -eu SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $SCRIPT_DIR/.. OLD_VERSION="$1" NEW_VERSION="$2" SYMBOLICATOR_VERSION=${SYMBOLICATOR_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/symbolicator/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} +WAL2JSON_VERSION=${WAL2JSON_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/wal2json/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} sed -i -e "s/^SYMBOLICATOR_IMAGE=\([^:]\+\):.\+\$/SYMBOLICATOR_IMAGE=\1:$SYMBOLICATOR_VERSION/" .env +sed -i -e "s/^WAL2JSON_VERSION=\([^:]\+\):.\+\$/WAL2JSON_VERSION=\1:$WAL2JSON_VERSION/" .env sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE echo "New version: $NEW_VERSION" echo "New Symbolicator version: $SYMBOLICATOR_VERSION" +echo "New wal2json version: $WAL2JSON_VERSION"