diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4069959 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +insert_final_newline = true + +[*.sh] +indent_size = 2 + +[*.yml] +indent_size = 2 + +[nginx/*.conf] +indent_style = tab diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1d2890..fc3e3ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,67 +1,67 @@ name: release on: workflow_dispatch: inputs: version: description: Version to release required: false skip_prepare: description: Skip preparation step (assume a release branch is ready) required: false default: false dry_run: description: Do not actually cut the release required: false default: false force: description: Force the release, bypassing the 'release-blocker' issue killswitch required: false default: false schedule: # We want the release to be at 10 or 11am Pacific Time # We also make this an hour after all others such as Sentry, # Snuba, and Relay to make sure their releases finish. - cron: '0 18 15 * *' jobs: release: runs-on: ubuntu-latest name: "Release a new version" steps: - - id: killswitch - if: ${{ !github.event.inputs.force }} - run: | - if curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY/issues?state=open&labels=release-blocker" | grep -Pzvo '\[[\s\n\r]*\]'; then - echo "Open release-blocking issues found, cancelling release..."; - curl -sf -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel; - fi - - id: calver - if: ${{ !github.event.inputs.version }} - run: | - DATE_PART=$(date +'%y.%-m') - declare -i PATCH_VERSION=0 - while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do - PATCH_VERSION+=1 - done - echo "::set-output name=version::$DATE_PART.$PATCH_VERSION" - - uses: actions/checkout@v2 - - uses: getsentry/craft@master - if: ${{ !github.event.inputs.skip_prepare }} - with: - action: prepare - version: ${{ github.event.inputs.version || steps.calver.outputs.version }} - env: - DRY_RUN: ${{ github.event.inputs.dry_run }} - GIT_COMMITTER_NAME: getsentry-bot - GIT_AUTHOR_NAME: getsentry-bot - EMAIL: bot@getsentry.com - - uses: getsentry/craft@master - with: - action: publish - version: ${{ github.event.inputs.version || steps.calver.outputs.version }} - keep_branch: '--keep-branch' - no_merge: '--no-merge' - env: - DRY_RUN: ${{ github.event.inputs.dry_run }} - GIT_COMMITTER_NAME: getsentry-bot - GIT_AUTHOR_NAME: getsentry-bot - EMAIL: bot@getsentry.com + - id: killswitch + if: ${{ !github.event.inputs.force }} + run: | + if curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY/issues?state=open&labels=release-blocker" | grep -Pzvo '\[[\s\n\r]*\]'; then + echo "Open release-blocking issues found, cancelling release..."; + curl -sf -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel; + fi + - id: calver + if: ${{ !github.event.inputs.version }} + run: | + DATE_PART=$(date +'%y.%-m') + declare -i PATCH_VERSION=0 + while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do + PATCH_VERSION+=1 + done + echo "::set-output name=version::$DATE_PART.$PATCH_VERSION" + - uses: actions/checkout@v2 + - uses: getsentry/craft@master + if: ${{ !github.event.inputs.skip_prepare }} + with: + action: prepare + version: ${{ github.event.inputs.version || steps.calver.outputs.version }} + env: + DRY_RUN: ${{ github.event.inputs.dry_run }} + GIT_COMMITTER_NAME: getsentry-bot + GIT_AUTHOR_NAME: getsentry-bot + EMAIL: bot@getsentry.com + - uses: getsentry/craft@master + with: + action: publish + version: ${{ github.event.inputs.version || steps.calver.outputs.version }} + keep_branch: '--keep-branch' + no_merge: '--no-merge' + env: + DRY_RUN: ${{ github.event.inputs.dry_run }} + GIT_COMMITTER_NAME: getsentry-bot + GIT_AUTHOR_NAME: getsentry-bot + EMAIL: bot@getsentry.com diff --git a/cron/entrypoint.sh b/cron/entrypoint.sh index b0f4d5b..baa833a 100755 --- a/cron/entrypoint.sh +++ b/cron/entrypoint.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash # Prior art: # - https://git.io/fjNOg # - https://blog.knoldus.com/running-a-cron-job-in-docker-container/ declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env { for cron_job in "$@"; do echo -e "SHELL=/bin/bash BASH_ENV=/container.env ${cron_job} > /proc/1/fd/1 2>/proc/1/fd/2"; done } \ - | sed --regexp-extended 's/\\(.)/\1/g' \ - | crontab - + | sed --regexp-extended 's/\\(.)/\1/g' \ + | crontab - crontab -l exec cron -f -l -L 15 diff --git a/install.sh b/install.sh index 19d3b8b..bd3587a 100755 --- a/install.sh +++ b/install.sh @@ -1,356 +1,356 @@ #!/usr/bin/env bash set -e # With a tip o' the hat to https://unix.stackexchange.com/a/79077 set -a && . ./.env && set +a dc="docker-compose --no-ansi" dcr="$dc run --rm" # Thanks to https://unix.stackexchange.com/a/145654/108960 log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") MIN_DOCKER_VERSION='17.05.0' MIN_COMPOSE_VERSION='1.23.0' MIN_RAM=2400 # MB SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' SYMBOLICATOR_CONFIG_YML='symbolicator/config.yml' RELAY_CONFIG_YML='relay/config.yml' RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' MINIMIZE_DOWNTIME= load_options() { while [[ -n "$@" ]]; do case "$1" in - -h | --help) show_help; exit;; - --no-user-prompt) SKIP_USER_PROMPT=1;; - --minimize-downtime) MINIMIZE_DOWNTIME=1;; - --) ;; - *) echo "Unexpected argument: $1. Use --help for usage information."; exit 1;; + -h | --help) show_help; exit;; + --no-user-prompt) SKIP_USER_PROMPT=1;; + --minimize-downtime) MINIMIZE_DOWNTIME=1;; + --) ;; + *) echo "Unexpected argument: $1. Use --help for usage information."; exit 1;; esac shift done } show_help() { cat < /dev/null fi } trap_with_arg cleanup ERR INT TERM EXIT echo "Checking minimum requirements..." DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') COMPOSE_VERSION=$($dc --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); # Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } # Thanks to https://stackoverflow.com/a/25123013/90297 for the quick `sed` pattern function ensure_file_from_example { if [ -f "$1" ]; then echo "$1 already exists, skipped creation." else echo "Creating $1..." cp -n $(echo "$1" | sed 's/\.[^.]*$/.example&/') "$1" fi } if [ $(ver $DOCKER_VERSION) -lt $(ver $MIN_DOCKER_VERSION) ]; then - echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" - exit 1 + echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" + exit 1 fi if [ $(ver $COMPOSE_VERSION) -lt $(ver $MIN_COMPOSE_VERSION) ]; then - echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" - exit 1 + echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" + exit 1 fi if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then - echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" - exit 1 + echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" + exit 1 fi #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) # On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297 IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :) if (($IS_KVM == 0)); then SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :) if (($SUPPORTS_SSE42 == 0)); then echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." exit 1 fi fi echo "" echo "Creating volumes for persistent storage..." echo "Created $(docker volume create --name=sentry-data)." echo "Created $(docker volume create --name=sentry-postgres)." echo "Created $(docker volume create --name=sentry-redis)." echo "Created $(docker volume create --name=sentry-zookeeper)." echo "Created $(docker volume create --name=sentry-kafka)." echo "Created $(docker volume create --name=sentry-clickhouse)." echo "Created $(docker volume create --name=sentry-symbolicator)." echo "" ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS ensure_file_from_example $SYMBOLICATOR_CONFIG_YML ensure_file_from_example $RELAY_CONFIG_YML if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then - echo "" - echo "Generating secret key..." - # This is to escape the secret key to be used in sed below - # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode - # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 - SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') - sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML - echo "Secret key written to $SENTRY_CONFIG_YML" + echo "" + echo "Generating secret key..." + # This is to escape the secret key to be used in sed below + # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode + # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 + SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') + sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML + echo "Secret key written to $SENTRY_CONFIG_YML" fi replace_tsdb() { if ( [ -f "$SENTRY_CONFIG_PY" ] && ! grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY" ); then # Do NOT indent the following string as it would be reflected in the end result, # breaking the final config file. See getsentry/onpremise#624. tsdb_settings="\ SENTRY_TSDB = \"sentry.tsdb.redissnuba.RedisSnubaTSDB\" # Automatic switchover 90 days after $(date). Can be removed afterwards. SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)}\ " if grep -q 'SENTRY_TSDB_OPTIONS = ' "$SENTRY_CONFIG_PY"; then echo "Not attempting automatic TSDB migration due to presence of SENTRY_TSDB_OPTIONS" else echo "Attempting to automatically migrate to new TSDB" # Escape newlines for sed tsdb_settings="${tsdb_settings//$'\n'/\\n}" cp "$SENTRY_CONFIG_PY" "$SENTRY_CONFIG_PY.bak" sed -i -e "s/^SENTRY_TSDB = .*$/${tsdb_settings}/g" "$SENTRY_CONFIG_PY" || true if grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY"; then echo "Migrated TSDB to Snuba. Old configuration file backed up to $SENTRY_CONFIG_PY.bak" return fi echo "Failed to automatically migrate TSDB. Reverting..." mv "$SENTRY_CONFIG_PY.bak" "$SENTRY_CONFIG_PY" echo "$SENTRY_CONFIG_PY restored from backup." fi echo "WARN: Your Sentry configuration uses a legacy data store for time-series data. Remove the options SENTRY_TSDB and SENTRY_TSDB_OPTIONS from $SENTRY_CONFIG_PY and add:" echo "" echo "$tsdb_settings" echo "" echo "For more information please refer to https://github.com/getsentry/onpremise/pull/430" fi } replace_tsdb echo "" echo "Fetching and updating Docker images..." echo "" # We tag locally built images with an '-onpremise-local' suffix. docker-compose pull tries to pull these too and # shows a 404 error on the console which is confusing and unnecessary. To overcome this, we add the stderr>stdout # redirection below and pass it through grep, ignoring all lines having this '-onpremise-local' suffix. $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true # We may not have the set image on the repo (local images) so allow fails docker pull $SENTRY_IMAGE || true; echo "" echo "Building and tagging Docker images..." echo "" # Build the sentry onpremise image first as it is needed for the cron image $dc build --force-rm web $dc build --force-rm --parallel echo "" echo "Docker images built." if [[ "$MINIMIZE_DOWNTIME" ]]; then # Stop everything but relay and nginx $dc rm -fsv $($dc config --services | grep -v -E '^(nginx|relay)$') else # Clean up old stuff and ensure nothing is working while we install/update # This is for older versions of on-premise: $dc -p onpremise down --rmi local --remove-orphans # This is for newer versions $dc down --rmi local --remove-orphans fi ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') if [ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq "1" ]; then ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') # This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 if [ "$ZOOKEEPER_LOG_FILE_COUNT" -gt "0" ] && [ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq "0" ]; then $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper fi fi # [begin] Snuba/Clickhouse transactions table rebuild clickhouse_query () { $dcr clickhouse clickhouse-client --host clickhouse -q "$1"; } $dc up -d clickhouse set +e CLICKHOUSE_CLIENT_MAX_RETRY=5 # Wait until clickhouse server is up until clickhouse_query 'SELECT 1' > /dev/null; do ((CLICKHOUSE_CLIENT_MAX_RETRY--)) [[ CLICKHOUSE_CLIENT_MAX_RETRY -eq 0 ]] && echo "Clickhouse server failed to come up in 5 tries." && exit 1; - echo "Trying again. Remaining tries #$CLICKHOUSE_CLIENT_MAX_RETRY" + echo "Trying again. Remaining tries #$CLICKHOUSE_CLIENT_MAX_RETRY" sleep 0.5; done set -e SNUBA_HAS_TRANSACTIONS_TABLE=$(clickhouse_query 'EXISTS TABLE transactions_local' | tr -d '\n\r') SNUBA_TRANSACTIONS_NEEDS_UPDATE=$([ "$SNUBA_HAS_TRANSACTIONS_TABLE" == "1" ] && clickhouse_query 'SHOW CREATE TABLE transactions_local' | grep -v 'SAMPLE BY' || echo '') if [ "$SNUBA_TRANSACTIONS_NEEDS_UPDATE" ]; then SNUBA_TRANSACTIONS_TABLE_CONTENTS=$(clickhouse_query "SELECT * FROM transactions_local LIMIT 1") if [ -z $SNUBA_TRANSACTIONS_TABLE_CONTENTS ]; then echo "Dropping the old transactions table from Clickhouse..."; clickhouse_query 'DROP TABLE transactions_local' echo "Done." else echo "Seems like your Clickhouse transactions table is old and non-empty. You may experience issues if/when you have more than 10000 records in this table. See https://github.com/getsentry/sentry/pull/19882 for more information and consider disabling the 'discover2.tags_facet_enable_sampling' feature flag."; fi fi # [end] Snuba/Clickhouse transactions table rebuild echo "Bootstrapping and migrating Snuba..." $dcr snuba-api bootstrap --force echo "" # Very naively check whether there's an existing sentry-postgres volume and the PG version in it if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then - docker volume rm sentry-postgres-new || true - # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume - docker run --rm \ - -v sentry-postgres:/var/lib/postgresql/9.5/data \ - -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ - tianon/postgres-upgrade:9.5-to-9.6 - - # Get rid of the old volume as we'll rename the new one to that - docker volume rm sentry-postgres - docker volume create --name sentry-postgres - # There's no rename volume in Docker so copy the contents from old to new name - # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` - # doesn't do that automatically. - docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ - "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" - # Finally, remove the new old volume as we are all in sentry-postgres now - docker volume rm sentry-postgres-new + docker volume rm sentry-postgres-new || true + # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume + docker run --rm \ + -v sentry-postgres:/var/lib/postgresql/9.5/data \ + -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ + tianon/postgres-upgrade:9.5-to-9.6 + + # Get rid of the old volume as we'll rename the new one to that + docker volume rm sentry-postgres + docker volume create --name sentry-postgres + # There's no rename volume in Docker so copy the contents from old to new name + # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` + # doesn't do that automatically. + docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ + "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" + # Finally, remove the new old volume as we are all in sentry-postgres now + docker volume rm sentry-postgres-new fi echo "" echo "Setting up database..." if [ $CI ] || [ $SKIP_USER_PROMPT == 1 ]; then $dcr web upgrade --noinput echo "" echo "Did not prompt for user creation due to non-interactive shell." echo "Run the following command to create one yourself (recommended):" echo "" echo " docker-compose run --rm web createuser" echo "" else $dcr web upgrade fi SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l || true") if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then echo "Migrating file storage..." # Use the web (Sentry) image so the file owners are kept as sentry:sentry # The `\"` escape pattern is to make this compatible w/ Git Bash on Windows. See #329. $dcr --entrypoint \"/bin/bash\" web -c \ "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" fi if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "" echo "Generating Relay credentials..." # We need the ugly hack below as `relay generate credentials` tries to read the config and the credentials # even with the `--stdout` and `--overwrite` flags and then errors out when the credentials file exists but # not valid JSON. We hit this case as we redirect output to the same config folder, creating an empty # credentials file before relay runs. $dcr --no-deps -v $(pwd)/$RELAY_CONFIG_YML:/tmp/config.yml relay --config /tmp credentials generate --stdout > "$RELAY_CREDENTIALS_JSON" echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" fi if [[ "$MINIMIZE_DOWNTIME" ]]; then # Start the whole setup, except nginx and relay. $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') $dc exec -T nginx service nginx reload echo "Waiting for Sentry to start..." docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' # Make sure everything is up. This should only touch relay and nginx $dc up -d else echo "" echo "----------------" echo "You're all done! Run the following command to get Sentry running:" echo "" echo " docker-compose up -d" echo "" fi diff --git a/relay/config.example.yml b/relay/config.example.yml index f54c934..0488ba9 100644 --- a/relay/config.example.yml +++ b/relay/config.example.yml @@ -1,12 +1,12 @@ relay: upstream: "http://web:9000/" host: 0.0.0.0 port: 3000 logging: - level: WARN + level: WARN processing: enabled: true kafka_config: - {name: "bootstrap.servers", value: "kafka:9092"} - {name: "message.max.bytes", value: 50000000} #50MB or bust redis: redis://redis:6379 diff --git a/test.sh b/test.sh index a55d3d8..9d11d9d 100755 --- a/test.sh +++ b/test.sh @@ -1,107 +1,107 @@ #!/usr/bin/env bash set -e export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" TEST_USER='test@example.com' TEST_PASS='test123TEST' COOKIE_FILE=$(mktemp) # Courtesy of https://stackoverflow.com/a/2183063/90297 trap_with_arg() { - func="$1" ; shift - for sig ; do - trap "$func $sig "'$LINENO' "$sig" - done + func="$1" ; shift + for sig ; do + trap "$func $sig "'$LINENO' "$sig" + done } DID_CLEAN_UP=0 # the cleanup function will be the exit point cleanup () { if [ "$DID_CLEAN_UP" -eq 1 ]; then return 0; fi DID_CLEAN_UP=1 if [ "$1" != "EXIT" ]; then echo "An error occurred, caught SIG$1 on line $2"; fi echo "Cleaning up..." rm $COOKIE_FILE echo "Done." } trap_with_arg cleanup ERR INT TERM EXIT # Disable beacon for e2e tests echo 'SENTRY_BEACON=False' >> sentry/sentry.conf.py docker-compose run --rm web createuser --superuser --email $TEST_USER --password $TEST_PASS || true docker-compose up -d printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null $SENTRY_TEST_HOST); do printf '.'; sleep 0.5; done' get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; } sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; } login () { - INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) - if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then - echo "Initial /auth/login/ redirect failed, exiting..." - echo "$INITIAL_AUTH_REDIRECT" - exit -1 - fi - - CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' - /csrfmiddlewaretoken/ { - print $4 "=" $6; - exit; - }') - - curl -sL --data-urlencode 'op=login' --data-urlencode "username=$TEST_USER" --data-urlencode "password=$TEST_PASS" --data-urlencode "$CSRF_TOKEN_FOR_LOGIN" "$SENTRY_TEST_HOST/auth/login/sentry/" -H "Referer: $SENTRY_TEST_HOST/auth/login/sentry/" -b "$COOKIE_FILE" -c "$COOKIE_FILE"; + INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) + if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then + echo "Initial /auth/login/ redirect failed, exiting..." + echo "$INITIAL_AUTH_REDIRECT" + exit -1 + fi + + CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' + /csrfmiddlewaretoken/ { + print $4 "=" $6; + exit; + }') + + curl -sL --data-urlencode 'op=login' --data-urlencode "username=$TEST_USER" --data-urlencode "password=$TEST_PASS" --data-urlencode "$CSRF_TOKEN_FOR_LOGIN" "$SENTRY_TEST_HOST/auth/login/sentry/" -H "Referer: $SENTRY_TEST_HOST/auth/login/sentry/" -b "$COOKIE_FILE" -c "$COOKIE_FILE"; } LOGIN_RESPONSE=$(login); declare -a LOGIN_TEST_STRINGS=( - '"isAuthenticated":true' - '"username":"test@example.com"' - '"isSuperuser":true' + '"isAuthenticated":true' + '"username":"test@example.com"' + '"isSuperuser":true' ) for i in "${LOGIN_TEST_STRINGS[@]}" do - echo "Testing '$i'..." - echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null - echo "Pass." + echo "Testing '$i'..." + echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null + echo "Pass." done # Set up initial/required settings (InstallWizard request) sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }') # We ignore the protocol and the host as we already know those DSN_PIECES=(`echo $SENTRY_DSN | sed -ne 's|^https\?://\([0-9a-z]\+\)@[^/]\+/\([0-9]\+\)$|\1\n\2|p'`) SENTRY_KEY=${DSN_PIECES[0]} PROJECT_ID=${DSN_PIECES[1]} TEST_EVENT_ID=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-f0-9" | head -c 32) # Thanks @untitaker - https://forum.sentry.io/t/how-can-i-post-with-curl-a-sentry-event-which-authentication-credentials/4759/2?u=byk echo "Creating test event..." curl -sf --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"}}' -H 'Content-Type: application/json' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/store/" -o /dev/null EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/" export -f sentry_api_request get_csrf_token export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH printf "Getting the test event back" timeout 30 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' echo ""; EVENT_RESPONSE=$(sentry_api_request "$EVENT_PATH") declare -a EVENT_TEST_STRINGS=( - '"eventID":"'"$TEST_EVENT_ID"'"' - '"message":"a failure"' - '"title":"a failure"' - '"object":"42"' + '"eventID":"'"$TEST_EVENT_ID"'"' + '"message":"a failure"' + '"title":"a failure"' + '"object":"42"' ) for i in "${EVENT_TEST_STRINGS[@]}" do - echo "Testing '$i'..." - echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null - echo "Pass." + echo "Testing '$i'..." + echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null + echo "Pass." done