diff --git a/_integration-test/run.sh b/_integration-test/run.sh index 94f54b0..6a94be5 100755 --- a/_integration-test/run.sh +++ b/_integration-test/run.sh @@ -1,128 +1,130 @@ #!/usr/bin/env bash set -ex source "$(dirname $0)/../install/_lib.sh" +source ../install/dc-detect-version.sh + echo "${_group}Setting up variables and helpers ..." export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" TEST_USER='test@example.com' TEST_PASS='test123TEST' COOKIE_FILE=$(mktemp) # Courtesy of https://stackoverflow.com/a/2183063/90297 trap_with_arg() { func="$1" ; shift for sig ; do trap "$func $sig "'$LINENO' "$sig" done } DID_TEAR_DOWN=0 # the teardown function will be the exit point teardown() { if [ "$DID_TEAR_DOWN" -eq 1 ]; then return 0; fi DID_TEAR_DOWN=1 if [ "$1" != "EXIT" ]; then echo "An error occurred, caught SIG$1 on line $2"; fi echo "Tearing down ..." rm $COOKIE_FILE echo "Done." } trap_with_arg teardown ERR INT TERM EXIT echo "${_endgroup}" echo "${_group}Starting Sentry for tests ..." # Disable beacon for e2e tests echo 'SENTRY_BEACON=False' >> $SENTRY_CONFIG_PY $dcr web createuser --superuser --email $TEST_USER --password $TEST_PASS || true $dc up -d printf "Waiting for Sentry to be up"; timeout 90 bash -c 'until $(curl -Isf -o /dev/null $SENTRY_TEST_HOST); do printf '.'; sleep 0.5; done' echo "" echo "${_endgroup}" echo "${_group}Running tests ..." get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; } sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; } login () { INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then echo "Initial /auth/login/ redirect failed, exiting..." echo "$INITIAL_AUTH_REDIRECT" exit -1 fi CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "['\"]" ' /csrfmiddlewaretoken/ { print $4 "=" $6; exit; }') curl -sL --data-urlencode 'op=login' --data-urlencode "username=$TEST_USER" --data-urlencode "password=$TEST_PASS" --data-urlencode "$CSRF_TOKEN_FOR_LOGIN" "$SENTRY_TEST_HOST/auth/login/sentry/" -H "Referer: $SENTRY_TEST_HOST/auth/login/sentry/" -b "$COOKIE_FILE" -c "$COOKIE_FILE"; } LOGIN_RESPONSE=$(login); declare -a LOGIN_TEST_STRINGS=( '"isAuthenticated":true' '"username":"test@example.com"' '"isSuperuser":true' ) for i in "${LOGIN_TEST_STRINGS[@]}" do echo "Testing '$i'..." echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null echo "Pass." done echo "${_endgroup}" echo "${_group}Running moar tests !!!" # Set up initial/required settings (InstallWizard request) sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" && $4 ~ "^http" { print $4; exit; }') # We ignore the protocol and the host as we already know those DSN_PIECES=(`echo $SENTRY_DSN | sed -ne 's|^https\{0,1\}://\([0-9a-z]\{1,\}\)@[^/]\{1,\}/\([0-9]\{1,\}\)$|\1 \2|p' | tr ' ' '\n'`) SENTRY_KEY=${DSN_PIECES[0]} PROJECT_ID=${DSN_PIECES[1]} TEST_EVENT_ID=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-f0-9" | head -c 32) # Thanks @untitaker - https://forum.sentry.io/t/how-can-i-post-with-curl-a-sentry-event-which-authentication-credentials/4759/2?u=byk echo "Creating test event..." curl -sf --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"}}' -H 'Content-Type: application/json' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/store/" -o /dev/null EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/" export -f sentry_api_request get_csrf_token export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH printf "Getting the test event back" timeout 60 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' echo " got it!"; EVENT_RESPONSE=$(sentry_api_request "$EVENT_PATH") declare -a EVENT_TEST_STRINGS=( '"eventID":"'"$TEST_EVENT_ID"'"' '"message":"a failure"' '"title":"a failure"' '"object":"42"' ) for i in "${EVENT_TEST_STRINGS[@]}" do echo "Testing '$i'..." echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null echo "Pass." done echo "${_endgroup}" echo "${_group}Ensure cleanup crons are working ..." $dc ps | grep -q -E -e '\-cleanup\s+running\s+' -e '\-cleanup[_-].+\s+Up\s+' echo "${_endgroup}" echo "${_group}Test custom CAs work ..." source ./custom-ca-roots/setup.sh $dcr --no-deps web python3 /etc/sentry/test-custom-ca-roots.py source ./custom-ca-roots/teardown.sh echo "${_endgroup}" diff --git a/install.sh b/install.sh index 10525ef..887f7ac 100755 --- a/install.sh +++ b/install.sh @@ -1,30 +1,31 @@ #!/usr/bin/env bash set -e if [[ -n "$MSYSTEM" ]]; then echo "Seems like you are using an MSYS2-based system (such as Git Bash) which is not supported. Please use WSL instead."; exit 1 fi source "$(dirname $0)/install/_lib.sh" # does a `cd .../install/`, among other things +source dc-detect-version.sh source parse-cli.sh source check-latest-commit.sh source error-handling.sh source check-minimum-requirements.sh source create-docker-volumes.sh source ensure-files-from-examples.sh source generate-secret-key.sh source replace-tsdb.sh source update-docker-images.sh source build-docker-images.sh source turn-things-off.sh source set-up-zookeeper.sh source install-wal2json.sh source bootstrap-snuba.sh source create-kafka-topics.sh source upgrade-postgres.sh source set-up-and-migrate-database.sh source migrate-file-storage.sh source relay-credentials.sh source geoip.sh source wrap-up.sh diff --git a/install/_lib.sh b/install/_lib.sh index fb7b920..4af1480 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -1,56 +1,51 @@ set -euo pipefail test "${DEBUG:-}" && set -x # Thanks to https://unix.stackexchange.com/a/145654/108960 log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") # Work from /install/ for install.sh, project root otherwise if [[ "$(basename $0)" = "install.sh" ]]; then cd "$(dirname $0)/install/" else cd "$(dirname $0)" # assume we're a test script or some such fi # Allow `.env` overrides using the `.env.custom` file if [[ -f "../.env.custom" ]]; then _ENV="$(realpath ../.env.custom)" else _ENV="$(realpath ../.env)" fi # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 t=$(mktemp) && export -p > "$t" && set -a && . $_ENV && set +a && . "$t" && rm "$t" && unset t if [ "${GITHUB_ACTIONS:-}" = "true" ]; then _group="::group::" _endgroup="::endgroup::" else _group="▶ " _endgroup="" fi -# Some environments still use `docker-compose` even for Docker Compose v2. -dc_base="$(docker compose version &> /dev/null && echo 'docker compose' || echo 'docker-compose')" -dc="$dc_base --ansi never --env-file ${_ENV}" -dcr="$dc run --rm" - # A couple of the config files are referenced from other subscripts, so they # get vars, while multiple subscripts call ensure_file_from_example. function ensure_file_from_example { if [[ -f "$1" ]]; then echo "$1 already exists, skipped creation." else echo "Creating $1..." cp -n $(echo "$1" | sed 's/\.[^.]*$/.example&/') "$1" # sed from https://stackoverflow.com/a/25123013/90297 fi } SENTRY_CONFIG_PY='../sentry/sentry.conf.py' SENTRY_CONFIG_YML='../sentry/config.yml' # Increase the default 10 second SIGTERM timeout # to ensure celery queues are properly drained # between upgrades as task signatures may change across # versions STOP_TIMEOUT=60 # seconds diff --git a/install/dc-detect-version.sh b/install/dc-detect-version.sh new file mode 100644 index 0000000..1631a11 --- /dev/null +++ b/install/dc-detect-version.sh @@ -0,0 +1,8 @@ +# Some environments still use `docker-compose` even for Docker Compose v2. +dc_base="$(docker compose version &> /dev/null && echo 'docker compose' || echo 'docker-compose')" +if [[ "$(basename $0)" = "install.sh" ]]; then + dc="$dc_base --ansi never --env-file ${_ENV}" +else + dc="$dc_base --ansi never" +fi +dcr="$dc run --rm" diff --git a/reset.sh b/reset.sh index cba7568..b5cf4e0 100755 --- a/reset.sh +++ b/reset.sh @@ -1,62 +1,63 @@ #!/usr/bin/env bash # The purpose of this script is to make it easy to reset a local self-hosted # install to a clean state, optionally targeting a particular version. set -euo pipefail if [ -n "${DEBUG:-}" ]; then set -x fi cd "$(dirname $0)" +source install/dc-detect-version.sh function confirm () { read -p "$1 [y/n] " confirmation if [ "$confirmation" != "y" ]; then echo "Canceled. 😅" exit fi } # If we have a version given, validate it. # ---------------------------------------- # Note that arbitrary git refs won't work, because the *_IMAGE variables in # .env will almost certainly point to :latest. Tagged releases are generally # the only refs where these component versions are pinned, so enforce that # we're targeting a valid tag here. Do this early in order to fail fast. version="${1:-}" if [ -n "$version" ]; then set +e git rev-parse --verify --quiet "refs/tags/$version" > /dev/null if [ $? -gt 0 ]; then echo "Bad version: $version" exit fi set -e fi # Make sure they mean it. confirm "☠️ Warning! 😳 This is highly destructive! 😱 Are you sure you wish to proceed?" echo "Okay ... good luck! 😰" # Hit the reset button. -docker compose down --volumes --remove-orphans --rmi local +$dc down --volumes --remove-orphans --rmi local # Remove any remaining (likely external) volumes with name matching 'sentry-.*'. for volume in $(docker volume list --format '{{ .Name }}' | grep '^sentry-'); do docker volume remove $volume > /dev/null \ && echo "Removed volume: $volume" \ || echo "Skipped volume: $volume" done # If we have a version given, switch to it. if [ -n "$version" ]; then git checkout "$version" fi # Install. ./install.sh