diff --git a/README.md b/README.md index 656c50b..722d99f 100644 --- a/README.md +++ b/README.md @@ -1,63 +1,63 @@ # Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). ## Requirements * Docker 19.03.6+ * Compose 1.28.0+ * 4 CPU Cores * 8 GB RAM * 20 GB Free Disk Space ## Setup ### Customize DotEnv (.env) file Environment specific configurations can be done in the `.env.custom` file. It will be located in the root directory of the Sentry installation. By default, there exists no `.env.custom` file. In this case, you can manually add this file by copying the `.env` file to a new `.env.custom` file and adjust your settings in the `.env.custom` file. Please keep in mind to check the `.env` file for changes, when you perform an upgrade of Sentry, so that you can adjust your `.env.custom` accordingly, if required. ### Installation To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020 and Sentry 21.1.0 is the last version to support Python 2. During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --skip-user-prompt`. Thinking of not managing this yourself? Check out the [SaaS migration docs](https://docs.sentry.io/product/sentry-basics/migration/) or [contact us](https://sentry.io/from/self-hosted) for help. Please visit [our documentation](https://develop.sentry.dev/self-hosted/) for everything else. ## Tips & Tricks ### Event Retention Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file. ### Installing a specific SHA If you want to install a specific release of Sentry, use the tags/releases on this repo. We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: ```shell SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh ``` Note that this may not work for all commit SHAs as this repository evolves with Sentry and its satellite projects. It is highly recommended to check out a version of this repository that is close to the timestamp of the Sentry commit you are installing. ### Using Linux If you are using Linux and you need to use `sudo` when running `./install.sh`, make sure to place the environment variable *after* `sudo`: ```shell sudo SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 ./install.sh ``` Where you replace `83b1380` with the sha you want to use. [build-status-image]: https://github.com/getsentry/self-hosted/workflows/test/badge.svg -[build-status-url]: https://git.io/JUYkh +[build-status-url]: https://github.com/getsentry/self-hosted/actions?query=workflow%3Atest+branch%3Amaster+event%3Apush diff --git a/cron/entrypoint.sh b/cron/entrypoint.sh index 383c8b2..49255bc 100755 --- a/cron/entrypoint.sh +++ b/cron/entrypoint.sh @@ -1,19 +1,19 @@ #!/usr/bin/env bash if [ "$(ls -A /usr/local/share/ca-certificates/)" ]; then update-ca-certificates fi # Prior art: -# - https://git.io/fjNOg +# - https://github.com/renskiy/cron-docker-image/blob/5600db37acf841c6d7a8b4f3866741bada5b4622/debian/start-cron#L34-L36 # - https://blog.knoldus.com/running-a-cron-job-in-docker-container/ declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env { for cron_job in "$@"; do echo -e "SHELL=/bin/bash BASH_ENV=/container.env ${cron_job} > /proc/1/fd/1 2>/proc/1/fd/2"; done } \ | sed --regexp-extended 's/\\(.)/\1/g' \ | crontab - crontab -l exec cron -f -l -L 15 diff --git a/install/check-minimum-requirements.sh b/install/check-minimum-requirements.sh index b05952a..61484d3 100644 --- a/install/check-minimum-requirements.sh +++ b/install/check-minimum-requirements.sh @@ -1,50 +1,50 @@ echo "${_group}Checking minimum requirements ..." source "$(dirname $0)/_min-requirements.sh" # Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') if [[ "$(ver $DOCKER_VERSION)" -lt "$(ver $MIN_DOCKER_VERSION)" ]]; then echo "FAIL: Expected minimum docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" exit 1 fi echo "Found Docker version $DOCKER_VERSION" # See https://github.com/getsentry/self-hosted/issues/1132#issuecomment-982823712 ff. for regex testing. COMPOSE_VERSION=$($dc_base version | head -n1 | sed -E 's/^.* version:? v?([0-9.]+),?.*$/\1/') if [[ "$(ver $COMPOSE_VERSION)" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then echo "FAIL: Expected minimum $dc_base version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" exit 1 fi echo "Found Docker Compose version $COMPOSE_VERSION" CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all); if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER" exit 1 elif [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT, found $CPU_AVAILABLE_IN_DOCKER" fi RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then echo "FAIL: Required minimum RAM available to Docker is $MIN_RAM_HARD MB, found $RAM_AVAILABLE_IN_DOCKER MB" exit 1 elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_SOFT" ]]; then echo "WARN: Recommended minimum RAM available to Docker is $MIN_RAM_SOFT MB, found $RAM_AVAILABLE_IN_DOCKER MB" fi #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) # On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297 IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :) if [[ "$IS_KVM" -eq 0 ]]; then SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :) if [[ "$SUPPORTS_SSE42" -eq 0 ]]; then - echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." + echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://github.com/getsentry/self-hosted/issues/340 for more info." exit 1 fi fi echo "${_endgroup}" diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 355ae63..fece283 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -1,281 +1,281 @@ # This file is just Python, with a touch of Django which means # you can inherit and tweak settings to your hearts content. from sentry.conf.server import * # NOQA -# Generously adapted from pynetlinux: https://git.io/JJmga +# Generously adapted from pynetlinux: https://github.com/rlisagor/pynetlinux/blob/e3f16978855c6649685f0c43d4c3fcf768427ae5/pynetlinux/ifconfig.py#L197-L223 def get_internal_network(): import ctypes import fcntl import math import socket import struct iface = b"eth0" sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) ifreq = struct.pack(b"16sH14s", iface, socket.AF_INET, b"\x00" * 14) try: ip = struct.unpack( b"!I", struct.unpack(b"16sH2x4s8x", fcntl.ioctl(sockfd, 0x8915, ifreq))[2] )[0] netmask = socket.ntohl( struct.unpack(b"16sH2xI8x", fcntl.ioctl(sockfd, 0x891B, ifreq))[2] ) except IOError: return () base = socket.inet_ntoa(struct.pack(b"!I", ip & netmask)) netmask_bits = 32 - int(round(math.log(ctypes.c_uint32(~netmask).value + 1, 2), 1)) return "{0:s}/{1:d}".format(base, netmask_bits) INTERNAL_SYSTEM_IPS = (get_internal_network(),) DATABASES = { "default": { "ENGINE": "sentry.db.postgres", "NAME": "postgres", "USER": "postgres", "PASSWORD": "", "HOST": "postgres", "PORT": "", } } # You should not change this setting after your database has been created # unless you have altered all schemas first SENTRY_USE_BIG_INTS = True # If you're expecting any kind of real traffic on Sentry, we highly recommend # configuring the CACHES and Redis settings ########### # General # ########### # Instruct Sentry that this install intends to be run by a single organization # and thus various UI optimizations should be enabled. SENTRY_SINGLE_ORGANIZATION = True SENTRY_OPTIONS["system.event-retention-days"] = int( env("SENTRY_EVENT_RETENTION_DAYS", "90") ) ######### # Redis # ######### # Generic Redis configuration used as defaults for various things including: # Buffers, Quotas, TSDB SENTRY_OPTIONS["redis.clusters"] = { "default": { "hosts": {0: {"host": "redis", "password": "", "port": "6379", "db": "0"}} } } ######### # Queue # ######### # See https://develop.sentry.dev/services/queue/ for more # information on configuring your queue broker and workers. Sentry relies # on a Python framework called Celery to manage queues. rabbitmq_host = None if rabbitmq_host: BROKER_URL = "amqp://{username}:{password}@{host}/{vhost}".format( username="guest", password="guest", host=rabbitmq_host, vhost="/" ) else: BROKER_URL = "redis://:{password}@{host}:{port}/{db}".format( **SENTRY_OPTIONS["redis.clusters"]["default"]["hosts"][0] ) ######### # Cache # ######### # Sentry currently utilizes two separate mechanisms. While CACHES is not a # requirement, it will optimize several high throughput patterns. CACHES = { "default": { "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", "LOCATION": ["memcached:11211"], "TIMEOUT": 3600, } } # A primary cache is required for things such as processing events SENTRY_CACHE = "sentry.cache.redis.RedisCache" DEFAULT_KAFKA_OPTIONS = { "bootstrap.servers": "kafka:9092", "message.max.bytes": 50000000, "socket.timeout.ms": 1000, } SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" SENTRY_EVENTSTREAM_OPTIONS = {"producer_configuration": DEFAULT_KAFKA_OPTIONS} KAFKA_CLUSTERS["default"] = DEFAULT_KAFKA_OPTIONS ############### # Rate Limits # ############### # Rate limits apply to notification handlers and are enforced per-project # automatically. SENTRY_RATELIMITER = "sentry.ratelimits.redis.RedisRateLimiter" ################## # Update Buffers # ################## # Buffers (combined with queueing) act as an intermediate layer between the # database and the storage API. They will greatly improve efficiency on large # numbers of the same events being sent to the API in a short amount of time. # (read: if you send any kind of real data to Sentry, you should enable buffers) SENTRY_BUFFER = "sentry.buffer.redis.RedisBuffer" ########## # Quotas # ########## # Quotas allow you to rate limit individual projects or the Sentry install as # a whole. SENTRY_QUOTAS = "sentry.quotas.redis.RedisQuota" ######## # TSDB # ######## # The TSDB is used for building charts as well as making things like per-rate # alerts possible. SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB" ######### # SNUBA # ######### SENTRY_SEARCH = "sentry.search.snuba.EventsDatasetSnubaSearchBackend" SENTRY_SEARCH_OPTIONS = {} SENTRY_TAGSTORE_OPTIONS = {} ########### # Digests # ########### # The digest backend powers notification summaries. SENTRY_DIGESTS = "sentry.digests.backends.redis.RedisBackend" ############## # Web Server # ############## SENTRY_WEB_HOST = "0.0.0.0" SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), "protocol": "uwsgi", - # This is needed in order to prevent https://git.io/fj7Lw + # This is needed in order to prevent https://github.com/getsentry/sentry/blob/c6f9660e37fcd9c1bbda8ff4af1dcfd0442f5155/src/sentry/services/http.py#L70 "uwsgi-socket": None, "so-keepalive": True, # Keep this between 15s-75s as that's what Relay supports "http-keepalive": 15, "http-chunked-input": True, # the number of web workers "workers": 3, "threads": 4, "memory-report": False, # Some stuff so uwsgi will cycle workers sensibly "max-requests": 100000, "max-requests-delta": 500, "max-worker-lifetime": 86400, # Duplicate options from sentry default just so we don't get # bit by sentry changing a default value that we depend on. "thunder-lock": True, "log-x-forwarded-for": False, "buffer-size": 32768, "limit-post": 209715200, "disable-logging": True, "reload-on-rss": 600, "ignore-sigpipe": True, "ignore-write-errors": True, "disable-write-exception": True, } ########### # SSL/TLS # ########### # If you're using a reverse SSL proxy, you should enable the X-Forwarded-Proto # header and enable the settings below # SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') # SESSION_COOKIE_SECURE = True # CSRF_COOKIE_SECURE = True # SOCIAL_AUTH_REDIRECT_IS_HTTPS = True # End of SSL/TLS settings ######## # Mail # ######## SENTRY_OPTIONS["mail.list-namespace"] = env('SENTRY_MAIL_HOST', 'localhost') SENTRY_OPTIONS["mail.from"] = f"sentry@{SENTRY_OPTIONS['mail.list-namespace']}" ############ # Features # ############ SENTRY_FEATURES["projects:sample-events"] = False SENTRY_FEATURES.update( { feature: True for feature in ( "organizations:discover", "organizations:events", "organizations:global-views", "organizations:incidents", "organizations:integrations-issue-basic", "organizations:integrations-issue-sync", "organizations:invite-members", "organizations:metric-alert-builder-aggregate", "organizations:sso-basic", "organizations:sso-rippling", "organizations:sso-saml2", "organizations:performance-view", "organizations:advanced-search", "projects:custom-inbound-filters", "projects:data-forwarding", "projects:discard-groups", "projects:plugins", "projects:rate-limits", "projects:servicehooks", ) } ) ####################### # MaxMind Integration # ####################### GEOIP_PATH_MMDB = '/geoip/GeoLite2-City.mmdb' ######################### # Bitbucket Integration # ######################### # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY' # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET'