diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 282537a..039aa0e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,60 +1,57 @@
 name: Test
 on:
   # Run CI on all pushes to the master and release/** branches, and on all new
   # pull requests, and on all pushes to pull requests (even if a pull request
   # is not against master).
   push:
     branches:
       - "master"
       - "release/**"
   pull_request:
 env:
   DOCKER_COMPOSE_VERSION: 1.29.2
 defaults:
   run:
     shell: bash
 jobs:
   unit-test:
     runs-on: ubuntu-20.04
     name: "unit tests"
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
       - name: Unit Tests
         working-directory: install
         run: find ./ -type f -name "*-test.sh" -exec "./{}" \;
 
   integration-test:
     runs-on: ubuntu-20.04
     name: "integration test"
     steps:
       - name: Pin docker-compose
         run: |
           sudo rm /usr/local/bin/docker-compose
           curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
           chmod +x docker-compose
           sudo mv docker-compose /usr/local/bin
 
       - name: Checkout
         uses: actions/checkout@v2
 
       - name: Integration Test
         run: |
           echo "Testing initial install"
-          # Create ./certificates here because install.sh will create it with root:root
-          # and then run.sh (-> setup.sh) won't be able to write to it.
-          mkdir certificates
           ./install.sh
           ./_integration-test/run.sh
           echo "Testing in-place upgrade"
           # Also test plugin installation here
           echo "sentry-auth-oidc" >> sentry/requirements.txt
           ./install.sh --minimize-downtime
           ./_integration-test/run.sh
 
       - name: Inspect failure
         if: failure()
         run: |
           docker-compose ps
           docker-compose logs
diff --git a/.gitignore b/.gitignore
index c8967cd..4a86daa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,96 +1,93 @@
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
 
 # C extensions
 *.so
 
 # Distribution / packaging
 .Python
 env/
 build/
 develop-eggs/
 dist/
 downloads/
 eggs/
 .eggs/
 lib/
 lib64/
 parts/
 sdist/
 var/
 *.egg-info/
 .installed.cfg
 *.egg
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
 
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
 sentry_install_log*.txt
 
 # Unit test / coverage reports
 htmlcov/
 .tox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
 
 # Translations
 *.mo
 *.pot
 
 # Django stuff:
 *.log
 local_settings.py
 
 # Sphinx documentation
 docs/_build/
 
 # PyBuilder
 target/
 
 # Ipython Notebook
 .ipynb_checkpoints
 
 # pyenv
 .python-version
 
 # https://docs.docker.com/compose/extends/
 docker-compose.override.yml
 
 *.tar
 data/
 .vscode/tags
 
 # custom Sentry config
 sentry/sentry.conf.py
 sentry/config.yml
 sentry/*.bak
 sentry/requirements.txt
 relay/credentials.json
 relay/config.yml
 symbolicator/config.yml
 geoip/GeoIP.conf
 geoip/*.mmdb
 geoip/.geoipupdate.lock
 
 # wal2json download
 postgres/wal2json
 
-# custom certificate authorities
-certificates
-
 # integration testing
 _integration-test/custom-ca-roots/nginx/*
 sentry/test-custom-ca-roots.py
diff --git a/certificates/.gitignore b/certificates/.gitignore
new file mode 100644
index 0000000..30d0607
--- /dev/null
+++ b/certificates/.gitignore
@@ -0,0 +1,3 @@
+# Add all custom CAs in this folder
+*
+!.gitignore