diff --git a/.mrconfig b/.mrconfig index 8bb4d7c..3c6fba4 100644 --- a/.mrconfig +++ b/.mrconfig @@ -1,194 +1,198 @@ # keep the list sorted alphabetically by local repo name [snippets] checkout = git clone 'https://forge.softwareheritage.org/source/snippets.git' 'snippets' [swh-auth] checkout = git clone 'https://forge.softwareheritage.org/source/swh-auth.git' 'swh-auth' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-core] checkout = git clone 'https://forge.softwareheritage.org/source/swh-core.git' 'swh-core' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-counters] checkout = git clone 'https://forge.softwareheritage.org/source/swh-counters.git' 'swh-counters' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-dataset] checkout = git clone 'https://forge.softwareheritage.org/source/swh-dataset.git' 'swh-dataset' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-deposit] checkout = git clone 'https://forge.softwareheritage.org/source/swh-deposit.git' 'swh-deposit' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-docs] checkout = git clone 'https://forge.softwareheritage.org/source/swh-docs.git' 'swh-docs' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-fuse] checkout = git clone 'https://forge.softwareheritage.org/source/swh-fuse.git' 'swh-fuse' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-graph] checkout = git clone 'https://forge.softwareheritage.org/source/swh-graph.git' 'swh-graph' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg +[swh-graphql] +checkout = git clone 'https://forge.softwareheritage.org/source/swh-graphql.git' 'swh-graphql' +fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install + [swh-icinga-plugins] checkout = git clone 'https://forge.softwareheritage.org/source/swh-icinga-plugins.git' 'swh-icinga-plugins' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-indexer] checkout = git clone 'https://forge.softwareheritage.org/source/swh-indexer.git' 'swh-indexer' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-journal] checkout = git clone 'https://forge.softwareheritage.org/source/swh-journal.git' 'swh-journal' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-lister] checkout = git clone 'https://forge.softwareheritage.org/source/swh-lister.git' 'swh-lister' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-loader-core] checkout = git clone 'https://forge.softwareheritage.org/source/swh-loader-core.git' 'swh-loader-core' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-loader-bzr] checkout = git clone 'https://forge.softwareheritage.org/source/swh-loader-bzr.git' 'swh-loader-bzr' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-loader-cvs] checkout = git clone 'https://forge.softwareheritage.org/source/swh-loader-cvs.git' 'swh-loader-cvs' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-loader-git] checkout = git clone 'https://forge.softwareheritage.org/source/swh-loader-git.git' 'swh-loader-git' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-loader-mercurial] checkout = git clone 'https://forge.softwareheritage.org/source/swh-loader-mercurial.git' 'swh-loader-mercurial' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-loader-metadata] checkout = git clone 'https://forge.softwareheritage.org/source/swh-loader-metadata.git' 'swh-loader-metadata' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-loader-svn] checkout = git clone 'https://forge.softwareheritage.org/source/swh-loader-svn.git' 'swh-loader-svn' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-model] checkout = git clone 'https://forge.softwareheritage.org/source/swh-model.git' 'swh-model' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-objstorage] checkout = git clone 'https://forge.softwareheritage.org/source/swh-objstorage.git' 'swh-objstorage' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-objstorage-replayer] checkout = git clone 'https://forge.softwareheritage.org/source/swh-objstorage-replayer.git' 'swh-objstorage-replayer' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-perfecthash] checkout = git clone 'https://forge.softwareheritage.org/source/swh-perfecthash.git' 'swh-perfecthash' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-py-template] checkout = git clone 'https://forge.softwareheritage.org/source/swh-py-template.git' 'swh-py-template' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-scanner] checkout = git clone 'https://forge.softwareheritage.org/source/swh-scanner.git' 'swh-scanner' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-scheduler] checkout = git clone 'https://forge.softwareheritage.org/source/swh-scheduler.git' 'swh-scheduler' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-scrubber] checkout = git clone 'https://forge.softwareheritage.org/source/swh-scrubber.git' 'swh-scrubber' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install [swh-search] checkout = git clone 'https://forge.softwareheritage.org/source/swh-search.git' 'swh-search' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-storage] checkout = git clone 'https://forge.softwareheritage.org/source/swh-storage.git' 'swh-storage' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-vault] checkout = git clone 'https://forge.softwareheritage.org/source/swh-vault.git' 'swh-vault' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-web] checkout = git clone 'https://forge.softwareheritage.org/source/swh-web.git' 'swh-web' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg [swh-web-client] checkout = git clone 'https://forge.softwareheritage.org/source/swh-web-client.git' 'swh-web-client' fixups = grep -q pre-commit.com .git/hooks/pre-commit || pre-commit install grep -q pre-commit.com .git/hooks/commit-msg || pre-commit install -t commit-msg diff --git a/docker/Dockerfile b/docker/Dockerfile index cdb80f7..700b948 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,90 +1,91 @@ FROM python:3.7 ARG PGDG_REPO=http://apt.postgresql.org/pub/repos/apt ARG PGDG_GPG_KEY=https://www.postgresql.org/media/keys/ACCC4CF8.asc ARG PGDG_KEYRING=/usr/share/keyrings/pgdg-archive-keyring.gpg ARG NODE_REPO=https://deb.nodesource.com/node_12.x ARG NODE_GPG_KEY=https://deb.nodesource.com/gpgkey/nodesource.gpg.key ARG NODE_KEYRING=/usr/share/keyrings/nodejs-archive-keyring.gpg ARG YARN_REPO=https://dl.yarnpkg.com/debian/ ARG YARN_GPG_KEY=https://dl.yarnpkg.com/debian/pubkey.gpg ARG YARN_KEYRING=/usr/share/keyrings/yarnpkg-archive-keyring.gpg RUN . /etc/os-release && \ echo "deb [signed-by=${PGDG_KEYRING}] ${PGDG_REPO} ${VERSION_CODENAME}-pgdg main" \ > /etc/apt/sources.list.d/pgdg.list && \ curl -fsSL ${PGDG_GPG_KEY} | gpg --dearmor > ${PGDG_KEYRING} && \ echo "deb [signed-by=${NODE_KEYRING}] ${NODE_REPO} ${VERSION_CODENAME} main" \ > /etc/apt/sources.list.d/nodejs.list && \ curl -fsSL ${NODE_GPG_KEY} | gpg --dearmor > ${NODE_KEYRING} && \ echo "deb [signed-by=${YARN_KEYRING}] ${YARN_REPO} stable main" \ > /etc/apt/sources.list.d/yarnpkg.list && \ curl -fsSL ${YARN_GPG_KEY} | gpg --dearmor > ${YARN_KEYRING} # warning: the py:3.7 image comes with python3.9 installed from debian; do not # add debian python packages here, they would not be usable for the py37 based # environment used in this image. RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y \ libapr1-dev \ libaprutil1-dev \ libcmph-dev \ libpq-dev \ librdkafka-dev \ libsvn-dev \ libsystemd-dev \ gcc \ openjdk-11-jre \ pkg-config \ pv \ postgresql-client-12 \ wait-for-it \ ngrep \ rsync \ nodejs \ yarn \ zstd && \ apt-get install -y --no-install-recommends \ opam \ r-base-core \ r-cran-jsonlite && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* RUN useradd -md /srv/softwareheritage -s /bin/bash swh USER swh RUN python3 -m venv /srv/softwareheritage/venv ENV PATH="/srv/softwareheritage/venv/bin:${PATH}" RUN pip install --upgrade pip setuptools wheel RUN pip install gunicorn httpie # cython and configjob are required to install the breeze (bzr) package RUN pip install cython configobj RUN pip install \ swh-core[db,http] \ swh-counters \ swh-deposit[server] \ + swh-graphql \ swh-indexer \ swh-journal \ swh-lister \ swh-loader-core \ swh-loader-bzr \ swh-loader-cvs \ swh-loader-git \ swh-loader-mercurial \ swh-loader-metadata \ swh-loader-svn \ swh-storage \ swh-objstorage \ swh-scheduler \ swh-vault \ swh-web COPY utils/*.sh /srv/softwareheritage/utils/ RUN mkdir -p /srv/softwareheritage/objects RUN rm -rd /srv/softwareheritage/.cache diff --git a/docker/conf/graphql.yml b/docker/conf/graphql.yml new file mode 100644 index 0000000..f4a00bb --- /dev/null +++ b/docker/conf/graphql.yml @@ -0,0 +1,5 @@ +storage: + cls: remote + url: http://swh-storage:5002 + +debug: yes diff --git a/docker/docker-compose.graphql.yml b/docker/docker-compose.graphql.yml new file mode 100644 index 0000000..84b3e34 --- /dev/null +++ b/docker/docker-compose.graphql.yml @@ -0,0 +1,18 @@ +version: "2.1" + +services: + swh-graphql: + image: swh/stack + build: ./ + depends_on: + - swh-storage + ports: + - 5013:5013 + environment: + SWH_CONFIG_FILENAME: /graphql.yml + env_file: + - ./env/common_python.env + volumes: + - "./conf/graphql.yml:/graphql.yml:ro" + - "./services/swh-graphql/entrypoint.sh:/entrypoint.sh:ro" + entrypoint: /entrypoint.sh diff --git a/docker/services/swh-graphql/entrypoint.sh b/docker/services/swh-graphql/entrypoint.sh new file mode 100755 index 0000000..09da33e --- /dev/null +++ b/docker/services/swh-graphql/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +source /srv/softwareheritage/utils/pyutils.sh +setup_pip + +case "$1" in + "shell") + exec bash -i + ;; + *) + echo Starting the swh-graphql API server + + exec gunicorn --bind 0.0.0.0:5013 \ + --reload \ + --threads 4 \ + --workers 2 \ + --log-level DEBUG \ + --timeout 3600 \ + --config 'python:swh.core.api.gunicorn_config' \ + 'swh.graphql.server:make_app_from_configfile()' + ;; +esac