diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -259,14 +259,13 @@ # Indexer Celery workers - swh-indexer-mimetype: + swh-indexer: image: swh/indexer-worker build: ./dockerfiles/swh-indexer-worker env_file: ./indexers.env environment: STATSD_HOST: prometheus-statsd-exporter STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: content_mimetype depends_on: - swh-scheduler-runner - swh-idx-storage @@ -274,44 +273,6 @@ - swh-objstorage - amqp - swh-indexer-license: - image: swh/indexer-worker - build: ./dockerfiles/swh-indexer-worker - env_file: ./indexers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: content_fossology_license - depends_on: - - swh-scheduler-runner - - swh-idx-storage - - swh-storage - - swh-objstorage - - amqp - - swh-indexer-origin-head: - image: swh/indexer-worker - build: ./dockerfiles/swh-indexer-worker - env_file: ./indexers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: origin_head - depends_on: - - swh-scheduler-api - - swh-scheduler-runner - - swh-idx-storage - - swh-storage - - amqp - - swh-indexer-revision-metadata: - image: swh/indexer-worker - build: ./dockerfiles/swh-indexer-worker - env_file: ./indexers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: revision_metadata depends_on: - swh-scheduler-runner - swh-idx-storage @@ -319,20 +280,6 @@ - swh-objstorage - amqp - swh-indexer-origin-intrinsic-metadata: - image: swh/indexer-worker - build: ./dockerfiles/swh-indexer-worker - env_file: ./indexers.env - environment: - STATSD_HOST: prometheus-statsd-exporter - STATSD_PORT: 9125 - SWH_WORKER_INSTANCE: origin_intrinsic_metadata - depends_on: - - swh-scheduler-runner - - swh-idx-storage - - swh-storage - - amqp - swh-indexer-journal-client: image: swh/indexer-journal-client build: ./dockerfiles/swh-indexer-journal-client diff --git a/dockerfiles/swh-indexer-worker/Dockerfile b/dockerfiles/swh-indexer-worker/Dockerfile --- a/dockerfiles/swh-indexer-worker/Dockerfile +++ b/dockerfiles/swh-indexer-worker/Dockerfile @@ -5,16 +5,12 @@ apt-get install -y \ libsystemd-dev postgresql-client +RUN pip install --upgrade pip RUN pip install swh-indexer RUN useradd -ms /bin/bash swh -COPY entrypoint.sh / - -COPY mimetype.yml /home/swh/.config/swh/indexer/ -COPY fossology_license.yml /home/swh/.config/swh/indexer/ -COPY origin_head.yml /home/swh/.config/swh/indexer/ -COPY revision_metadata.yml /home/swh/.config/swh/indexer/ -COPY origin_intrinsic_metadata.yml /home/swh/.config/swh/indexer/ -RUN chown -R swh: /home/swh/.config/ USER swh -ENTRYPOINT ["/entrypoint.sh"] +COPY entrypoint.sh / +COPY indexer.yml /home/swh/ + +ENTRYPOINT ["/home/swh/entrypoint.sh"] diff --git a/dockerfiles/swh-indexer-worker/fossology_license.yml b/dockerfiles/swh-indexer-worker/fossology_license.yml deleted file mode 100644 --- a/dockerfiles/swh-indexer-worker/fossology_license.yml +++ /dev/null @@ -1,12 +0,0 @@ -storage: - cls: remote - args: - url: http://swh-storage:5002/ -objstorage: - cls: remote - args: - url: http://swh-objstorage:5003/ -indexer_storage: - cls: remote - args: - url: http://swh-idx-storage:5007/ diff --git a/dockerfiles/swh-indexer-worker/revision_metadata.yml b/dockerfiles/swh-indexer-worker/indexer.yml rename from dockerfiles/swh-indexer-worker/revision_metadata.yml rename to dockerfiles/swh-indexer-worker/indexer.yml --- a/dockerfiles/swh-indexer-worker/revision_metadata.yml +++ b/dockerfiles/swh-indexer-worker/indexer.yml @@ -2,17 +2,14 @@ cls: remote args: url: http://swh-storage:5002/ - objstorage: cls: remote args: url: http://swh-objstorage:5003/ - indexer_storage: cls: remote args: url: http://swh-idx-storage:5007/ - scheduler: cls: remote args: diff --git a/dockerfiles/swh-indexer-worker/mimetype.yml b/dockerfiles/swh-indexer-worker/mimetype.yml deleted file mode 100644 --- a/dockerfiles/swh-indexer-worker/mimetype.yml +++ /dev/null @@ -1,12 +0,0 @@ -storage: - cls: remote - args: - url: http://swh-storage:5002/ -objstorage: - cls: remote - args: - url: http://swh-objstorage:5003/ -indexer_storage: - cls: remote - args: - url: http://swh-idx-storage:5007/ diff --git a/dockerfiles/swh-indexer-worker/origin_head.yml b/dockerfiles/swh-indexer-worker/origin_head.yml deleted file mode 100644 --- a/dockerfiles/swh-indexer-worker/origin_head.yml +++ /dev/null @@ -1,19 +0,0 @@ -storage: - cls: remote - args: - url: http://swh-storage:5002/ - -indexer_storage: - cls: remote - args: - url: http://swh-idx-storage:5007/ - -scheduler: - cls: remote - args: - url: http://swh-scheduler-api:5008/ - -tasks: - revision_metadata: indexer_revision_metadata - origin_intrinsic_metadata: indexer_origin_metadata - diff --git a/dockerfiles/swh-indexer-worker/origin_intrinsic_metadata.yml b/dockerfiles/swh-indexer-worker/origin_intrinsic_metadata.yml deleted file mode 100644 --- a/dockerfiles/swh-indexer-worker/origin_intrinsic_metadata.yml +++ /dev/null @@ -1,9 +0,0 @@ -storage: - cls: remote - args: - url: http://swh-storage:5002/ - -indexer_storage: - cls: remote - args: - url: http://swh-idx-storage:5007/ diff --git a/indexers.env b/indexers.env --- a/indexers.env +++ b/indexers.env @@ -1,3 +1,5 @@ -CONCURRENCY=1 +CONCURRENCY=4 MAX_TASKS_PER_CHILD=10 LOGLEVEL=DEBUG +SWH_WORKER_INSTANCE=indexer +SWH_CONFIG_FILENAME=/home/swh/indexer.yml