Page MenuHomeSoftware Heritage

D1052.id3402.diff
No OneTemporary

D1052.id3402.diff

diff --git a/docker-compose.yml b/docker-compose.yml
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -259,14 +259,13 @@
# Indexer Celery workers
- swh-indexer-mimetype:
+ swh-indexer:
image: swh/indexer-worker
build: ./dockerfiles/swh-indexer-worker
env_file: ./indexers.env
environment:
STATSD_HOST: prometheus-statsd-exporter
STATSD_PORT: 9125
- SWH_WORKER_INSTANCE: content_mimetype
depends_on:
- swh-scheduler-runner
- swh-idx-storage
@@ -274,44 +273,6 @@
- swh-objstorage
- amqp
- swh-indexer-license:
- image: swh/indexer-worker
- build: ./dockerfiles/swh-indexer-worker
- env_file: ./indexers.env
- environment:
- STATSD_HOST: prometheus-statsd-exporter
- STATSD_PORT: 9125
- SWH_WORKER_INSTANCE: content_fossology_license
- depends_on:
- - swh-scheduler-runner
- - swh-idx-storage
- - swh-storage
- - swh-objstorage
- - amqp
-
- swh-indexer-origin-head:
- image: swh/indexer-worker
- build: ./dockerfiles/swh-indexer-worker
- env_file: ./indexers.env
- environment:
- STATSD_HOST: prometheus-statsd-exporter
- STATSD_PORT: 9125
- SWH_WORKER_INSTANCE: origin_head
- depends_on:
- - swh-scheduler-api
- - swh-scheduler-runner
- - swh-idx-storage
- - swh-storage
- - amqp
-
- swh-indexer-revision-metadata:
- image: swh/indexer-worker
- build: ./dockerfiles/swh-indexer-worker
- env_file: ./indexers.env
- environment:
- STATSD_HOST: prometheus-statsd-exporter
- STATSD_PORT: 9125
- SWH_WORKER_INSTANCE: revision_metadata
depends_on:
- swh-scheduler-runner
- swh-idx-storage
@@ -319,20 +280,6 @@
- swh-objstorage
- amqp
- swh-indexer-origin-intrinsic-metadata:
- image: swh/indexer-worker
- build: ./dockerfiles/swh-indexer-worker
- env_file: ./indexers.env
- environment:
- STATSD_HOST: prometheus-statsd-exporter
- STATSD_PORT: 9125
- SWH_WORKER_INSTANCE: origin_intrinsic_metadata
- depends_on:
- - swh-scheduler-runner
- - swh-idx-storage
- - swh-storage
- - amqp
-
swh-indexer-journal-client:
image: swh/indexer-journal-client
build: ./dockerfiles/swh-indexer-journal-client
diff --git a/dockerfiles/swh-indexer-worker/Dockerfile b/dockerfiles/swh-indexer-worker/Dockerfile
--- a/dockerfiles/swh-indexer-worker/Dockerfile
+++ b/dockerfiles/swh-indexer-worker/Dockerfile
@@ -5,16 +5,12 @@
apt-get install -y \
libsystemd-dev postgresql-client
+RUN pip install --upgrade pip
RUN pip install swh-indexer
RUN useradd -ms /bin/bash swh
-COPY entrypoint.sh /
-
-COPY mimetype.yml /home/swh/.config/swh/indexer/
-COPY fossology_license.yml /home/swh/.config/swh/indexer/
-COPY origin_head.yml /home/swh/.config/swh/indexer/
-COPY revision_metadata.yml /home/swh/.config/swh/indexer/
-COPY origin_intrinsic_metadata.yml /home/swh/.config/swh/indexer/
-RUN chown -R swh: /home/swh/.config/
USER swh
-ENTRYPOINT ["/entrypoint.sh"]
+COPY entrypoint.sh /
+COPY indexer.yml /home/swh/
+
+ENTRYPOINT ["/home/swh/entrypoint.sh"]
diff --git a/dockerfiles/swh-indexer-worker/fossology_license.yml b/dockerfiles/swh-indexer-worker/fossology_license.yml
deleted file mode 100644
--- a/dockerfiles/swh-indexer-worker/fossology_license.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-storage:
- cls: remote
- args:
- url: http://swh-storage:5002/
-objstorage:
- cls: remote
- args:
- url: http://swh-objstorage:5003/
-indexer_storage:
- cls: remote
- args:
- url: http://swh-idx-storage:5007/
diff --git a/dockerfiles/swh-indexer-worker/revision_metadata.yml b/dockerfiles/swh-indexer-worker/indexer.yml
rename from dockerfiles/swh-indexer-worker/revision_metadata.yml
rename to dockerfiles/swh-indexer-worker/indexer.yml
--- a/dockerfiles/swh-indexer-worker/revision_metadata.yml
+++ b/dockerfiles/swh-indexer-worker/indexer.yml
@@ -2,17 +2,14 @@
cls: remote
args:
url: http://swh-storage:5002/
-
objstorage:
cls: remote
args:
url: http://swh-objstorage:5003/
-
indexer_storage:
cls: remote
args:
url: http://swh-idx-storage:5007/
-
scheduler:
cls: remote
args:
diff --git a/dockerfiles/swh-indexer-worker/mimetype.yml b/dockerfiles/swh-indexer-worker/mimetype.yml
deleted file mode 100644
--- a/dockerfiles/swh-indexer-worker/mimetype.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-storage:
- cls: remote
- args:
- url: http://swh-storage:5002/
-objstorage:
- cls: remote
- args:
- url: http://swh-objstorage:5003/
-indexer_storage:
- cls: remote
- args:
- url: http://swh-idx-storage:5007/
diff --git a/dockerfiles/swh-indexer-worker/origin_head.yml b/dockerfiles/swh-indexer-worker/origin_head.yml
deleted file mode 100644
--- a/dockerfiles/swh-indexer-worker/origin_head.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-storage:
- cls: remote
- args:
- url: http://swh-storage:5002/
-
-indexer_storage:
- cls: remote
- args:
- url: http://swh-idx-storage:5007/
-
-scheduler:
- cls: remote
- args:
- url: http://swh-scheduler-api:5008/
-
-tasks:
- revision_metadata: indexer_revision_metadata
- origin_intrinsic_metadata: indexer_origin_metadata
-
diff --git a/dockerfiles/swh-indexer-worker/origin_intrinsic_metadata.yml b/dockerfiles/swh-indexer-worker/origin_intrinsic_metadata.yml
deleted file mode 100644
--- a/dockerfiles/swh-indexer-worker/origin_intrinsic_metadata.yml
+++ /dev/null
@@ -1,9 +0,0 @@
-storage:
- cls: remote
- args:
- url: http://swh-storage:5002/
-
-indexer_storage:
- cls: remote
- args:
- url: http://swh-idx-storage:5007/
diff --git a/indexers.env b/indexers.env
--- a/indexers.env
+++ b/indexers.env
@@ -1,3 +1,5 @@
-CONCURRENCY=1
+CONCURRENCY=4
MAX_TASKS_PER_CHILD=10
LOGLEVEL=DEBUG
+SWH_WORKER_INSTANCE=indexer
+SWH_CONFIG_FILENAME=/home/swh/indexer.yml

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 3, 1:34 PM (6 d, 6 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3226256

Event Timeline