diff --git a/Dockerfile b/Dockerfile index 072a9df..af78d10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,46 @@ FROM python:3.7 RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update && apt-get upgrade -y && \ apt-get install -y \ libapr1-dev \ libaprutil1-dev \ libpq-dev \ libsvn-dev \ libsystemd-dev \ postgresql-client \ wait-for-it \ ngrep && \ apt-get install -y --no-install-recommends \ r-base-core \ r-cran-jsonlite && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* RUN useradd -md /srv/softwareheritage -s /bin/bash swh USER swh RUN python3 -m venv /srv/softwareheritage/venv ENV PATH="/srv/softwareheritage/venv/bin:${PATH}" RUN pip install --upgrade pip setuptools wheel RUN pip install 'gunicorn<20' RUN pip install cassandra-driver RUN pip install \ swh-core[db,http] \ swh-deposit[server] \ swh-indexer \ swh-journal \ swh-lister \ + swh-loader-core \ swh-loader-git \ swh-loader-mercurial \ swh-loader-svn \ swh-storage \ swh-objstorage \ swh-scheduler \ swh-vault \ swh-web COPY utils/*.sh /srv/softwareheritage/utils/ RUN mkdir -p /srv/softwareheritage/objects diff --git a/conf/loader.yml b/conf/loader.yml index 285779b..debcc93 100644 --- a/conf/loader.yml +++ b/conf/loader.yml @@ -1,54 +1,50 @@ storage: cls: filter args: storage: cls: buffer args: storage: cls: remote args: url: http://swh-storage:5002/ min_batch_size: content: 10000 content_bytes: 104857600 directory: 1000 revision: 1000 scheduler: cls: remote args: url: http://swh-scheduler-api:5008/ celery: task_broker: amqp://guest:guest@amqp// task_modules: - - swh.loader.dir.tasks - swh.loader.git.tasks - swh.loader.mercurial.tasks - - swh.loader.npm.tasks - - swh.loader.pypi.tasks - swh.loader.svn.tasks - - swh.loader.tar.tasks - swh.deposit.loader.tasks - swh.loader.package.tasks task_queues: - - swh.loader.package.tasks.LoadDebian - swh.loader.dir.tasks.LoadDirRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - swh.loader.git.tasks.UpdateGitRepository - swh.loader.mercurial.tasks.LoadArchiveMercurial - swh.loader.mercurial.tasks.LoadMercurial - - swh.loader.npm.tasks.LoadNpm - - swh.loader.pypi.tasks.LoadPyPI + - swh.loader.package.tasks.LoadArchive + - swh.loader.package.tasks.LoadDebian + - swh.loader.package.tasks.LoadNpm + - swh.loader.package.tasks.LoadPyPI - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - swh.loader.svn.tasks.LoadSvnRepository - swh.loader.svn.tasks.MountAndLoadSvnRepository - - swh.loader.tar.tasks.LoadTarRepository - swh.deposit.loader.tasks.LoadDepositArchiveTsk - swh.deposit.loader.tasks.ChecksDepositTsk lister_db_url: postgresql://postgres@swh-listers-db/swh-listers url: 'http://swh-deposit:5006'