diff --git a/docker/conf/deposit.yml b/docker/conf/deposit.yml index 74fe65e..ae00b79 100644 --- a/docker/conf/deposit.yml +++ b/docker/conf/deposit.yml @@ -1,29 +1,26 @@ scheduler: cls: remote - args: - url: http://swh-scheduler:5008 + url: http://swh-scheduler:5008 storage: cls: remote - args: - url: http://swh-storage:5002/ + url: http://swh-storage:5002/ storage_metadata: cls: remote - args: - url: http://swh-storage:5002/ + url: http://swh-storage:5002/ allowed_hosts: - '*' private: secret_key: prod-in-docker db: host: swh-deposit-db port: 5432 name: swh-deposit user: postgres password: testpassword media_root: /tmp/swh-deposit/uploads extraction_dir: "/tmp/swh-deposit/archive/" diff --git a/docker/conf/indexer.yml b/docker/conf/indexer.yml index bfcdfc8..4673622 100644 --- a/docker/conf/indexer.yml +++ b/docker/conf/indexer.yml @@ -1,30 +1,27 @@ storage: cls: remote - args: - url: http://swh-storage:5002/ + url: http://swh-storage:5002/ objstorage: cls: remote - args: - url: http://swh-objstorage:5003/ + url: http://swh-objstorage:5003/ indexer_storage: cls: remote url: http://swh-idx-storage:5007/ scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ celery: task_broker: amqp://guest:guest@amqp// task_modules: - swh.indexer.tasks task_queues: - swh.indexer.tasks.ContentFossologyLicense - swh.indexer.tasks.ContentLanguage - swh.indexer.tasks.ContentMimetype - swh.indexer.tasks.ContentRangeFossologyLicense - swh.indexer.tasks.ContentRangeMimetype - swh.indexer.tasks.Ctags - swh.indexer.tasks.OriginHead - swh.indexer.tasks.OriginMetadata - swh.indexer.tasks.RecomputeChecksums - swh.indexer.tasks.RevisionMetadata diff --git a/docker/conf/indexer_journal_client.yml b/docker/conf/indexer_journal_client.yml index e8c030d..23c3e3f 100644 --- a/docker/conf/indexer_journal_client.yml +++ b/docker/conf/indexer_journal_client.yml @@ -1,11 +1,10 @@ journal: brokers: - kafka group_id: swh.indexer.journal_client max_messages: 50 scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ diff --git a/docker/conf/lister.yml b/docker/conf/lister.yml index b887af0..ba5bbec 100644 --- a/docker/conf/lister.yml +++ b/docker/conf/lister.yml @@ -1,43 +1,41 @@ storage: cls: remote - args: - url: http://swh-storage:5002/ + url: http://swh-storage:5002/ scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ lister: cls: local args: db: postgresql://postgres@swh-listers-db/swh-listers celery: task_broker: amqp://guest:guest@amqp// task_queues: - swh.lister.bitbucket.tasks.FullBitBucketRelister - swh.lister.bitbucket.tasks.IncrementalBitBucketLister - swh.lister.bitbucket.tasks.RangeBitBucketLister - swh.lister.cgit.tasks.CGitListerTask - swh.lister.cran.tasks.CRANListerTask - swh.lister.debian.tasks.DebianListerTask - swh.lister.gitea.tasks.FullGiteaRelister - swh.lister.gitea.tasks.IncrementalGiteaLister - swh.lister.gitea.tasks.RangeGiteaLister - swh.lister.github.tasks.FullGitHubRelister - swh.lister.github.tasks.IncrementalGitHubLister - swh.lister.github.tasks.RangeGitHubLister - swh.lister.gitlab.tasks.FullGitLabRelister - swh.lister.gitlab.tasks.IncrementalGitLabLister - swh.lister.gitlab.tasks.RangeGitLabLister - swh.lister.gnu.tasks.GNUListerTask - swh.lister.npm.tasks.NpmIncrementalListerTask - swh.lister.npm.tasks.NpmListerTask - swh.lister.launchpad.tasks.IncrementalLaunchpadLister - swh.lister.launchpad.tasks.FullLaunchpadLister - swh.lister.launchpad.tasks.NewLaunchpadLister - swh.lister.packagist.tasks.PackagistListerTask - swh.lister.phabricator.tasks.FullPhabricatorLister - swh.lister.phabricator.tasks.IncrementalPhabricatorLister - swh.lister.pypi.tasks.PyPIListerTask diff --git a/docker/conf/loader.yml b/docker/conf/loader.yml index 985bfa7..78db274 100644 --- a/docker/conf/loader.yml +++ b/docker/conf/loader.yml @@ -1,49 +1,48 @@ storage: cls: pipeline steps: - cls: buffer min_batch_size: content: 10000 content_bytes: 104857600 directory: 1000 revision: 1000 - cls: filter - cls: remote url: http://swh-storage:5002/ scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ deposit: url: http://swh-deposit:5006/1/private auth: username: test password: test celery: task_broker: amqp://guest:guest@amqp// task_queues: - swh.loader.dir.tasks.LoadDirRepository - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository - swh.loader.git.tasks.UpdateGitRepository - swh.loader.mercurial.tasks.LoadArchiveMercurial - swh.loader.mercurial.tasks.LoadMercurial - swh.loader.package.archive.tasks.LoadArchive - swh.loader.package.cran.tasks.LoadCRAN - swh.loader.package.debian.tasks.LoadDebian - swh.loader.package.deposit.tasks.LoadDeposit - swh.loader.package.npm.tasks.LoadNpm - swh.loader.package.pypi.tasks.LoadPyPI - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository - swh.loader.svn.tasks.LoadSvnRepository - swh.loader.svn.tasks.MountAndLoadSvnRepository - swh.deposit.loader.tasks.ChecksDepositTsk lister_db_url: postgresql://postgres@swh-listers-db/swh-listers url: 'http://swh-deposit:5006' diff --git a/docker/conf/objstorage.yml b/docker/conf/objstorage.yml index dde0323..e7c6fb8 100644 --- a/docker/conf/objstorage.yml +++ b/docker/conf/objstorage.yml @@ -1,7 +1,6 @@ objstorage: cls: pathslicing - args: - root: /srv/softwareheritage/objects - slicing: 0:5 + root: /srv/softwareheritage/objects + slicing: 0:5 client_max_size: 1073741824 diff --git a/docker/conf/scheduler.yml b/docker/conf/scheduler.yml index 3f3949d..71346ad 100644 --- a/docker/conf/scheduler.yml +++ b/docker/conf/scheduler.yml @@ -1,8 +1,7 @@ scheduler: cls: local - args: - db: postgresql:///?service=swh-scheduler + db: postgresql:///?service=swh-scheduler celery: task_broker: amqp://guest:guest@amqp/%2f broker_transport_options: max_retries: 1 diff --git a/docker/conf/search.yml b/docker/conf/search.yml index 883f6f2..b5f469a 100644 --- a/docker/conf/search.yml +++ b/docker/conf/search.yml @@ -1,5 +1,4 @@ search: cls: elasticsearch - args: - hosts: - - elasticsearch:9200 + hosts: + - elasticsearch:9200 diff --git a/docker/conf/search_journal_client_objects.yml b/docker/conf/search_journal_client_objects.yml index 4248935..58b8740 100644 --- a/docker/conf/search_journal_client_objects.yml +++ b/docker/conf/search_journal_client_objects.yml @@ -1,10 +1,9 @@ search: cls: remote - args: - url: http://swh-search:5010/ + url: http://swh-search:5010/ journal: brokers: - kafka group_id: swh.search.journal_client.objects prefix: swh.journal.objects diff --git a/docker/conf/storage-mirror.yml b/docker/conf/storage-mirror.yml index 91bb727..6237bc7 100644 --- a/docker/conf/storage-mirror.yml +++ b/docker/conf/storage-mirror.yml @@ -1,8 +1,6 @@ storage: cls: local - args: - db: postgresql:///?service=swh-storage-mirror - objstorage: - cls: remote - args: - url: http://swh-objstorage:5003/ + db: postgresql:///?service=swh-storage-mirror + objstorage: + cls: remote + url: http://swh-objstorage:5003/ diff --git a/docker/conf/storage-read-replica.yml b/docker/conf/storage-read-replica.yml index 5c05c56..45b3227 100644 --- a/docker/conf/storage-read-replica.yml +++ b/docker/conf/storage-read-replica.yml @@ -1,13 +1,10 @@ storage: cls: local - args: - db: postgresql:///?service=swh-storage-read-replica - objstorage: - cls: filtered - args: - storage_conf: - cls: remote - args: - url: http://swh-objstorage:5003/ - filters_conf: - - type: readonly + db: postgresql:///?service=swh-storage-read-replica + objstorage: + cls: filtered + storage_conf: + cls: remote + url: http://swh-objstorage:5003/ + filters_conf: + - type: readonly diff --git a/docker/conf/storage.yml b/docker/conf/storage.yml index a27b3be..cd7d4dd 100644 --- a/docker/conf/storage.yml +++ b/docker/conf/storage.yml @@ -1,15 +1,13 @@ storage: cls: local - args: - db: postgresql:///?service=swh-storage - objstorage: - cls: remote - args: - url: http://swh-objstorage:5003/ - journal_writer: - cls: kafka - args: - brokers: - - kafka - prefix: swh.journal.objects - client_id: swh.storage.master + db: postgresql:///?service=swh-storage + objstorage: + cls: remote + url: http://swh-objstorage:5003/ + journal_writer: + cls: kafka + args: + brokers: + - kafka + prefix: swh.journal.objects + client_id: swh.storage.master diff --git a/docker/conf/storage_cassandra.yml b/docker/conf/storage_cassandra.yml index a4d1ec4..a463715 100644 --- a/docker/conf/storage_cassandra.yml +++ b/docker/conf/storage_cassandra.yml @@ -1,11 +1,9 @@ storage: cls: cassandra - args: - hosts: + hosts: - cassandra-seed - keyspace: swh - objstorage: - cls: remote - args: - url: http://swh-objstorage:5003/ + keyspace: swh + objstorage: + cls: remote + url: http://swh-objstorage:5003/ diff --git a/docker/conf/vault-read-replica.yml b/docker/conf/vault-read-replica.yml index 081ced8..ba28aca 100644 --- a/docker/conf/vault-read-replica.yml +++ b/docker/conf/vault-read-replica.yml @@ -1,17 +1,14 @@ storage: cls: remote - args: - url: http://swh-storage-read-replica:5002/ + url: http://swh-storage-read-replica:5002/ scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ vault: cls: local args: db: postgresql:///?service=swh-vault cache: cls: pathslicing - args: - root: /srv/softwareheritage/vault - slicing: 0:5 + root: /srv/softwareheritage/vault + slicing: 0:5 diff --git a/docker/conf/vault-worker-read-replica.yml b/docker/conf/vault-worker-read-replica.yml index 67fa182..879b594 100644 --- a/docker/conf/vault-worker-read-replica.yml +++ b/docker/conf/vault-worker-read-replica.yml @@ -1,17 +1,15 @@ storage: cls: remote - args: - url: http://swh-storage-read-replica:5002/ + url: http://swh-storage-read-replica:5002/ vault: cls: remote - args: - url: http://swh-vault:5005/ + url: http://swh-vault:5005/ celery: task_broker: amqp://guest:guest@amqp// task_modules: - swh.vault.cooking_tasks task_queues: - swh.vault.cooking_tasks.SWHBatchCookingTask - swh.vault.cooking_tasks.SWHCookingTask max_bundle_size: 536870912 diff --git a/docker/conf/vault-worker.yml b/docker/conf/vault-worker.yml index b9b463e..f6c0554 100644 --- a/docker/conf/vault-worker.yml +++ b/docker/conf/vault-worker.yml @@ -1,17 +1,16 @@ storage: cls: remote - args: - url: http://swh-storage:5002/ + url: http://swh-storage:5002/ vault: cls: remote args: url: http://swh-vault:5005/ celery: task_broker: amqp://guest:guest@amqp// task_modules: - swh.vault.cooking_tasks task_queues: - swh.vault.cooking_tasks.SWHBatchCookingTask - swh.vault.cooking_tasks.SWHCookingTask max_bundle_size: 536870912 diff --git a/docker/conf/vault.yml b/docker/conf/vault.yml index 5d00ae1..ca191f0 100644 --- a/docker/conf/vault.yml +++ b/docker/conf/vault.yml @@ -1,17 +1,14 @@ storage: cls: remote - args: - url: http://swh-storage:5002/ + url: http://swh-storage:5002/ scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ vault: cls: local args: db: postgresql:///?service=swh-vault cache: cls: pathslicing - args: - root: /srv/softwareheritage/vault - slicing: 0:5 + root: /srv/softwareheritage/vault + slicing: 0:5 diff --git a/docker/conf/web-keycloak.yml b/docker/conf/web-keycloak.yml index 40a5f54..576e57c 100644 --- a/docker/conf/web-keycloak.yml +++ b/docker/conf/web-keycloak.yml @@ -1,65 +1,61 @@ storage: cls: remote - args: - url: http://swh-storage:5002/ - timeout: 1 + url: http://swh-storage:5002/ + timeout: 1 objstorage: cls: remote - args: - url: http://swh-objstorage:5003/ + url: http://swh-objstorage:5003/ indexer_storage: cls: remote - args: - url: http://swh-idx-storage:5007/ + url: http://swh-idx-storage:5007/ scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ vault: cls: remote args: url: http://swh-vault:5005/ deposit: private_api_url: https://swh-deposit:5006/1/private/ private_api_user: swhworker private_api_password: '' allowed_hosts: - "*" debug: yes serve_assets: yes development_db: /tmp/db.sqlite3 production_db: /tmp/db.sqlite3 throttling: cache_uri: 127.0.0.1:11211 scopes: swh_api: limiter_rate: default: 120/h swh_api_origin_search: limiter_rate: default: 70/m swh_api_origin_visit_latest: limiter_rate: default: 700/m swh_vault_cooking: limiter_rate: default: 120/h swh_save_origin: limiter_rate: default: 120/h search: {} keycloak: server_url: http://keycloak:8080/keycloak/auth/ realm_name: SoftwareHeritage diff --git a/docker/conf/web-mirror.yml b/docker/conf/web-mirror.yml index 88253ba..57a0605 100644 --- a/docker/conf/web-mirror.yml +++ b/docker/conf/web-mirror.yml @@ -1,37 +1,33 @@ storage: cls: remote - args: - url: http://swh-storage-mirror:5002/ - timeout: 1 + url: http://swh-storage-mirror:5002/ + timeout: 1 objstorage: cls: remote - args: - url: http://swh-objstorage:5003/ + url: http://swh-objstorage:5003/ indexer_storage: cls: remote - args: - url: http://swh-idx-storage:5007/ + url: http://swh-idx-storage:5007/ scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ vault: cls: remote args: url: http://swh-vault:5005/ deposit: private_api_url: https://swh-deposit:5006/1/private/ private_api_user: swhworker private_api_password: '' allowed_hosts: - "*" debug: yes serve_assets: yes diff --git a/docker/conf/web-read-replica.yml b/docker/conf/web-read-replica.yml index daa289e..7fdea9a 100644 --- a/docker/conf/web-read-replica.yml +++ b/docker/conf/web-read-replica.yml @@ -1,66 +1,63 @@ storage: cls: remote - args: - url: http://swh-storage-read-replica:5002/ - timeout: 1 + url: http://swh-storage-read-replica:5002/ + timeout: 1 indexer_storage: cls: remote - args: - url: http://swh-idx-storage:5007/ + url: http://swh-idx-storage:5007/ scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ vault: cls: remote args: url: http://swh-vault:5005/ deposit: private_api_url: https://swh-deposit:5006/1/private/ private_api_user: swhworker private_api_password: '' allowed_hosts: - "*" debug: yes serve_assets: yes development_db: /tmp/db.sqlite3 production_db: /tmp/db.sqlite3 throttling: cache_uri: 127.0.0.1:11211 scopes: swh_api: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 swh_api_origin_search: limiter_rate: default: 70/m exempted_networks: - 0.0.0.0/0 swh_api_origin_visit_latest: limiter_rate: default: 700/m exempted_networks: - 0.0.0.0/0 swh_vault_cooking: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 swh_save_origin: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 search: {} diff --git a/docker/conf/web.yml b/docker/conf/web.yml index 9864daf..d07b627 100644 --- a/docker/conf/web.yml +++ b/docker/conf/web.yml @@ -1,66 +1,63 @@ storage: cls: remote - args: - url: http://swh-storage:5002/ - timeout: 1 + url: http://swh-storage:5002/ + timeout: 1 indexer_storage: cls: remote - args: - url: http://swh-idx-storage:5007/ + url: http://swh-idx-storage:5007/ scheduler: cls: remote - args: - url: http://swh-scheduler:5008/ + url: http://swh-scheduler:5008/ vault: cls: remote args: url: http://swh-vault:5005/ deposit: private_api_url: https://swh-deposit:5006/1/private/ private_api_user: swhworker private_api_password: '' allowed_hosts: - "*" debug: yes serve_assets: yes development_db: /tmp/db.sqlite3 production_db: /tmp/db.sqlite3 throttling: cache_uri: 127.0.0.1:11211 scopes: swh_api: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 swh_api_origin_search: limiter_rate: default: 70/m exempted_networks: - 0.0.0.0/0 swh_api_origin_visit_latest: limiter_rate: default: 700/m exempted_networks: - 0.0.0.0/0 swh_vault_cooking: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 swh_save_origin: limiter_rate: default: 120/h exempted_networks: - 0.0.0.0/0 search: {}