Changeset View
Changeset View
Standalone View
Standalone View
data/defaults.yaml
Show First 20 Lines • Show All 1,892 Lines • ▼ Show 20 Lines | - host: esnode3.internal.softwareheritage.org | ||||
port: 9200 | port: 9200 | ||||
- host: esnode1.internal.softwareheritage.org | - host: esnode1.internal.softwareheritage.org | ||||
port: 9200 | port: 9200 | ||||
client_options: | client_options: | ||||
sniff_on_start: false | sniff_on_start: false | ||||
sniff_on_connection_fail: True | sniff_on_connection_fail: True | ||||
http_compress: false | http_compress: false | ||||
swh::deploy::loader_debian::db::host: belvedere.internal.softwareheritage.org | # Main lister configuration | ||||
ardumont: As discussed earlier, i tried to rename that to `swhlister` but then i could no longer connect… | |||||
swh::deploy::loader_debian::db::port: "%{alias('swh::deploy::db::secondary::port')}" | swh::deploy::worker::lister::db::user: swh-lister | ||||
swh::deploy::loader_debian::db::user: lister-debian | swh::deploy::worker::lister::db::name: swh-lister | ||||
# in private data: swh::deploy::loader_debian::db::password | swh::deploy::worker::lister::db::host: belvedere.internal.softwareheritage.org | ||||
swh::deploy::loader_debian::db::dbname: lister-debian | swh::deploy::worker::lister::db::port: "%{alias('swh::deploy::db::secondary::port')}" | ||||
# swh::deploy::worker::lister::db::password in private data | |||||
swh::deploy::worker::loader_debian::db::host: belvedere.internal.softwareheritage.org | |||||
swh::deploy::worker::loader_debian::db::port: "%{alias('swh::deploy::db::secondary::port')}" | |||||
# in private data: swh::deploy::worker::loader_debian::db::password | |||||
swh::deploy::worker::loader_debian::db::user: "%{alias('swh::deploy::worker::lister::db::user')}" | |||||
swh::deploy::worker::loader_debian::db::dbname: "%{alias('swh::deploy::worker::lister::db::name')}" | |||||
# swh::deploy::worker::task_broker::password in private data | # swh::deploy::worker::task_broker::password in private data | ||||
swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@rabbitmq:5672//" | swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@rabbitmq:5672//" | ||||
swh::deploy::worker::instances: | swh::deploy::worker::instances: | ||||
- loader_debian | - loader_debian | ||||
- loader_git | - loader_git | ||||
- lister_debian | - lister | ||||
- lister_github | |||||
swh::deploy::worker::loader_git::config_file: "%{hiera('swh::conf_directory')}/loader_git.yml" | swh::deploy::worker::loader_git::config_file: "%{hiera('swh::conf_directory')}/loader_git.yml" | ||||
swh::deploy::worker::loader_git::concurrency: 2 | swh::deploy::worker::loader_git::concurrency: 2 | ||||
swh::deploy::worker::loader_git::loglevel: info | swh::deploy::worker::loader_git::loglevel: info | ||||
swh::deploy::worker::loader_git::config: | swh::deploy::worker::loader_git::config: | ||||
storage: "%{alias('swh::remote_service::storage::config::writable')}" | storage: "%{alias('swh::remote_service::storage::config::writable')}" | ||||
save_data: true | save_data: true | ||||
save_data_path: /srv/storage/space/data/sharded_packfiles | save_data_path: /srv/storage/space/data/sharded_packfiles | ||||
Show All 10 Lines | task_queues: | ||||
- swh.loader.git.tasks.UncompressAndLoadDiskGitRepository | - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository | ||||
swh::deploy::worker::loader_debian::config_file: "%{hiera('swh::conf_directory')}/loader_debian.yml" | swh::deploy::worker::loader_debian::config_file: "%{hiera('swh::conf_directory')}/loader_debian.yml" | ||||
swh::deploy::worker::loader_debian::concurrency: 1 | swh::deploy::worker::loader_debian::concurrency: 1 | ||||
swh::deploy::worker::loader_debian::loglevel: info | swh::deploy::worker::loader_debian::loglevel: info | ||||
swh::deploy::worker::loader_debian::config: | swh::deploy::worker::loader_debian::config: | ||||
storage: "%{alias('swh::remote_service::storage::config::writable')}" | storage: "%{alias('swh::remote_service::storage::config::writable')}" | ||||
directory_packet_size: 100 | directory_packet_size: 100 | ||||
lister_db_url: "postgresql://%{hiera('swh::deploy::loader_debian::db::user')}:%{hiera('swh::deploy::loader_debian::db::password')}@%{hiera('swh::deploy::loader_debian::db::host')}:%{hiera('swh::deploy::loader_debian::db::port')}/%{hiera('swh::deploy::loader_debian::db::dbname')}" | lister_db_url: "postgresql://%{hiera('swh::deploy::worker::loader_debian::db::user')}:%{hiera('swh::deploy::worker::loader_debian::db::password')}@%{hiera('swh::deploy::worker::loader_debian::db::host')}:%{hiera('swh::deploy::worker::loader_debian::db::port')}/%{hiera('swh::deploy::worker::loader_debian::db::dbname')}" | ||||
celery: | celery: | ||||
task_broker: "%{alias('swh::deploy::worker::task_broker')}" | task_broker: "%{alias('swh::deploy::worker::task_broker')}" | ||||
task_modules: | task_modules: | ||||
- swh.loader.debian.tasks | - swh.loader.debian.tasks | ||||
task_queues: | task_queues: | ||||
- swh.loader.debian.tasks.LoadDebianPackage | - swh.loader.debian.tasks.LoadDebianPackage | ||||
- swh_loader_debian | - swh_loader_debian | ||||
swh::deploy::worker::lister::config_file: "%{hiera('swh::conf_directory')}/lister.yml" | |||||
swh::deploy::worker::lister_debian::config_file: "%{hiera('swh::conf_directory')}/lister_debian.yml" | swh::deploy::worker::lister::concurrency: 5 | ||||
swh::deploy::worker::lister_debian::concurrency: 1 | swh::deploy::worker::lister::loglevel: warning | ||||
swh::deploy::worker::lister_debian::loglevel: info | swh::deploy::worker::lister::config: | ||||
swh::deploy::worker::lister_debian::config: | |||||
storage: "%{alias('swh::remote_service::storage::config::writable')}" | storage: "%{alias('swh::remote_service::storage::config::writable')}" | ||||
scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" | scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" | ||||
lister: | lister: | ||||
cls: local | cls: local | ||||
args: | args: | ||||
db: "postgresql://%{hiera('swh::deploy::loader_debian::db::user')}:%{hiera('swh::deploy::loader_debian::db::password')}@%{hiera('swh::deploy::loader_debian::db::host')}:%{hiera('swh::deploy::loader_debian::db::port')}/%{hiera('swh::deploy::loader_debian::db::dbname')}" | db: "postgresql://%{hiera('swh::deploy::worker::lister::db::user')}:%{hiera('swh::deploy::worker::lister::db::password')}@%{hiera('swh::deploy::worker::lister::db::host')}:%{hiera('swh::deploy::worker::lister::db::port')}/%{hiera('swh::deploy::worker::lister::db::name')}" | ||||
celery: | celery: | ||||
task_broker: "%{alias('swh::deploy::worker::task_broker')}" | task_broker: "%{alias('swh::deploy::worker::task_broker')}" | ||||
task_modules: | task_modules: | ||||
- swh.lister.debian.tasks | - swh.lister.debian.tasks | ||||
- swh.lister.github.tasks | |||||
- swh.lister.gitlab.tasks | |||||
- swh.lister.npm.tasks | |||||
- swh.lister.pypi.tasks | |||||
task_queues: | task_queues: | ||||
- swh.lister.debian.tasks.DebianListerTask | - swh.lister.debian.tasks.DebianListerTask | ||||
- swh_lister_debian | - swh.lister.github.tasks.IncrementalGitHubLister | ||||
- swh.lister.github.tasks.RangeGitHubLister | |||||
- swh.lister.github.tasks.FullGitHubRelister | |||||
- swh.lister.gitlab.tasks.IncrementalGitLabLister | |||||
- swh.lister.gitlab.tasks.RangeGitLabLister | |||||
- swh.lister.gitlab.tasks.FullGitLabRelister | |||||
- swh.lister.npm.tasks.NpmListerTask | |||||
Done Inline ActionsI dropped the legacy queue names which are no longer used btw. ardumont: I dropped the legacy queue names which are no longer used btw. | |||||
- swh.lister.pypi.tasks.PyPIListerTask | |||||
credentials: "%{alias('swh::deploy::worker::lister::config::credentials')}" | |||||
swh::deploy::worker::loader_mercurial::config_file: "%{hiera('swh::conf_directory')}/loader_mercurial.yml" | swh::deploy::worker::loader_mercurial::config_file: "%{hiera('swh::conf_directory')}/loader_mercurial.yml" | ||||
swh::deploy::worker::loader_mercurial::concurrency: 1 | swh::deploy::worker::loader_mercurial::concurrency: 1 | ||||
swh::deploy::worker::loader_mercurial::private_tmp: true | swh::deploy::worker::loader_mercurial::private_tmp: true | ||||
swh::deploy::worker::loader_mercurial::loglevel: info | swh::deploy::worker::loader_mercurial::loglevel: info | ||||
swh::deploy::worker::loader_mercurial::config: | swh::deploy::worker::loader_mercurial::config: | ||||
storage: "%{alias('swh::remote_service::storage::config::writable')}" | storage: "%{alias('swh::remote_service::storage::config::writable')}" | ||||
reduce_effort: False | reduce_effort: False | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | task_modules: | ||||
- swh.loader.svn.tasks | - swh.loader.svn.tasks | ||||
task_queues: | task_queues: | ||||
- swh.loader.svn.tasks.LoadSvnRepository | - swh.loader.svn.tasks.LoadSvnRepository | ||||
- swh.loader.svn.tasks.MountAndLoadSvnRepository | - swh.loader.svn.tasks.MountAndLoadSvnRepository | ||||
- swh.loader.svn.tasks.DumpMountAndLoadSvnRepository | - swh.loader.svn.tasks.DumpMountAndLoadSvnRepository | ||||
- swh_loader_svn | - swh_loader_svn | ||||
- swh_loader_svn_mount_and_load | - swh_loader_svn_mount_and_load | ||||
swh::deploy::worker::lister_github::config_file: "%{hiera('swh::conf_directory')}/lister_github.yml" | |||||
swh::deploy::worker::lister_github::concurrency: 1 | |||||
swh::deploy::worker::lister_github::loglevel: warning | |||||
# Contains a password: in private data | |||||
swh::deploy::worker::lister_github::db::host: belvedere.internal.softwareheritage.org | |||||
swh::deploy::worker::lister_github::db::port: "%{alias('swh::deploy::db::secondary::port')}" | |||||
swh::deploy::worker::lister_github::db::dbname: lister-github | |||||
swh::deploy::worker::lister_github::db::user: lister-github | |||||
#swh::deploy::worker::lister_github::db::password in private data | |||||
swh::deploy::worker::lister_github::config: | |||||
storage: "%{alias('swh::remote_service::storage::config::writable')}" | |||||
scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" | |||||
lister: | |||||
cls: local | |||||
args: | |||||
db: "postgresql://%{hiera('swh::deploy::worker::lister_github::db::user')}:%{hiera('swh::deploy::worker::lister_github::db::password')}@%{hiera('swh::deploy::worker::lister_github::db::host')}:%{hiera('swh::deploy::worker::lister_github::db::port')}/%{hiera('swh::deploy::worker::lister_github::db::dbname')}" | |||||
# credentials in private data | |||||
celery: | |||||
task_broker: "%{alias('swh::deploy::worker::task_broker')}" | |||||
task_modules: | |||||
- swh.lister.github.tasks | |||||
task_queues: | |||||
- swh.lister.github.tasks.IncrementalGitHubLister | |||||
- swh.lister.github.tasks.RangeGitHubLister | |||||
- swh.lister.github.tasks.FullGitHubRelister | |||||
- swh_lister_github_discover | |||||
- swh_lister_github_refresh | |||||
swh::deploy::worker::lister_gitlab::config_file: "%{hiera('swh::conf_directory')}/lister_gitlab.yml" | |||||
swh::deploy::worker::lister_gitlab::concurrency: 1 | |||||
swh::deploy::worker::lister_gitlab::loglevel: warning | |||||
# Contains a password: in private data | |||||
swh::deploy::worker::lister_gitlab::db::host: belvedere.internal.softwareheritage.org | |||||
swh::deploy::worker::lister_gitlab::db::port: "%{alias('swh::deploy::db::secondary::port')}" | |||||
swh::deploy::worker::lister_gitlab::db::dbname: lister-gitlab | |||||
swh::deploy::worker::lister_gitlab::db::user: lister-gitlab | |||||
#swh::deploy::worker::lister_gitlab::db::password in private data | |||||
swh::deploy::worker::lister_gitlab::config: | |||||
storage: "%{alias('swh::remote_service::storage::config::writable')}" | |||||
scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" | |||||
lister: | |||||
cls: local | |||||
args: | |||||
db: "postgresql://%{hiera('swh::deploy::worker::lister_gitlab::db::user')}:%{hiera('swh::deploy::worker::lister_gitlab::db::password')}@%{hiera('swh::deploy::worker::lister_gitlab::db::host')}:%{hiera('swh::deploy::worker::lister_gitlab::db::port')}/%{hiera('swh::deploy::worker::lister_gitlab::db::dbname')}" | |||||
# credentials in private data | |||||
celery: | |||||
task_broker: "%{alias('swh::deploy::worker::task_broker')}" | |||||
task_modules: | |||||
- swh.lister.gitlab.tasks | |||||
task_queues: | |||||
- swh.lister.gitlab.tasks.IncrementalGitLabLister | |||||
- swh.lister.gitlab.tasks.RangeGitLabLister | |||||
- swh.lister.gitlab.tasks.FullGitLabRelister | |||||
- swh_lister_gitlab_discover | |||||
- swh_lister_gitlab_refresh | |||||
swh::deploy::worker::lister_pypi::config_file: "%{hiera('swh::conf_directory')}/lister_pypi.yml" | |||||
swh::deploy::worker::lister_pypi::concurrency: 1 | |||||
swh::deploy::worker::lister_pypi::loglevel: warning | |||||
# Contains a password: in private data | |||||
swh::deploy::worker::lister_pypi::db::host: belvedere.internal.softwareheritage.org | |||||
swh::deploy::worker::lister_pypi::db::port: "%{alias('swh::deploy::db::secondary::port')}" | |||||
swh::deploy::worker::lister_pypi::db::dbname: lister-pypi | |||||
swh::deploy::worker::lister_pypi::db::user: lister-pypi | |||||
#swh::deploy::worker::lister_pypi::db::password in private data | |||||
swh::deploy::worker::lister_pypi::config: | |||||
storage: "%{alias('swh::remote_service::storage::config::writable')}" | |||||
scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" | |||||
lister: | |||||
cls: local | |||||
args: | |||||
db: "postgresql://%{hiera('swh::deploy::worker::lister_pypi::db::user')}:%{hiera('swh::deploy::worker::lister_pypi::db::password')}@%{hiera('swh::deploy::worker::lister_pypi::db::host')}:%{hiera('swh::deploy::worker::lister_pypi::db::port')}/%{hiera('swh::deploy::worker::lister_pypi::db::dbname')}" | |||||
# credentials in private data | |||||
celery: | |||||
task_broker: "%{alias('swh::deploy::worker::task_broker')}" | |||||
task_modules: | |||||
- swh.lister.pypi.tasks | |||||
task_queues: | |||||
- swh.lister.pypi.tasks.PyPIListerTask | |||||
- swh_lister_pypi_refresh | |||||
swh::deploy::worker::lister_npm::config_file: "%{hiera('swh::conf_directory')}/lister_npm.yml" | |||||
swh::deploy::worker::lister_npm::concurrency: 1 | |||||
swh::deploy::worker::lister_npm::loglevel: warning | |||||
# Contains a password: in private data | |||||
swh::deploy::worker::lister_npm::db::host: belvedere.internal.softwareheritage.org | |||||
swh::deploy::worker::lister_npm::db::port: "%{alias('swh::deploy::db::secondary::port')}" | |||||
swh::deploy::worker::lister_npm::db::dbname: lister-npm | |||||
swh::deploy::worker::lister_npm::db::user: lister-npm | |||||
#swh::deploy::worker::lister_npm::db::password in private data | |||||
swh::deploy::worker::lister_npm::config: | |||||
storage: "%{alias('swh::remote_service::storage::config::writable')}" | |||||
scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" | |||||
lister: | |||||
cls: local | |||||
args: | |||||
db: "postgresql://%{hiera('swh::deploy::worker::lister_npm::db::user')}:%{hiera('swh::deploy::worker::lister_npm::db::password')}@%{hiera('swh::deploy::worker::lister_npm::db::host')}:%{hiera('swh::deploy::worker::lister_npm::db::port')}/%{hiera('swh::deploy::worker::lister_npm::db::dbname')}" | |||||
# credentials in private data | |||||
celery: | |||||
task_broker: "%{alias('swh::deploy::worker::task_broker')}" | |||||
task_modules: | |||||
- swh.lister.npm.tasks | |||||
task_queues: | |||||
- swh.lister.npm.tasks.NpmListerTask | |||||
swh::deploy::worker::archiver::config_file: "%{hiera('swh::conf_directory')}/archiver.yml" | swh::deploy::worker::archiver::config_file: "%{hiera('swh::conf_directory')}/archiver.yml" | ||||
swh::deploy::worker::archiver::archival_max_age: 3600 | swh::deploy::worker::archiver::archival_max_age: 3600 | ||||
swh::deploy::worker::archiver::max_queue_length: 100000 | swh::deploy::worker::archiver::max_queue_length: 100000 | ||||
swh::deploy::worker::archiver::retention_policy: 3 | swh::deploy::worker::archiver::retention_policy: 3 | ||||
swh::deploy::worker::archiver::batch_max_size: 5 | swh::deploy::worker::archiver::batch_max_size: 5 | ||||
swh::deploy::worker::archiver::db::host: belvedere.internal.softwareheritage.org | swh::deploy::worker::archiver::db::host: belvedere.internal.softwareheritage.org | ||||
swh::deploy::worker::archiver::db::port: "%{alias('swh::deploy::db::secondary::port')}" | swh::deploy::worker::archiver::db::port: "%{alias('swh::deploy::db::secondary::port')}" | ||||
▲ Show 20 Lines • Show All 1,421 Lines • Show Last 20 Lines |
As discussed earlier, i tried to rename that to swhlister but then i could no longer connect to the db.
When renaming, the instruction mentions it drops the password, even setting it up back (for swhlister), that refuses to connect.
I did not want to lose too much time on it so for now, i moved along.