diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -1898,19 +1898,25 @@ sniff_on_connection_fail: True http_compress: false -swh::deploy::loader_debian::db::host: belvedere.internal.softwareheritage.org -swh::deploy::loader_debian::db::port: "%{alias('swh::deploy::db::secondary::port')}" -swh::deploy::loader_debian::db::user: lister-debian -# in private data: swh::deploy::loader_debian::db::password -swh::deploy::loader_debian::db::dbname: lister-debian +# Main lister configuration +swh::deploy::worker::lister::db::user: swh-lister +swh::deploy::worker::lister::db::name: swh-lister +swh::deploy::worker::lister::db::host: belvedere.internal.softwareheritage.org +swh::deploy::worker::lister::db::port: "%{alias('swh::deploy::db::secondary::port')}" +# swh::deploy::worker::lister::db::password in private data + +swh::deploy::worker::loader_debian::db::host: belvedere.internal.softwareheritage.org +swh::deploy::worker::loader_debian::db::port: "%{alias('swh::deploy::db::secondary::port')}" +# in private data: swh::deploy::worker::loader_debian::db::password +swh::deploy::worker::loader_debian::db::user: "%{alias('swh::deploy::worker::lister::db::user')}" +swh::deploy::worker::loader_debian::db::dbname: "%{alias('swh::deploy::worker::lister::db::name')}" # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@rabbitmq:5672//" swh::deploy::worker::instances: - loader_debian - loader_git - - lister_debian - - lister_github + - lister swh::deploy::worker::loader_git::config_file: "%{hiera('swh::conf_directory')}/loader_git.yml" swh::deploy::worker::loader_git::concurrency: 2 @@ -1937,7 +1943,7 @@ swh::deploy::worker::loader_debian::config: storage: "%{alias('swh::remote_service::storage::config::writable')}" directory_packet_size: 100 - lister_db_url: "postgresql://%{hiera('swh::deploy::loader_debian::db::user')}:%{hiera('swh::deploy::loader_debian::db::password')}@%{hiera('swh::deploy::loader_debian::db::host')}:%{hiera('swh::deploy::loader_debian::db::port')}/%{hiera('swh::deploy::loader_debian::db::dbname')}" + lister_db_url: "postgresql://%{hiera('swh::deploy::worker::loader_debian::db::user')}:%{hiera('swh::deploy::worker::loader_debian::db::password')}@%{hiera('swh::deploy::worker::loader_debian::db::host')}:%{hiera('swh::deploy::worker::loader_debian::db::port')}/%{hiera('swh::deploy::worker::loader_debian::db::dbname')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_modules: @@ -1946,24 +1952,35 @@ - swh.loader.debian.tasks.LoadDebianPackage - swh_loader_debian - -swh::deploy::worker::lister_debian::config_file: "%{hiera('swh::conf_directory')}/lister_debian.yml" -swh::deploy::worker::lister_debian::concurrency: 1 -swh::deploy::worker::lister_debian::loglevel: info -swh::deploy::worker::lister_debian::config: +swh::deploy::worker::lister::config_file: "%{hiera('swh::conf_directory')}/lister.yml" +swh::deploy::worker::lister::concurrency: 5 +swh::deploy::worker::lister::loglevel: warning +swh::deploy::worker::lister::config: storage: "%{alias('swh::remote_service::storage::config::writable')}" scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" lister: cls: local args: - db: "postgresql://%{hiera('swh::deploy::loader_debian::db::user')}:%{hiera('swh::deploy::loader_debian::db::password')}@%{hiera('swh::deploy::loader_debian::db::host')}:%{hiera('swh::deploy::loader_debian::db::port')}/%{hiera('swh::deploy::loader_debian::db::dbname')}" + db: "postgresql://%{hiera('swh::deploy::worker::lister::db::user')}:%{hiera('swh::deploy::worker::lister::db::password')}@%{hiera('swh::deploy::worker::lister::db::host')}:%{hiera('swh::deploy::worker::lister::db::port')}/%{hiera('swh::deploy::worker::lister::db::name')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" task_modules: - swh.lister.debian.tasks + - swh.lister.github.tasks + - swh.lister.gitlab.tasks + - swh.lister.npm.tasks + - swh.lister.pypi.tasks task_queues: - swh.lister.debian.tasks.DebianListerTask - - swh_lister_debian + - swh.lister.github.tasks.IncrementalGitHubLister + - swh.lister.github.tasks.RangeGitHubLister + - swh.lister.github.tasks.FullGitHubRelister + - swh.lister.gitlab.tasks.IncrementalGitLabLister + - swh.lister.gitlab.tasks.RangeGitLabLister + - swh.lister.gitlab.tasks.FullGitLabRelister + - swh.lister.npm.tasks.NpmListerTask + - swh.lister.pypi.tasks.PyPIListerTask + credentials: "%{alias('swh::deploy::worker::lister::config::credentials')}" swh::deploy::worker::loader_mercurial::config_file: "%{hiera('swh::conf_directory')}/loader_mercurial.yml" swh::deploy::worker::loader_mercurial::concurrency: 1 @@ -2028,110 +2045,6 @@ - swh_loader_svn - swh_loader_svn_mount_and_load -swh::deploy::worker::lister_github::config_file: "%{hiera('swh::conf_directory')}/lister_github.yml" -swh::deploy::worker::lister_github::concurrency: 1 -swh::deploy::worker::lister_github::loglevel: warning -# Contains a password: in private data -swh::deploy::worker::lister_github::db::host: belvedere.internal.softwareheritage.org -swh::deploy::worker::lister_github::db::port: "%{alias('swh::deploy::db::secondary::port')}" -swh::deploy::worker::lister_github::db::dbname: lister-github -swh::deploy::worker::lister_github::db::user: lister-github -#swh::deploy::worker::lister_github::db::password in private data -swh::deploy::worker::lister_github::config: - storage: "%{alias('swh::remote_service::storage::config::writable')}" - scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" - lister: - cls: local - args: - db: "postgresql://%{hiera('swh::deploy::worker::lister_github::db::user')}:%{hiera('swh::deploy::worker::lister_github::db::password')}@%{hiera('swh::deploy::worker::lister_github::db::host')}:%{hiera('swh::deploy::worker::lister_github::db::port')}/%{hiera('swh::deploy::worker::lister_github::db::dbname')}" - # credentials in private data - celery: - task_broker: "%{alias('swh::deploy::worker::task_broker')}" - task_modules: - - swh.lister.github.tasks - task_queues: - - swh.lister.github.tasks.IncrementalGitHubLister - - swh.lister.github.tasks.RangeGitHubLister - - swh.lister.github.tasks.FullGitHubRelister - - swh_lister_github_discover - - swh_lister_github_refresh - -swh::deploy::worker::lister_gitlab::config_file: "%{hiera('swh::conf_directory')}/lister_gitlab.yml" -swh::deploy::worker::lister_gitlab::concurrency: 1 -swh::deploy::worker::lister_gitlab::loglevel: warning -# Contains a password: in private data -swh::deploy::worker::lister_gitlab::db::host: belvedere.internal.softwareheritage.org -swh::deploy::worker::lister_gitlab::db::port: "%{alias('swh::deploy::db::secondary::port')}" -swh::deploy::worker::lister_gitlab::db::dbname: lister-gitlab -swh::deploy::worker::lister_gitlab::db::user: lister-gitlab -#swh::deploy::worker::lister_gitlab::db::password in private data -swh::deploy::worker::lister_gitlab::config: - storage: "%{alias('swh::remote_service::storage::config::writable')}" - scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" - lister: - cls: local - args: - db: "postgresql://%{hiera('swh::deploy::worker::lister_gitlab::db::user')}:%{hiera('swh::deploy::worker::lister_gitlab::db::password')}@%{hiera('swh::deploy::worker::lister_gitlab::db::host')}:%{hiera('swh::deploy::worker::lister_gitlab::db::port')}/%{hiera('swh::deploy::worker::lister_gitlab::db::dbname')}" - # credentials in private data - celery: - task_broker: "%{alias('swh::deploy::worker::task_broker')}" - task_modules: - - swh.lister.gitlab.tasks - task_queues: - - swh.lister.gitlab.tasks.IncrementalGitLabLister - - swh.lister.gitlab.tasks.RangeGitLabLister - - swh.lister.gitlab.tasks.FullGitLabRelister - - swh_lister_gitlab_discover - - swh_lister_gitlab_refresh - -swh::deploy::worker::lister_pypi::config_file: "%{hiera('swh::conf_directory')}/lister_pypi.yml" -swh::deploy::worker::lister_pypi::concurrency: 1 -swh::deploy::worker::lister_pypi::loglevel: warning -# Contains a password: in private data -swh::deploy::worker::lister_pypi::db::host: belvedere.internal.softwareheritage.org -swh::deploy::worker::lister_pypi::db::port: "%{alias('swh::deploy::db::secondary::port')}" -swh::deploy::worker::lister_pypi::db::dbname: lister-pypi -swh::deploy::worker::lister_pypi::db::user: lister-pypi -#swh::deploy::worker::lister_pypi::db::password in private data -swh::deploy::worker::lister_pypi::config: - storage: "%{alias('swh::remote_service::storage::config::writable')}" - scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" - lister: - cls: local - args: - db: "postgresql://%{hiera('swh::deploy::worker::lister_pypi::db::user')}:%{hiera('swh::deploy::worker::lister_pypi::db::password')}@%{hiera('swh::deploy::worker::lister_pypi::db::host')}:%{hiera('swh::deploy::worker::lister_pypi::db::port')}/%{hiera('swh::deploy::worker::lister_pypi::db::dbname')}" - # credentials in private data - celery: - task_broker: "%{alias('swh::deploy::worker::task_broker')}" - task_modules: - - swh.lister.pypi.tasks - task_queues: - - swh.lister.pypi.tasks.PyPIListerTask - - swh_lister_pypi_refresh - -swh::deploy::worker::lister_npm::config_file: "%{hiera('swh::conf_directory')}/lister_npm.yml" -swh::deploy::worker::lister_npm::concurrency: 1 -swh::deploy::worker::lister_npm::loglevel: warning -# Contains a password: in private data -swh::deploy::worker::lister_npm::db::host: belvedere.internal.softwareheritage.org -swh::deploy::worker::lister_npm::db::port: "%{alias('swh::deploy::db::secondary::port')}" -swh::deploy::worker::lister_npm::db::dbname: lister-npm -swh::deploy::worker::lister_npm::db::user: lister-npm -#swh::deploy::worker::lister_npm::db::password in private data -swh::deploy::worker::lister_npm::config: - storage: "%{alias('swh::remote_service::storage::config::writable')}" - scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" - lister: - cls: local - args: - db: "postgresql://%{hiera('swh::deploy::worker::lister_npm::db::user')}:%{hiera('swh::deploy::worker::lister_npm::db::password')}@%{hiera('swh::deploy::worker::lister_npm::db::host')}:%{hiera('swh::deploy::worker::lister_npm::db::port')}/%{hiera('swh::deploy::worker::lister_npm::db::dbname')}" - # credentials in private data - celery: - task_broker: "%{alias('swh::deploy::worker::task_broker')}" - task_modules: - - swh.lister.npm.tasks - task_queues: - - swh.lister.npm.tasks.NpmListerTask swh::deploy::worker::archiver::config_file: "%{hiera('swh::conf_directory')}/archiver.yml" swh::deploy::worker::archiver::archival_max_age: 3600 diff --git a/data/location/sesi_rocquencourt.yaml b/data/location/sesi_rocquencourt.yaml --- a/data/location/sesi_rocquencourt.yaml +++ b/data/location/sesi_rocquencourt.yaml @@ -21,11 +21,7 @@ smtp::relayhost: '[smtp.inria.fr]' swh::deploy::worker::instances: - - lister_debian - - lister_github - - lister_gitlab - - lister_npm - - lister_pypi + - lister - loader_debian - loader_deposit - loader_git diff --git a/site-modules/profile/manifests/swh/deploy/base_lister.pp b/site-modules/profile/manifests/swh/deploy/base_lister.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/base_lister.pp +++ /dev/null @@ -1,20 +0,0 @@ -# deployment for the lister package -class profile::swh::deploy::base_lister { - $config_dir = '/etc/softwareheritage/lister' - - # Contains passwords - file {$config_dir: - ensure => 'directory', - owner => 'swhworker', - group => 'swhdev', - mode => '0644', - purge => true, - recurse => true, - } - - $packages = ['python3-swh.lister'] - - package {$packages: - ensure => present, - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister.pp b/site-modules/profile/manifests/swh/deploy/worker/lister.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/lister.pp @@ -0,0 +1,15 @@ +# Deployment for swh-lister-github +class profile::swh::deploy::worker::lister { + $packages = ['python3-swh.lister'] + + package {$packages: + ensure => present, + } + + ::profile::swh::deploy::worker::instance {'lister': + ensure => present, + require => [ + Package['python3-swh.lister'], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/lister_debian.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/lister_debian.pp +++ /dev/null @@ -1,11 +0,0 @@ -# Deployment for swh-lister-debian -class profile::swh::deploy::worker::lister_debian { - include ::profile::swh::deploy::base_lister - - ::profile::swh::deploy::worker::instance {'lister_debian': - ensure => present, - require => [ - Package['python3-swh.lister'], - ], - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister_github.pp b/site-modules/profile/manifests/swh/deploy/worker/lister_github.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/lister_github.pp +++ /dev/null @@ -1,11 +0,0 @@ -# Deployment for swh-lister-github -class profile::swh::deploy::worker::lister_github { - include ::profile::swh::deploy::base_lister - - ::profile::swh::deploy::worker::instance {'lister_github': - ensure => present, - require => [ - Package['python3-swh.lister'], - ], - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister_gitlab.pp b/site-modules/profile/manifests/swh/deploy/worker/lister_gitlab.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/lister_gitlab.pp +++ /dev/null @@ -1,11 +0,0 @@ -# Deployment for swh-lister-gitlab -class profile::swh::deploy::worker::lister_gitlab { - include ::profile::swh::deploy::base_lister - - ::profile::swh::deploy::worker::instance {'lister_gitlab': - ensure => present, - require => [ - Package['python3-swh.lister'], - ], - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister_npm.pp b/site-modules/profile/manifests/swh/deploy/worker/lister_npm.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/lister_npm.pp +++ /dev/null @@ -1,11 +0,0 @@ -# Deployment for swh-lister-npm -class profile::swh::deploy::worker::lister_npm { - include ::profile::swh::deploy::base_lister - - ::profile::swh::deploy::worker::instance {'lister_npm': - ensure => present, - require => [ - Package['python3-swh.lister'], - ], - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/lister_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/lister_pypi.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/lister_pypi.pp +++ /dev/null @@ -1,11 +0,0 @@ -# Deployment for swh-lister-pypi -class profile::swh::deploy::worker::lister_pypi { - include ::profile::swh::deploy::base_lister - - ::profile::swh::deploy::worker::instance {'lister_pypi': - ensure => present, - require => [ - Package['python3-swh.lister'], - ], - } -}