diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -1469,133 +1469,122 @@ # Objstorage backend configurations swh::remote_service::objstorage::config::azure: cls: azure-prefixed - args: - accounts: - "0": - account_name: 0euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::0euwestswh')}" - container_name: contents - "1": - account_name: 1euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::1euwestswh')}" - container_name: contents - "2": - account_name: 2euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::2euwestswh')}" - container_name: contents - "3": - account_name: 3euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::3euwestswh')}" - container_name: contents - "4": - account_name: 4euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::4euwestswh')}" - container_name: contents - "5": - account_name: 5euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::5euwestswh')}" - container_name: contents - "6": - account_name: 6euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::6euwestswh')}" - container_name: contents - "7": - account_name: 7euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::7euwestswh')}" - container_name: contents - "8": - account_name: 8euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::8euwestswh')}" - container_name: contents - "9": - account_name: 9euwestswh - api_secret_key: "%{hiera('swh::azure::credentials::9euwestswh')}" - container_name: contents - "a": - account_name: aeuwestswh - api_secret_key: "%{hiera('swh::azure::credentials::aeuwestswh')}" - container_name: contents - "b": - account_name: beuwestswh - api_secret_key: "%{hiera('swh::azure::credentials::beuwestswh')}" - container_name: contents - "c": - account_name: ceuwestswh - api_secret_key: "%{hiera('swh::azure::credentials::ceuwestswh')}" - container_name: contents - "d": - account_name: deuwestswh - api_secret_key: "%{hiera('swh::azure::credentials::deuwestswh')}" - container_name: contents - "e": - account_name: eeuwestswh - api_secret_key: "%{hiera('swh::azure::credentials::eeuwestswh')}" - container_name: contents - "f": - account_name: feuwestswh - api_secret_key: "%{hiera('swh::azure::credentials::feuwestswh')}" - container_name: contents + accounts: + "0": + account_name: 0euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::0euwestswh')}" + container_name: contents + "1": + account_name: 1euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::1euwestswh')}" + container_name: contents + "2": + account_name: 2euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::2euwestswh')}" + container_name: contents + "3": + account_name: 3euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::3euwestswh')}" + container_name: contents + "4": + account_name: 4euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::4euwestswh')}" + container_name: contents + "5": + account_name: 5euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::5euwestswh')}" + container_name: contents + "6": + account_name: 6euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::6euwestswh')}" + container_name: contents + "7": + account_name: 7euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::7euwestswh')}" + container_name: contents + "8": + account_name: 8euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::8euwestswh')}" + container_name: contents + "9": + account_name: 9euwestswh + api_secret_key: "%{hiera('swh::azure::credentials::9euwestswh')}" + container_name: contents + "a": + account_name: aeuwestswh + api_secret_key: "%{hiera('swh::azure::credentials::aeuwestswh')}" + container_name: contents + "b": + account_name: beuwestswh + api_secret_key: "%{hiera('swh::azure::credentials::beuwestswh')}" + container_name: contents + "c": + account_name: ceuwestswh + api_secret_key: "%{hiera('swh::azure::credentials::ceuwestswh')}" + container_name: contents + "d": + account_name: deuwestswh + api_secret_key: "%{hiera('swh::azure::credentials::deuwestswh')}" + container_name: contents + "e": + account_name: eeuwestswh + api_secret_key: "%{hiera('swh::azure::credentials::eeuwestswh')}" + container_name: contents + "f": + account_name: feuwestswh + api_secret_key: "%{hiera('swh::azure::credentials::feuwestswh')}" + container_name: contents swh::remote_service::objstorage::config::azure::readonly: cls: filtered - args: - storage_conf: "%{alias('swh::remote_service::objstorage::config::azure')}" - filters_conf: - - type: readonly + storage_conf: "%{alias('swh::remote_service::objstorage::config::azure')}" + filters_conf: + - type: readonly swh::remote_service::objstorage::config::saam: cls: remote - args: - url: "http://saam.internal.softwareheritage.org:%{hiera('swh::remote_service::objstorage::port')}/" + url: "http://saam.internal.softwareheritage.org:%{hiera('swh::remote_service::objstorage::port')}/" swh::remote_service::objstorage::config::saam::readonly: cls: filtered - args: - storage_conf: "%{alias('swh::remote_service::objstorage::config::saam')}" - filters_conf: - - type: readonly + storage_conf: "%{alias('swh::remote_service::objstorage::config::saam')}" + filters_conf: + - type: readonly swh::remote_service::objstorage::config::banco: cls: remote - args: - url: "http://banco.internal.softwareheritage.org:%{hiera('swh::remote_service::objstorage::port')}/" + url: "http://banco.internal.softwareheritage.org:%{hiera('swh::remote_service::objstorage::port')}/" swh::remote_service::objstorage::config::banco::readonly: cls: filtered - args: - storage_conf: "%{alias('swh::remote_service::objstorage::config::banco')}" - filters_conf: - - type: readonly + storage_conf: "%{alias('swh::remote_service::objstorage::config::banco')}" + filters_conf: + - type: readonly swh::remote_service::objstorage::config::azure_readonly_with_fallback: &swh_azure_readonly_with_fallback cls: multiplexer - args: - objstorages: - - "%{alias('swh::remote_service::objstorage::config::azure::readonly')}" - - "%{alias('swh::remote_service::objstorage::config::banco::readonly')}" - - "%{alias('swh::remote_service::objstorage::config::saam::readonly')}" + objstorages: + - "%{alias('swh::remote_service::objstorage::config::azure::readonly')}" + - "%{alias('swh::remote_service::objstorage::config::banco::readonly')}" + - "%{alias('swh::remote_service::objstorage::config::saam::readonly')}" swh::remote_service::objstorage::config::localhost: cls: remote - args: - url: "http://127.0.0.1:%{hiera('swh::remote_service::objstorage::port')}/" + url: "http://127.0.0.1:%{hiera('swh::remote_service::objstorage::port')}/" # Storage backend configurations swh::remote_service::storage::config::saam: cls: remote - args: - url: "http://saam.internal.softwareheritage.org:%{hiera('swh::remote_service::storage::port')}/" + url: "http://saam.internal.softwareheritage.org:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config::azure: cls: remote - args: - url: "http://storage01.euwest.azure.internal.softwareheritage.org:%{hiera('swh::remote_service::storage::port')}/" + url: "http://storage01.euwest.azure.internal.softwareheritage.org:%{hiera('swh::remote_service::storage::port')}/" swh::remote_service::storage::config::local_internal_network: cls: remote - args: - url: "http://%{lookup('swh::deploy::storage::backend::listen::host')}:%{lookup('swh::remote_service::storage::port')}/" + url: "http://%{lookup('swh::deploy::storage::backend::listen::host')}:%{lookup('swh::remote_service::storage::port')}/" swh::remote_service::search::config::empty: {} @@ -1726,21 +1715,19 @@ # This can be overriden per storage node swh::deploy::storage::config::local: cls: local - args: - db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" - objstorage: "%{alias('swh::remote_service::objstorage::config')}" + db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" + objstorage: "%{alias('swh::remote_service::objstorage::config')}" swh::deploy::storage::config: storage: "%{alias('swh::deploy::storage::config::local')}" swh::deploy::journal::writer::config: cls: kafka - args: - brokers: "%{alias('swh::deploy::journal::brokers')}" - prefix: "%{alias('swh::deploy::journal::prefix')}" - client_id: "swh.storage.journal_writer.%{::swh_hostname.short}" - anonymize: true - producer_config: + brokers: "%{alias('swh::deploy::journal::brokers')}" + prefix: "%{alias('swh::deploy::journal::prefix')}" + client_id: "swh.storage.journal_writer.%{::swh_hostname.short}" + anonymize: true + producer_config: message.max.bytes: 1000000000 swh::deploy::journal::backfill::config_file: "%{hiera('swh::deploy::journal::conf_directory')}/backfill.yml" @@ -1948,9 +1935,8 @@ swh::deploy::objstorage::config: objstorage: cls: pathslicing - args: - root: "%{hiera('swh::deploy::objstorage::directory')}" - slicing: "%{hiera('swh::deploy::objstorage::slicing')}" + root: "%{hiera('swh::deploy::objstorage::directory')}" + slicing: "%{hiera('swh::deploy::objstorage::slicing')}" client_max_size: 1073741824 # 1 GiB swh::deploy::objstorage::backend::listen::host: 127.0.0.1 @@ -2121,9 +2107,8 @@ swh::deploy::objstorage_log_checker:config: storage: cls: pathslicing - args: - root: "%{hiera('swh::deploy::objstorage::directory')}" - slicing: "%{hiera('swh::deploy::objstorage::slicing')}" + root: "%{hiera('swh::deploy::objstorage::directory')}" + slicing: "%{hiera('swh::deploy::objstorage::slicing')}" batch_size: 1000 log_tag: objstorage.checker.log @@ -2135,9 +2120,8 @@ swh::deploy::objstorage_repair_checker::config: storage: cls: pathslicing - args: - root: "%{hiera('swh::deploy::objstorage::directory')}" - slicing: "%{hiera('swh::deploy::objstorage::slicing')}" + root: "%{hiera('swh::deploy::objstorage::directory')}" + slicing: "%{hiera('swh::deploy::objstorage::slicing')}" batch_size: 1000 log_tag: objstorage.checker.repair backup_storages: "%{alias('swh::remote_service::objstorage::config_as_dict')}" @@ -2627,8 +2611,7 @@ swh::deploy::lister::db::local: cls: local - args: - db: "postgresql://%{hiera('swh::deploy::worker::lister::db::user')}:%{hiera('swh::deploy::lister::db::password')}@%{hiera('swh::deploy::worker::lister::db::host')}:%{hiera('swh::deploy::worker::lister::db::port')}/%{hiera('swh::deploy::worker::lister::db::name')}" + db: "postgresql://%{hiera('swh::deploy::worker::lister::db::user')}:%{hiera('swh::deploy::lister::db::password')}@%{hiera('swh::deploy::worker::lister::db::host')}:%{hiera('swh::deploy::worker::lister::db::port')}/%{hiera('swh::deploy::worker::lister::db::name')}" swh::deploy::lister::queues: - swh.lister.bitbucket.tasks.IncrementalBitBucketLister diff --git a/data/deployments/production/azure_euwest.yaml b/data/deployments/production/azure_euwest.yaml --- a/data/deployments/production/azure_euwest.yaml +++ b/data/deployments/production/azure_euwest.yaml @@ -9,11 +9,10 @@ swh::remote_service::objstorage::config::azure_readonly_with_fallback: &swh_azure_readonly_with_fallback cls: multiplexer - args: - objstorages: - - "%{alias('swh::remote_service::objstorage::config::azure::readonly')}" - - "%{alias('swh::remote_service::objstorage::config::banco::readonly')}" - - "%{alias('swh::remote_service::objstorage::config::saam::readonly')}" + objstorages: + - "%{alias('swh::remote_service::objstorage::config::azure::readonly')}" + - "%{alias('swh::remote_service::objstorage::config::banco::readonly')}" + - "%{alias('swh::remote_service::objstorage::config::saam::readonly')}" # Activate graph options to production cookers swh::deploy::worker::vault_cooker::config: diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -94,16 +94,14 @@ swh::remote_service::storage::config::storage0: cls: remote - args: - url: "%{alias('swh::remote_service::storage0::url')}" + url: "%{alias('swh::remote_service::storage0::url')}" swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::storage::config::writable: &swh_remote_service_storage_config_writable "%{alias('swh::remote_service::storage::config::storage0')}" swh::remote_service::vault::config::vault0: cls: remote - args: - url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" + url: "http://vault.internal.staging.swh.network:%{hiera('swh::remote_service::vault::port')}/" swh::remote_service::vault::config: "%{alias('swh::remote_service::vault::config::vault0')}" swh::remote_service::vault::config::writable: "%{alias('swh::remote_service::vault::config::vault0')}" @@ -115,8 +113,7 @@ swh::remote_service::scheduler::config::scheduler0: cls: remote - args: - url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" + url: "http://scheduler0.internal.staging.swh.network:%{hiera('swh::remote_service::scheduler::port')}/" swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" diff --git a/data/hostname/moma.softwareheritage.org.yaml b/data/hostname/moma.softwareheritage.org.yaml --- a/data/hostname/moma.softwareheritage.org.yaml +++ b/data/hostname/moma.softwareheritage.org.yaml @@ -41,10 +41,9 @@ swh::deploy::objstorage::config: objstorage: cls: multiplexer - args: - objstorages: - - "%{alias('swh::remote_service::objstorage::config::banco::readonly')}" - - "%{alias('swh::remote_service::objstorage::config::saam::readonly')}" + objstorages: + - "%{alias('swh::remote_service::objstorage::config::banco::readonly')}" + - "%{alias('swh::remote_service::objstorage::config::saam::readonly')}" swh::deploy::webapp::timers_enabled: true diff --git a/data/hostname/saam.internal.softwareheritage.org.yaml b/data/hostname/saam.internal.softwareheritage.org.yaml --- a/data/hostname/saam.internal.softwareheritage.org.yaml +++ b/data/hostname/saam.internal.softwareheritage.org.yaml @@ -16,29 +16,24 @@ swh::deploy::storage::config::local: cls: local - args: - db: "host=%{hiera('swh::deploy::storage::db::host')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" - objstorage: - cls: multiplexer - args: - objstorages: - - cls: pathslicing - args: - root: "%{hiera('swh::deploy::storage::directory')}" - slicing: "0:2/0:5" - compression: none - - cls: filtered - args: - storage_conf: - cls: pathslicing - args: - root: "%{hiera('swh::deploy::storage::legacy_directory')}" - slicing: "0:1/0:2/2:4/4:6" - compression: gzip - filters_conf: - - type: readonly - - "%{alias('swh::remote_service::objstorage::config::azure')}" - journal_writer: "%{alias('swh::deploy::journal::writer::config')}" + db: "host=%{hiera('swh::deploy::storage::db::host')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" + journal_writer: "%{alias('swh::deploy::journal::writer::config')}" + objstorage: + cls: multiplexer + objstorages: + - cls: pathslicing + root: "%{hiera('swh::deploy::storage::directory')}" + slicing: "0:2/0:5" + compression: none + - cls: filtered + storage_conf: + cls: pathslicing + root: "%{hiera('swh::deploy::storage::legacy_directory')}" + slicing: "0:1/0:2/2:4/4:6" + compression: gzip + filters_conf: + - type: readonly + - "%{alias('swh::remote_service::objstorage::config::azure')}" # Deploy the indexer storage server as a public resource swh::deploy::indexer::storage::backend::listen::host: 0.0.0.0 @@ -50,23 +45,19 @@ swh::deploy::objstorage::config: objstorage: cls: multiplexer - args: - objstorages: - - cls: pathslicing - args: - root: "%{hiera('swh::deploy::storage::directory')}" - slicing: "0:2/0:5" - compression: none - - cls: filtered - args: - storage_conf: - cls: pathslicing - args: - root: "%{hiera('swh::deploy::storage::legacy_directory')}" - slicing: "0:1/0:2/2:4/4:6" - compression: gzip - filters_conf: - - type: readonly + objstorages: + - cls: pathslicing + root: "%{hiera('swh::deploy::storage::directory')}" + slicing: "0:2/0:5" + compression: none + - cls: filtered + storage_conf: + cls: pathslicing + root: "%{hiera('swh::deploy::storage::legacy_directory')}" + slicing: "0:1/0:2/2:4/4:6" + compression: gzip + filters_conf: + - type: readonly client_max_size: 1073741824 # 1 GiB icinga2::host::vars: diff --git a/data/hostname/storage01.euwest.azure.internal.softwareheritage.org.yaml b/data/hostname/storage01.euwest.azure.internal.softwareheritage.org.yaml --- a/data/hostname/storage01.euwest.azure.internal.softwareheritage.org.yaml +++ b/data/hostname/storage01.euwest.azure.internal.softwareheritage.org.yaml @@ -5,8 +5,7 @@ swh::deploy::storage::config::local: cls: local - args: - db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" - objstorage: "%{alias('swh::remote_service::objstorage::config::azure_readonly_with_fallback')}" + db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" + objstorage: "%{alias('swh::remote_service::objstorage::config::azure_readonly_with_fallback')}" swh::deploy::worker::instances: [] diff --git a/data/hostname/storage1.internal.staging.swh.network.yaml b/data/hostname/storage1.internal.staging.swh.network.yaml --- a/data/hostname/storage1.internal.staging.swh.network.yaml +++ b/data/hostname/storage1.internal.staging.swh.network.yaml @@ -54,14 +54,12 @@ swh::deploy::storage::config::local: cls: local - args: - db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" - objstorage: - cls: pathslicing - args: - root: "%{hiera('swh::deploy::storage::directory')}" - slicing: "%{hiera('swh::deploy::objstorage::slicing')}" - journal_writer: "%{alias('swh::deploy::journal::writer::config')}" + db: "host=%{hiera('swh::deploy::storage::db::host')} port=%{hiera('swh::deploy::storage::db::port')} user=%{hiera('swh::deploy::storage::db::user')} dbname=%{hiera('swh::deploy::storage::db::dbname')} password=%{hiera('swh::deploy::storage::db::password')}" + objstorage: + cls: pathslicing + root: "%{hiera('swh::deploy::storage::directory')}" + slicing: "%{hiera('swh::deploy::objstorage::slicing')}" + journal_writer: "%{alias('swh::deploy::journal::writer::config')}" syncoid::configuration: sources: diff --git a/data/hostname/vangogh.euwest.azure.internal.softwareheritage.org.yaml b/data/hostname/vangogh.euwest.azure.internal.softwareheritage.org.yaml --- a/data/hostname/vangogh.euwest.azure.internal.softwareheritage.org.yaml +++ b/data/hostname/vangogh.euwest.azure.internal.softwareheritage.org.yaml @@ -4,10 +4,9 @@ # vault's cache backend is an azure objstorage swh::deploy::vault::config::cache: cls: azure - args: - account_name: "%{hiera('swh::azure::credentials::vaultstorage::account')}" - api_secret_key: "%{hiera('swh::azure::credentials::vaultstorage::key')}" - container_name: contents + account_name: "%{hiera('swh::azure::credentials::vaultstorage::account')}" + api_secret_key: "%{hiera('swh::azure::credentials::vaultstorage::key')}" + container_name: contents swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::azure')}" diff --git a/site-modules/profile/manifests/swh/deploy/objstorage.pp b/site-modules/profile/manifests/swh/deploy/objstorage.pp --- a/site-modules/profile/manifests/swh/deploy/objstorage.pp +++ b/site-modules/profile/manifests/swh/deploy/objstorage.pp @@ -27,7 +27,7 @@ $objstorage_cfg = lookup('swh::deploy::objstorage::config', Hash)['objstorage'] if $objstorage_cfg['cls'] == 'pathslicing' { - $obj_directory = $objstorage_cfg['args']['root'] + $obj_directory = $objstorage_cfg['root'] file {$obj_directory: ensure => directory, owner => $user, diff --git a/site-modules/profile/manifests/swh/deploy/storage.pp b/site-modules/profile/manifests/swh/deploy/storage.pp --- a/site-modules/profile/manifests/swh/deploy/storage.pp +++ b/site-modules/profile/manifests/swh/deploy/storage.pp @@ -15,8 +15,8 @@ $storage_config = lookup('swh::deploy::storage::config')['storage'] if ($storage_config['cls'] == 'local' - and $storage_config['args']['journal_writer'] - and $storage_config['args']['journal_writer']['cls'] == 'kafka') { + and $storage_config['journal_writer'] + and $storage_config['journal_writer']['cls'] == 'kafka') { include ::profile::swh::deploy::journal }