diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -1017,7 +1017,7 @@ swh::remote_service::deposit::port: 5006 swh::remote_service::indexer::port: 5007 swh::remote_service::scheduler::port: 5008 - +swh::remote_service::search::port: 5010 # Default backend services. Override in specific sites if needed. Configurations # are split between read-only (the default) and writable storages. In most cases @@ -1192,6 +1192,14 @@ args: url: "http://localhost:%{hiera('swh::remote_service::storage::port')}/" +swh::remote_service::search::config::empty: {} + +swh::remote_service::search::config::storage0: + cls: remote + args: + url: "http://storage0.euwest.azure.internal.softwareheritage.org:%{hiera('swh::remote_service::search::port')}/" + +swh::remote_service::search::config: "%{alias('swh::remote_service::search::config::empty')}" # Indexer backend configurations swh::remote_service::indexer::config::uffizi: @@ -1645,7 +1653,7 @@ # deposit_basic_auth_swhworker_username # deposit_basic_auth_swhworker_password swh::deploy::webapp::config: - search: {} + search: "%{alias('swh::remote_service::search::config')}" storage: "%{alias('swh::remote_service::storage::config')}" vault: "%{alias('swh::remote_service::vault::config::writable')}" indexer_storage: "%{alias('swh::remote_service::indexer::config')}" @@ -1716,6 +1724,14 @@ - "::1" swh::deploy::scheduler::remote::config: "%{alias('swh::deploy::scheduler::config::local')}" +swh::elasticsearch::nodes: + - host: esnode2.internal.softwareheritage.org + port: 9200 + - host: esnode3.internal.softwareheritage.org + port: 9200 + - host: esnode1.internal.softwareheritage.org + port: 9200 + swh::deploy::scheduler::archive::conf_dir: "%{hiera('swh::conf_directory')}/backend" swh::deploy::scheduler::archive::conf_file: "%{hiera('swh::deploy::scheduler::archive::conf_dir')}/elastic.yml" swh::deploy::scheduler::archive::user: "%{hiera('swh::deploy::scheduler::user')}" @@ -1726,12 +1742,7 @@ args: index_name_prefix: swh-tasks storage_nodes: - - host: esnode2.internal.softwareheritage.org - port: 9200 - - host: esnode3.internal.softwareheritage.org - port: 9200 - - host: esnode1.internal.softwareheritage.org - port: 9200 + "%{alias('swh::elasticsearch::nodes')}" client_options: sniff_on_start: false sniff_on_connection_fail: true @@ -4194,3 +4205,30 @@ borg::encryption: repokey-blake2 +swh::deploy::search::sentry_swh_package: swh.search +swh::deploy::search::sentry_environment: "%{alias('swh::deploy::environment')}" +swh::deploy::search::conf_directory: "%{hiera('swh::conf_directory')}" +swh::deploy::search::conf_file: "%{hiera('swh::deploy::search::conf_directory')}/search.yml" +swh::deploy::search::user: swhstorage +swh::deploy::search::group: swhstorage +swh::deploy::search::config: + search: + cls: elasticsearch + args: + hosts: "%{alias('swh::elasticsearch::nodes')}" + +swh::deploy::search::backend::listen::host: 127.0.0.1 +swh::deploy::search::backend::listen::port: "%{alias('swh::remote_service::search::port')}" +swh::deploy::search::backend::workers: 4 +swh::deploy::search::backend::reload_mercy: 3600 +swh::deploy::search::backend::http_keepalive: 5 +swh::deploy::search::backend::http_timeout: 3600 +swh::deploy::search::backend::max_requests: 10000 +swh::deploy::search::backend::max_requests_jitter: 1000 +swh::deploy::search::backend::server_names: + - "%{::swh_hostname.internal_fqdn}" + - "%{::hostname}" + - 127.0.0.1 + - localhost + - "::1" + diff --git a/data/hostname/storage0.euwest.azure.internal.softwareheritage.org.yaml b/data/hostname/storage0.euwest.azure.internal.softwareheritage.org.yaml --- a/data/hostname/storage0.euwest.azure.internal.softwareheritage.org.yaml +++ b/data/hostname/storage0.euwest.azure.internal.softwareheritage.org.yaml @@ -33,3 +33,5 @@ objstorage: "%{alias('swh::remote_service::objstorage::config::azure_readonly_with_fallback')}" swh::deploy::worker::instances: [] + +swh::deploy::search::backend::listen::host: 0.0.0.0 diff --git a/data/hostname/webapp0.softwareheritage.org.yaml b/data/hostname/webapp0.softwareheritage.org.yaml --- a/data/hostname/webapp0.softwareheritage.org.yaml +++ b/data/hostname/webapp0.softwareheritage.org.yaml @@ -6,3 +6,5 @@ # Disable default vhost on port 80 apache::default_vhost: false + +swh::remote_service::search::config: "%{alias('swh::remote_service::search::config::storage0')}" diff --git a/manifests/site.pp b/manifests/site.pp --- a/manifests/site.pp +++ b/manifests/site.pp @@ -68,6 +68,7 @@ node /^storage\d+\.[^.]+\.azure\.internal\.softwareheritage\.org$/ { include role::swh_base_storage + include ::profile::swh::deploy::search } node /^getty.(internal.)?softwareheritage.org$/ { diff --git a/site-modules/profile/manifests/swh/deploy/search.pp b/site-modules/profile/manifests/swh/deploy/search.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/search.pp @@ -0,0 +1,12 @@ +# Deployment of the swh.search.api server +class profile::swh::deploy::search { + $packages = ['python3-swh.search'] + + package {$packages: + ensure => 'present', + } ~> Service['gunicorn-swh-search'] + + ::profile::swh::deploy::rpc_server {'search': + executable => 'swh.search.api.server:make_app_from_configfile()', + } +}