diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -2085,6 +2085,7 @@ realm_name: "%{alias('swh::config::keycloak::realm_name')}" swh::deploy::webapp::metadata_search_backend: swh-indexer-storage +swh::deploy::webapp::history_counters_url: "https://stats.export.softwareheritage.org/history_counters.json" # in private data: # deposit_basic_auth_swhworker_username @@ -2109,7 +2110,8 @@ client_config: sentry_dsn: "%{lookup('swh::deploy::webapp::sentry_dsn')}" keycloak: "%{alias('swh::deploy::webapp::config::keycloak')}" - + history_counters_url: "%{alias('swh::deploy::webapp::history_counters_url')}" + swh::deploy::webapp::locked_endpoints: - /api/1/content/[^/]+/symbol/ - /api/1/entity/ @@ -3236,6 +3238,13 @@ swh::deploy::counters::sentry_dsn: "https://%{lookup('swh::deploy::counters::sentry_token')}@sentry.softwareheritage.org/19" swh::deploy::counters::user: "%{alias('swh::deploy::base_counters::user')}" swh::deploy::counters::group: "%{alias('swh::deploy::base_counters::group')}" +swh::deploy::counters::cache_directory: /srv/softwareheritage/counters +swh::deploy::counters::cache_static_file: static_history.json +# will be always true when the deployment will be done in production +swh::deploy::counters::refresh_cache::activate: false +swh::deploy::counters::refresh_cache::cron: + minute: 0 + hour: "*/4" swh::deploy::counters::backend::listen::host: 0.0.0.0 swh::deploy::counters::backend::listen::port: "%{alias('swh::remote_service::counters::port')}" swh::deploy::counters::backend::workers: 2 diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -82,6 +82,8 @@ swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}" +swh::remote_service::counters::url: "http://counters0.internal.staging.swh.network:%{hiera('swh::remote_service::counters::port')}/" + swh::deploy::deposit::url: https://deposit.staging.swh.network swh::deploy::deposit::internal_url: "https://deposit-rp.internal.staging.swh.network" @@ -318,6 +320,8 @@ swh::deploy::webapp::config::es_workers_index_url: http://search-esnode0.internal.staging.swh.network:9200/swh_workers-* swh::deploy::webapp::metadata_search_backend: swh-search +swh::deploy::webapp::history_counters_url: "%{alias('swh::remote_service::counters::url')}" + swh::deploy::search::journal_client::service_types: - objects - indexed @@ -327,3 +331,20 @@ swh::deploy::vault::e2e::webapp: "https://webapp.staging.swh.network" swh::config::keycloak::realm_name: SoftwareHeritageStaging + +swh::deploy::counters::cache_static_file: +swh::deploy::counters::refresh_cache::activate: true + +swh::deploy::counters::config: + counters: + cls: redis + host: localhost:6379 + history: + cls: prometheus + prometheus_host: pergamon.internal.softwareheritage.org + prometheus_port: "%{alias('prometheus::server::listen_port')}" + live_data_start: 1609462861 # 2021-01-01 + cache_base_directory: "%{alias('swh::deploy::counters::cache_directory')}" + interval: 24h + labels: + environment: "%{alias('swh::deploy::environment')}" diff --git a/site-modules/profile/files/swh/deploy/counters/refresh_counters_cache.sh b/site-modules/profile/files/swh/deploy/counters/refresh_counters_cache.sh new file mode 100644 --- /dev/null +++ b/site-modules/profile/files/swh/deploy/counters/refresh_counters_cache.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +## +# File managed by puppet (class profile::swh::deploy::counters), changes will be lost. + +set -e + +cache_file=$1 +static_file=$2 + +static_file_stanza="" +if [ -n "${static_file}" ]; then + static_file_stanza=", \"static_file\": \"${static_file}\"" +fi + +tmp_file=$(mktemp) + +trap "rm -f ${tmp_file}" EXIT + +cat >"${tmp_file}" < '127.0.0.1', save_db_to_disk_interval => { '30' => '1' }, } + file { $cache_directory: + ensure => directory, + owner => 'swhstorage', + group => 'swhstorage', + mode => '0775', + } + ::profile::swh::deploy::rpc_server {'counters': executable => 'swh.counters.api.server:make_app_from_configfile()', } @@ -20,4 +31,21 @@ metrics_path => '/metrics', } + file { '/usr/local/bin/refresh_counters_cache.sh': + ensure => present, + owner => 'swhstorage', + group => 'swhstorage', + mode => '0755', + source => 'puppet:///modules/profile/swh/deploy/counters/refresh_counters_cache.sh', + } + + + if $cron_activate { + ::profile::cron::d { 'refresh_counters_cache': + command => "chronic /usr/local/bin/refresh_counters_cache.sh history.json ${static_history_file}", + user => 'swhstorage', + * => $cron_expression, + } + } + }