Page MenuHomeSoftware Heritage
Files F9749651

D5470.diff
No OneTemporary
Actions

D5470.diff
View Options

diff --git a/data/common/common.yaml b/data/common/common.yaml
--- a/data/common/common.yaml
+++ b/data/common/common.yaml
@@ -2085,6 +2085,7 @@
realm_name: "%{alias('swh::config::keycloak::realm_name')}"
swh::deploy::webapp::metadata_search_backend: swh-indexer-storage
+swh::deploy::webapp::history_counters_url: "https://stats.export.softwareheritage.org/history_counters.json"
# in private data:
# deposit_basic_auth_swhworker_username
@@ -2109,7 +2110,8 @@
client_config:
sentry_dsn: "%{lookup('swh::deploy::webapp::sentry_dsn')}"
keycloak: "%{alias('swh::deploy::webapp::config::keycloak')}"
-
+ history_counters_url: "%{alias('swh::deploy::webapp::history_counters_url')}"
+
swh::deploy::webapp::locked_endpoints:
- /api/1/content/[^/]+/symbol/
- /api/1/entity/
@@ -3236,6 +3238,13 @@
swh::deploy::counters::sentry_dsn: "https://%{lookup('swh::deploy::counters::sentry_token')}@sentry.softwareheritage.org/19"
swh::deploy::counters::user: "%{alias('swh::deploy::base_counters::user')}"
swh::deploy::counters::group: "%{alias('swh::deploy::base_counters::group')}"
+swh::deploy::counters::cache_directory: /srv/softwareheritage/counters
+swh::deploy::counters::cache_static_file: static_history.json
+# will be always true when the deployment will be done in production
+swh::deploy::counters::refresh_cache::activate: false
+swh::deploy::counters::refresh_cache::cron:
+ minute: 0
+ hour: "*/4"
swh::deploy::counters::backend::listen::host: 0.0.0.0
swh::deploy::counters::backend::listen::port: "%{alias('swh::remote_service::counters::port')}"
swh::deploy::counters::backend::workers: 2
diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml
--- a/data/deployments/staging/common.yaml
+++ b/data/deployments/staging/common.yaml
@@ -82,6 +82,8 @@
swh::remote_service::scheduler::config: "%{alias('swh::remote_service::scheduler::config::scheduler0')}"
swh::remote_service::scheduler::config::writable: "%{alias('swh::remote_service::scheduler::config::scheduler0')}"
+swh::remote_service::counters::url: "http://counters0.internal.staging.swh.network:%{hiera('swh::remote_service::counters::port')}/"
+
swh::deploy::deposit::url: https://deposit.staging.swh.network
swh::deploy::deposit::internal_url: "https://deposit-rp.internal.staging.swh.network"
@@ -318,6 +320,8 @@
swh::deploy::webapp::config::es_workers_index_url: http://search-esnode0.internal.staging.swh.network:9200/swh_workers-*
swh::deploy::webapp::metadata_search_backend: swh-search
+swh::deploy::webapp::history_counters_url: "%{alias('swh::remote_service::counters::url')}"
+
swh::deploy::search::journal_client::service_types:
- objects
- indexed
@@ -327,3 +331,20 @@
swh::deploy::vault::e2e::webapp: "https://webapp.staging.swh.network"
swh::config::keycloak::realm_name: SoftwareHeritageStaging
+
+swh::deploy::counters::cache_static_file:
+swh::deploy::counters::refresh_cache::activate: true
+
+swh::deploy::counters::config:
+ counters:
+ cls: redis
+ host: localhost:6379
+ history:
+ cls: prometheus
+ prometheus_host: pergamon.internal.softwareheritage.org
+ prometheus_port: "%{alias('prometheus::server::listen_port')}"
+ live_data_start: 1609462861 # 2021-01-01
+ cache_base_directory: "%{alias('swh::deploy::counters::cache_directory')}"
+ interval: 24h
+ labels:
+ environment: "%{alias('swh::deploy::environment')}"
diff --git a/site-modules/profile/files/swh/deploy/counters/refresh_counters_cache.sh b/site-modules/profile/files/swh/deploy/counters/refresh_counters_cache.sh
new file mode 100644
--- /dev/null
+++ b/site-modules/profile/files/swh/deploy/counters/refresh_counters_cache.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+##
+# File managed by puppet (class profile::swh::deploy::counters), changes will be lost.
+
+set -e
+
+cache_file=$1
+static_file=$2
+
+static_file_stanza=""
+if [ -n "${static_file}" ]; then
+ static_file_stanza=", \"static_file\": \"${static_file}\""
+fi
+
+tmp_file=$(mktemp)
+
+trap "rm -f ${tmp_file}" EXIT
+
+cat >"${tmp_file}" <<EOF
+{
+ "cache_file": "${cache_file}",
+ "objects": ["content", "origin", "revision"]
+ ${static_file_stanza}
+}
+EOF
+
+curl -s -XPOST -H 'Content-Type: application/json' http://localhost:5011/refresh_history -d @"${tmp_file}"
diff --git a/site-modules/profile/manifests/swh/deploy/counters.pp b/site-modules/profile/manifests/swh/deploy/counters.pp
--- a/site-modules/profile/manifests/swh/deploy/counters.pp
+++ b/site-modules/profile/manifests/swh/deploy/counters.pp
@@ -3,12 +3,23 @@
include ::profile::swh::deploy::base_counters
$service_port = lookup('swh::remote_service::counters::port')
+ $cache_directory = lookup('swh::deploy::counters::cache_directory')
+ $cron_activate = lookup('swh::deploy::counters::refresh_cache::activate')
+ $cron_expression = lookup('swh::deploy::counters::refresh_cache::cron')
+ $static_history_file = lookup('swh::deploy::counters::cache_static_file')
class { '::redis':
bind => '127.0.0.1',
save_db_to_disk_interval => { '30' => '1' },
}
+ file { $cache_directory:
+ ensure => directory,
+ owner => 'swhstorage',
+ group => 'swhstorage',
+ mode => '0775',
+ }
+
::profile::swh::deploy::rpc_server {'counters':
executable => 'swh.counters.api.server:make_app_from_configfile()',
}
@@ -20,4 +31,21 @@
metrics_path => '/metrics',
}
+ file { '/usr/local/bin/refresh_counters_cache.sh':
+ ensure => present,
+ owner => 'swhstorage',
+ group => 'swhstorage',
+ mode => '0755',
+ source => 'puppet:///modules/profile/swh/deploy/counters/refresh_counters_cache.sh',
+ }
+
+
+ if $cron_activate {
+ ::profile::cron::d { 'refresh_counters_cache':
+ command => "chronic /usr/local/bin/refresh_counters_cache.sh history.json ${static_history_file}",
+ user => 'swhstorage',
+ * => $cron_expression,
+ }
+ }
+
}

File Metadata

Mime Type
text/plain
Expires
Sun, Aug 24, 6:04 PM (4 d, 1 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220664

Event Timeline

About · Developer information · Legal