diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -1814,6 +1814,11 @@ cls: local db: "host=%{hiera('swh::deploy::vault::db::host')} port=%{hiera('swh::deploy::vault::db::port')} user=%{hiera('swh::deploy::vault::db::user')} dbname=%{hiera('swh::deploy::vault::db::dbname')} password=%{hiera('swh::deploy::vault::db::password')}" +swh::deploy::graph::user: swhworker +swh::deploy::graph::group: swhworker +swh::deploy::graph::backend::listen::host: 127.0.0.1 +swh::deploy::graph::backend::listen::port: "%{alias('swh::remote_service::graph::port')}" + swh::deploy::journal::conf_directory: "%{hiera('swh::conf_directory')}/journal" swh::deploy::journal::brokers: diff --git a/site-modules/profile/manifests/swh/deploy/graph.pp b/site-modules/profile/manifests/swh/deploy/graph.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/graph.pp @@ -0,0 +1,86 @@ +# Deployment of graph (checks for now) + +# FIXME: Graph is currently managed manually and running through a venv. At some point, +# adapt here to also install fully the graph from that manifest +class profile::swh::deploy::graph { + + $packages = ['python3-venv'] + package {$packages: + ensure => 'present', + } + + $user = lookup('swh::deploy::graph::user') + $group = lookup('swh::deploy::graph::group') + + # install services from templates + $services = [ { # this matches the current status + 'name' => 'swhgraphshm', + 'status' => 'present', + 'enable' => false, + }, { + 'name' => 'swhgraphdev', + 'status' => 'running', + 'enable' => true, + } + ] + each($services) | $service | { + $unit_name = "${service['name']}.service" + + # template uses: + # $user + # $group + ::systemd::unit_file {$unit_name: + ensure => present, + content => template("profile/swh/deploy/graph/${unit_name}.erb"), + mode => '0644', + } ~> service {$service['name']: + ensure => $service['status'], + enable => $service['enable'], + } + } + + $backend_listen_host = lookup("swh::deploy::graph::backend::listen::host") + $backend_listen_port = lookup("swh::deploy::graph::backend::listen::port") + + $http_check_string = "graph API server" + $icinga_checks_file = lookup('icinga2::exported_checks::filename') + + # swhgraphdev.service exposes the main graph server. + # Ensure the port is working ok through icinga checks + @@::icinga2::object::service {"swh-graph api (local on ${::fqdn})": + service_name => "swh-graph api (localhost)", + import => ['generic-service'], + host_name => $::fqdn, + check_command => 'http', + command_endpoint => $::fqdn, + vars => { + http_address => $local_check_address, + http_vhost => $local_check_address, + http_port => $backend_listen_port, + http_uri => '/', + http_header => ['Accept: application/json'], + http_string => $http_check_string, + }, + target => $icinga_checks_file, + tag => 'icinga2::exported', + } + + if $backend_listen_host != '127.0.0.1' { + @@::icinga2::object::service {"swh-graph api (remote on ${::fqdn})": + service_name => "swh-graph api (remote)", + import => ['generic-service'], + host_name => $::fqdn, + check_command => 'http', + vars => { + http_vhost => $::swh_hostname['internal_fqdn'], + http_port => $backend_listen_port, + http_uri => '/', + http_header => ['Accept: application/json'], + http_string => $http_check_string, + }, + target => $icinga_checks_file, + tag => 'icinga2::exported', + } + } + +} diff --git a/site-modules/profile/templates/swh/deploy/graph/swhgraphdev.service.erb b/site-modules/profile/templates/swh/deploy/graph/swhgraphdev.service.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/graph/swhgraphdev.service.erb @@ -0,0 +1,16 @@ +# Managed by puppet class profile::swh::deploy::graph +# Changes will be overwritten + +[Unit] +Description=swh graph +After=swhgraphshm.service +Requires=swhgraphshm.service + +[Service] +Type=simple +User=<%= @user %> +Group=<%= @group %> +ExecStart=/opt/swhgraph_venv/bin/swh graph rpc-serve -g /dev/shm/swh-graph/default/graph + +[Install] +WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/graph/swhgraphshm.service.erb b/site-modules/profile/templates/swh/deploy/graph/swhgraphshm.service.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/graph/swhgraphshm.service.erb @@ -0,0 +1,20 @@ +# Managed by puppet class profile::swh::deploy::graph +# Changes will be overwritten + +[Unit] +Description=swh graph shm mapper + +[Service] +Type=oneshot +User=<%= @user %> +Group=<%= @group %> +RemainAfterExit=yes +# ExecStart=/opt/swhgraph_venv/bin/swh graph cachemount --graph /srv/softwareheritage/ssd/graph/2020-05-20/compressed/graph --cache /dev/shm/swh-graph/default +ExecStart=mkdir -p /dev/shm/swh-graph/default +ExecStart=sh -c "ln -s /srv/softwareheritage/ssd/graph/2020-12-15/compressed/* /dev/shm/swh-graph/default" +ExecStart=sh -c "cp --remove-destination /srv/softwareheritage/ssd/graph/2020-12-15/compressed/graph.graph /dev/shm/swh-graph/default" +ExecStart=sh -c "cp --remove-destination /srv/softwareheritage/ssd/graph/2020-12-15/compressed/graph-transposed.graph /dev/shm/swh-graph/default" +ExecStop=rm -rf /dev/shm/swh-graph/default + +[Install] +WantedBy=multi-user.target diff --git a/site-modules/role/manifests/swh_graph_backend.pp b/site-modules/role/manifests/swh_graph_backend.pp --- a/site-modules/role/manifests/swh_graph_backend.pp +++ b/site-modules/role/manifests/swh_graph_backend.pp @@ -2,4 +2,5 @@ class role::swh_graph_backend inherits role::swh_base { include profile::docker include profile::megacli + include profile::swh::deploy::graph }