vsellier@search-esnode0 ~ % curl -XGET -H "Content-Type: application/json" http://192.168.130.80:9200/_cat/indices
green open  origin                      HthJj42xT5uO7w3Aoxzppw 80 0 929692 137147 4gb 4gb
green close origin-backup-20210209-1736 P1CKjXW0QiWM5zlzX46-fg 80 0                      
green close origin-v0.5.0               SGplSaqPR_O9cPYU4ZsmdQ 80 0                      
vsellier@search-esnode0 ~ % curl -XGET -H "Content-Type: application/json" http://192.168.130.80:9200/_cat/aliases
origin-read  origin - - - -
origin-write origin - - - -

Mar 04 16:22:40 search0 swh[3598137]: INFO:elasticsearch:POST http://search-esnode0.internal.staging.swh.network:9200/origin-write/_bulk [status:200 request:0.013s]
Mar 04 16:22:41 search0 swh[3598137]: INFO:elasticsearch:POST http://search-esnode0.internal.staging.swh.network:9200/origin-write/_bulk [status:200 request:0.012s]

Mar 04 15:40:20 search0 python3[3598040]: 2021-03-04 15:40:20 [3598040] swh.search.api.server:INFO Initializing indexes with configuration: 
Mar 04 15:40:20 search0 python3[3598040]: 2021-03-04 15:40:20 [3598040] elasticsearch:INFO HEAD http://search-esnode0.internal.staging.swh.network:9200/origin [status:200 request:0.005s]
Mar 04 15:40:20 search0 python3[3598040]: 2021-03-04 15:40:20 [3598040] elasticsearch:INFO HEAD http://search-esnode0.internal.staging.swh.network:9200/origin-read/_alias [status:200 request:0.001s]
Mar 04 15:40:20 search0 python3[3598040]: 2021-03-04 15:40:20 [3598040] elasticsearch:INFO HEAD http://search-esnode0.internal.staging.swh.network:9200/origin-write/_alias [status:200 request:0.001s]
Mar 04 15:40:20 search0 python3[3598040]: 2021-03-04 15:40:20 [3598040] elasticsearch:INFO PUT http://search-esnode0.internal.staging.swh.network:9200/origin/_mapping [status:200 request:0.006s]
Mar 04 16:19:27 search0 python3[3598042]: 2021-03-04 16:19:27 [3598042] elasticsearch:INFO GET http://search-esnode0.internal.staging.swh.network:9200/origin-read/_search?size=100 [status:200 request:0.076s]

  File "/usr/lib/python3/dist-packages/elasticsearch/connection/http_urllib3.py", line 249, in perform_request
    raise ConnectionTimeout("TIMEOUT", str(e), e)
elasticsearch.exceptions.ConnectionTimeout: ConnectionTimeout caused by - ReadTimeoutError(HTTPConnectionPool(host='search-esnode3.internal.softwareheritage.org', port=9200): Read timed out. (read timeout=10))

green  open   origin-production hZfuv0lVRImjOjO_rYgDzg  90   1  152795694       297907    217.6gb        109.2gb

root@search-esnode1:~# curl -XPOST -H "Content-Type: application/json" ${ES_SERVER}/_reindex\?pretty\&refresh=true\&requests_per_second=-1\&\&wait_for_completion=true -d @reindex-origin.json
{
  "took" : 115296461,
  "timed_out" : false,
  "total" : 152756759,
  "updated" : 0,
  "created" : 152756759,
  "deleted" : 0,
  "batches" : 152757,
  "version_conflicts" : 0,
  "noops" : 0,
  "retries" : {
    "bulk" : 0,
    "search" : 0
  },
  "throttled_millis" : 0,
  "requests_per_second" : -1.0,
  "throttled_until_millis" : 0,
  "failures" : [ ]
}

Every 10.0s: curl -s http://192.168.100.81:9200/_cat/nodes\?v; echo ; curl -s http://192.168.100.81:9200/_cat/indices\?v ; echo ; df -h | grep elastic                                                 search-esnode1: Wed Feb 24 16:14:58 2021

root@search-esnode1:~# curl -XPOST -H "Content-Type: application/json" http://${ES_SERVER}/_reindex\?pretty\&refresh=true\&requests_per_second=-1\&\&wait_for_completion=true -d @reindex-origin.json
{
  "took" : 91121031,
  "timed_out" : false,
  "total" : 152756759,
  "updated" : 0,
  "created" : 152756759,
  "deleted" : 0,
  "batches" : 152757,
  "version_conflicts" : 0,
  "noops" : 0,
  "retries" : {
    "bulk" : 0,
    "search" : 0
  },
  "throttled_millis" : 0,
  "requests_per_second" : -1.0,
  "throttled_until_millis" : 0,
  "failures" : [ ]
}

root@search0:~# systemctl stop swh-search-journal-client@objects.service 
root@search0:~# puppet agent --disable "stop search journal client to reset offsets"

vsellier@journal0 ~ % /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server $SERVER --reset-offsets --topic swh.journal.objects.origin_visit --to-earliest --group swh.search.journal_client --execute

% curl  -s http://${ES_SERVER}/_cat/indices\?v                                                       
health status index                       uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   origin                      HthJj42xT5uO7w3Aoxzppw  80   0     868634      8577610     10.5gb         10.5gb
green  close  origin-backup-20210209-1736 P1CKjXW0QiWM5zlzX46-fg  80   0                                                  
green  open   origin-v0.5.0               SGplSaqPR_O9cPYU4ZsmdQ  80   0     868121            0    987.7mb        987.7mb
green  open   origin-toremove             PL7WEs3FTJSQy4dgGIwpeQ  80   0     868610            0    987.5mb        987.5mb  <-- A clean copy of the origin index has almose the same size as yesterday

% curl -XPOST -H "Content-Type: application/json" http://${ES_SERVER}/origin/_forcemerge                           
{"_shards":{"total":80,"successful":80,"failed":0}}%

% curl  -s http://${ES_SERVER}/_cat/indices\?v      
health status index                       uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   origin                      HthJj42xT5uO7w3Aoxzppw  80   0     868684         3454        1gb            1gb
green  close  origin-backup-20210209-1736 P1CKjXW0QiWM5zlzX46-fg  80   0                                                  
green  open   origin-v0.5.0               SGplSaqPR_O9cPYU4ZsmdQ  80   0     868121            0    987.7mb        987.7mb
green  open   origin-toremove             PL7WEs3FTJSQy4dgGIwpeQ  80   0     868610            0    987.5mb        987.5mb

swh=> select * from origin where url='deb://Debian/packages/node-response-time';
  id   |                   url                    
-------+------------------------------------------
 15552 | deb://Debian/packages/node-response-time
(1 row)

root@search0:~# puppet agent --disable "swh-search upgrade"
root@search0:~# systemctl stop swh-search-journal-client@objects.service 
root@search0:~# systemctl stop swh-search-journal-client@indexed.service
root@search0:~# systemctl stop gunicorn-swh-search.service

root@search0:~# apt update && apt list --upgradable
...
python3-swh.search/unknown 0.6.0-1~swh1~bpo10+1 all [upgradable from: 0.5.0-1~swh1~bpo10+1]
...

swhscheduler@saatchi:~$ /usr/bin/swh scheduler --config-file /etc/softwareheritage/scheduler/backend.yml task schedule_
origins --storage-url http://saam.internal.softwareheritage.org:5002 --batch-size 20 index-origin-metadata | tee /tmp/schedule-origins.txt