cat /srv/storage/space/mirrors/code.google.com/sources/INDEX-hg | ./schedule_with_queue_length_check.py --queue-name mercurial --threshold 1000 --waiting-time 60 | tee -a scheduling-mercurial-googlecode

...snippets/ardumont $ ./kibana_fetch_logs.py > hg-loader-error-logs-from-kibana-20180209-20180212.txt
...snippets/ardumont $ cat hg-loader-error-logs-from-kibana-20180209-20180212.txt | ./group_by_exception.py --loader-type hg | jq .
{
  "gitorious": {
    "total": 0,
    "errors": {}
  },
  "googlecode": {
    "total": 381,
    "errors": {
      "FileNotFoundError(2, 'No such file or directory')": 359,
      "OSError(12, 'Cannot allocate memory')": 13,
      "IntegrityError('duplicate key value violates uniqu": 3,
      "PatoolError('error extracting /srv/storage/space/m": 2,
      "WorkerLostError('Worker exited prematurely: signal": 2,
      "CommandError(b'bundle', b'-t', b'none-v2', b'-a', ": 1,
      "PatoolError(\"error extracting /srv/storage/space/m": 1
    }
  },
  "unknown": {
    "total": 15,
    "errors": {
      "Loading failure, updating to `partial` status\nTrac": 15
    }
  }
}

select * from origin_visit inner join origin on origin_visit.origin = origin.id where origin.type = 'hg' and snapshot_id != 16;

ardumont@uffizi:~% wc -l /srv/storage/space/mirrors/code.google.com/sources/INDEX-hg-dumps-with-size-{inferior,superior}-than-2gib.txt
  126981 /srv/storage/space/mirrors/code.google.com/sources/INDEX-hg-dumps-with-size-inferior-than-2gib.txt
      67 /srv/storage/space/mirrors/code.google.com/sources/INDEX-hg-dumps-with-size-superior-than-2gib.txt
  127048 total

softwareheritage=> select count(*) from origin o inner join origin_visit ov on o.id=ov.origin and ov.visit = (select max(visit) from origin_visit where origin=o.id) and o.type='hg' and ov.status = 'full';
 count
--------
 126920
(1 row)

softwareheritage=> select count(*) from origin o inner join origin_visit ov on o.id=ov.origin and ov.visit = (select max(visit) from origin_visit where origin=o.id) and o.type='hg' and ov.status <> 'full';
 count
-------
    56
(1 row)

$ cat ~/.config/swh/kibana/query.yml
indexes:
  - swh_workers-2018.03.*

size: 100
from: 0
_source:
  - message
  - swh_logging_args_args
  - swh_logging_args_exc
  - swh_logging_args_kwargs

query:
  bool:
    must:
    - match:
        systemd_unit:
          query: 'swh-worker@swh_loader_mercurial.service'
          type: phrase
    - term:
        priority: '3'
    # must_not:
    # - match:
    #     message:
    #       query: '[.*] consumer: Cannot connect to amqp.*'
    #       type: phrase
    # - match:
    #     message:
    #       query: '[.*] pidbox command error.*'
    #       type: phrase

sort:
- '@timestamp': 'asc'
$ ./kibana_fetch_logs.py > hg-loader-error-logs-from-kibana-201803.txt
$ cat hg-loader-error-logs-from-kibana-201803.txt| ./group_by_exception.py --loader-type hg | jq . 
{
  "gitorious": {
    "total": 0,
    "errors": {}
  },
  "googlecode": {
    "total": 5,
    "errors": {
      "'Worker exited prematurely: signal 9 (SIGKILL).',)": 5
    }
  },
  "unknown": {
    "total": 51,
    "errors": {
      "sion, preexec_fn)\nOSError:  Cannot allocate memory": 32,
      "impleBlob' object does not support item assignment": 3,
      "v)\nValueError: could not convert string to float: ": 3,
      "nction swh_revision_add() line 5 at SQL statement\n": 1,
      ">nginx/1.10.3</center>\\r\\n</body>\\r\\n</html>\\r\\n')": 1,
      "node.remove_tree_node_for_path(rest)\nKeyError: b''": 1,
      "ion aborted.', BrokenPipeError(32, 'Broken pipe'))": 1,
      "ist O. D. - Google Chrome 2014-02-20 17.07.29.png'": 1,
      "r: b'\\x90\\x90t`\\xf6\\x7fkJ@Z\\x86M-\\xf9BV\\xd3\\xae$D'": 1,
      "ing-gfd-source-archive.zip: File is not a zip file": 1,
      "-slides-source-archive.zip: File is not a zip file": 1,
      "or missing revlog for data/sword/mods.d/kjv.conf')": 1,
      "ezeus-source-archive.zip:  No space left on device": 1,
      "ce-ac-source-archive.zip:  No space left on device": 1,
      "redit-source-archive.zip:  No space left on device": 1,
      "ce: '/tmp/swh.loader.mercurial.u7i3awxx.dump-8805'": 1
    }
  }
}