diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py --- a/swh/web/misc/urls.py +++ b/swh/web/misc/urls.py @@ -14,6 +14,7 @@ from django.shortcuts import render from swh.web.common import archive +from swh.web.common.utils import parse_iso8601_date_to_utc from swh.web.config import get_config from swh.web.misc.metrics import prometheus_metrics @@ -27,16 +28,48 @@ return render(request, "misc/jslicenses.html", {"jslicenses_data": jslicenses_data}) +# historical data for backfilling missing counter values +# https://forge.softwareheritage.org/T2619 +_stat_counters_backfill = { + "2015-09-01": {"revision": 0, "origin": 0, "content": 0}, + "2016-07-01": {"revision": 594305600, "origin": 22777052}, + "2016-09-14": {"revision": 644628800, "origin": 25258776}, + "2016-11-24": {"revision": 704845952, "origin": 53488904}, + "2017-05-10": {"revision": 780882048, "origin": 58257484}, + "2017-09-26": {"revision": 853277241, "origin": 65546644}, + "2018-01-24": {"revision": 943061517, "origin": 71814787}, + "2018-02-13": {"revision": 946216028, "origin": 81655813}, + "2018-03-25": {"revision": 980390191, "origin": 83797945}, + "2018-10-04": {"revision": 1126348335, "origin": 85202432}, + "2019-01-27": {"revision": 1248389319, "origin": 88288721}, + "2019-04-08": {"revision": 1293870115, "origin": 88297714}, + "2019-06-27": {"revision": 1326776432, "origin": 89301694}, + "2019-07-24": {"revision": 1358421267, "origin": 89601149}, + "2019-09-22": {"revision": 1379380527, "origin": 90231104}, + "2019-09-29": {"revision": 1385477933, "origin": 90487661}, + "2020-01-01": {"revision": 1414420369, "origin": 91400586}, + "2020-02-06": {"revision": 1428955761, "origin": 91512130}, + "2020-04-07": {"revision": 1590436149, "origin": 107875943}, + "2020-05-17": {"revision": 1717420203, "origin": 121172621}, + "2020-05-27": {"revision": 1744034936, "origin": 123781438}, +} + + def _stat_counters(request): stat_counters = archive.stat_counters() url = get_config()["history_counters_url"] - stat_counters_history = "null" - if url: - try: - response = requests.get(url, timeout=5) - stat_counters_history = json.loads(response.text) - except Exception as exc: - sentry_sdk.capture_exception(exc) + stat_counters_history = {} + try: + response = requests.get(url, timeout=5) + stat_counters_history = json.loads(response.text) + for d, object_counts in _stat_counters_backfill.items(): + # convert date to javascript timestamp (in ms) + timestamp = int(parse_iso8601_date_to_utc(d).timestamp()) * 1000 + for object_type, object_count in object_counts.items(): + stat_counters_history[object_type].append([timestamp, object_count]) + except Exception as exc: + sentry_sdk.capture_exception(exc) + counters = { "stat_counters": stat_counters, "stat_counters_history": stat_counters_history,