diff --git a/Makefile.local b/Makefile.local --- a/Makefile.local +++ b/Makefile.local @@ -15,6 +15,7 @@ .PHONY: run-migrations run-migrations: python3 swh/web/manage.py migrate 2>/dev/null + python3 swh/web/manage.py createcachetable 2>/dev/null .PHONY: run-migrations-prod run-migrations-prod: diff --git a/debian/postinst b/debian/postinst --- a/debian/postinst +++ b/debian/postinst @@ -1,3 +1,5 @@ #!/bin/bash django-admin migrate --settings=swh.web.settings.production +django-admin createcachetable --settings=swh.web.settings.production + diff --git a/swh/web/assets/src/bundles/webapp/webapp.css b/swh/web/assets/src/bundles/webapp/webapp.css --- a/swh/web/assets/src/bundles/webapp/webapp.css +++ b/swh/web/assets/src/bundles/webapp/webapp.css @@ -431,7 +431,7 @@ } .swh-coverage { - height: 65px; + height: 78px; padding-top: 0.3rem; border: none; } diff --git a/swh/web/browse/views/origin.py b/swh/web/browse/views/origin.py --- a/swh/web/browse/views/origin.py +++ b/swh/web/browse/views/origin.py @@ -7,6 +7,7 @@ from distutils.util import strtobool +from django.views.decorators.cache import cache_page from django.http import HttpResponse from django.shortcuts import render, redirect @@ -20,6 +21,7 @@ get_origin_info, get_snapshot_context ) from swh.web.browse.browseurls import browse_route +from swh.web.misc.coverage import code_providers from .utils.snapshot_context import ( browse_snapshot_directory, browse_snapshot_content, @@ -211,6 +213,31 @@ return HttpResponse(results, content_type='application/json') +@browse_route(r'origin/coverage_count/', + view_name='browse-origin-coverage-count') +@cache_page(24 * 60 * 60, cache='db_cache') +def _origin_coverage_count(request): + """Internal browse endpoint to count the number of origins associated + to each code provider declared in the archive coverage list. + As this operation takes some times, we execute it once per day and + cache its results to database. The cached origin counts are then served. + """ + try: + results = [] + for code_provider in code_providers: + pattern = code_provider['origin_url_pattern'] + results.append({ + 'provider_id': code_provider['provider_id'], + 'origin_count': service.storage.origin_count(pattern), + 'origin_types': code_provider['origin_types'] + }) + results = json.dumps(results) + except Exception as exc: + return handle_view_exception(request, exc, html_response=False) + + return HttpResponse(results, content_type='application/json') + + @browse_route(r'origin/(?P[0-9]+)/latest_snapshot/', view_name='browse-origin-latest-snapshot') def _origin_latest_snapshot(request, origin_id): diff --git a/swh/web/misc/coverage.py b/swh/web/misc/coverage.py --- a/swh/web/misc/coverage.py +++ b/swh/web/misc/coverage.py @@ -8,69 +8,100 @@ # Current coverage list of the archive # TODO: Retrieve that list dynamically instead of hardcoding it -_code_providers = [ +code_providers = [ { + 'provider_id': 'debian', 'provider_url': 'https://www.debian.org/', 'provider_logo': 'img/logos/debian.png', 'provider_info': 'source packages from the Debian distribution ' '(continuously archived)', + 'origin_url_pattern': 'deb://', + 'origin_types': 'packages' }, { + 'provider_id': 'framagit', 'provider_url': 'https://framagit.org/', 'provider_logo': 'img/logos/framagit.png', 'provider_info': 'public repositories from Framagit ' '(continuously archived)', + 'origin_url_pattern': 'https://framagit.org/', + 'origin_types': 'repositories' }, { + 'provider_id': 'github', 'provider_url': 'https://github.com', 'provider_logo': 'img/logos/github.png', 'provider_info': 'public repositories from GitHub ' '(continuously archived)', + 'origin_url_pattern': 'https://github.com/', + 'origin_types': 'repositories' }, { + 'provider_id': 'gitlab', 'provider_url': 'https://gitlab.com', 'provider_logo': 'img/logos/gitlab.svg', 'provider_info': 'public repositories from GitLab ' '(continuously archived)', + 'origin_url_pattern': 'https://gitlab.com', + 'origin_types': 'repositories' }, { + 'provider_id': 'gitorious', 'provider_url': 'https://gitorious.org/', 'provider_logo': 'img/logos/gitorious.png', 'provider_info': 'public repositories from the former Gitorious code ' 'hosting service', + 'origin_url_pattern': 'https://gitorious.org/', + 'origin_types': 'repositories' }, { + 'provider_id': 'googlecode', 'provider_url': 'https://code.google.com/archive/', 'provider_logo': 'img/logos/googlecode.png', 'provider_info': 'public repositories from the former Google Code ' 'project hosting service', + 'origin_url_pattern': '.googlecode.com/', + 'origin_types': 'repositories' }, { + 'provider_id': 'gnu', 'provider_url': 'https://www.gnu.org', 'provider_logo': 'img/logos/gnu.png', 'provider_info': 'releases from the GNU project (as of August 2015)', + 'origin_url_pattern': 'rsync://ftp.gnu.org/', + 'origin_types': 'releases' }, { + 'provider_id': 'hal', 'provider_url': 'https://hal.archives-ouvertes.fr/', 'provider_logo': 'img/logos/hal.png', 'provider_info': 'scientific software source code deposited in the ' - 'open archive HAL' + 'open archive HAL', + 'origin_url_pattern': 'https://hal.archives-ouvertes.fr', + 'origin_types': 'deposits' + }, { + 'provider_id': 'inria', 'provider_url': 'https://gitlab.inria.fr', 'provider_logo': 'img/logos/inria.jpg', 'provider_info': 'public repositories from Inria GitLab ' '(continuously archived)', + 'origin_url_pattern': 'https://gitlab.inria.fr', + 'origin_types': 'repositories' }, { + 'provider_id': 'pypi', 'provider_url': 'https://pypi.org', 'provider_logo': 'img/logos/pypi.svg', 'provider_info': 'source packages from the Python Packaging Index ' '(continuously archived)', + 'origin_url_pattern': 'https://pypi.org', + 'origin_types': 'packages' }, ] @xframe_options_exempt def swh_coverage(request): - return render(request, 'coverage.html', {'providers': _code_providers}) + return render(request, 'coverage.html', {'providers': code_providers}) diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -233,3 +233,13 @@ LOGIN_REDIRECT_URL = 'admin' SESSION_ENGINE = 'django.contrib.sessions.backends.cache' + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache' + }, + 'db_cache': { + 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', + 'LOCATION': 'swh_web_cache', + } +} diff --git a/swh/web/settings/production.py b/swh/web/settings/production.py --- a/swh/web/settings/production.py +++ b/swh/web/settings/production.py @@ -24,7 +24,7 @@ MIDDLEWARE += ['swh.web.common.middlewares.HtmlMinifyMiddleware', 'django.middleware.cache.FetchFromCacheMiddleware'] -CACHES = { +CACHES.update({ 'default': { 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': swh_web_config['throttling']['cache_uri'], diff --git a/swh/web/templates/coverage.html b/swh/web/templates/coverage.html --- a/swh/web/templates/coverage.html +++ b/swh/web/templates/coverage.html @@ -18,6 +18,7 @@ Software Heritage archive coverage {% render_bundle 'vendors' %} {% render_bundle 'webapp' %} + @@ -28,10 +29,11 @@
{% for provider in providers %}
-
+
+
{% endfor %} @@ -40,4 +42,19 @@
+ diff --git a/swh/web/templates/homepage.html b/swh/web/templates/homepage.html --- a/swh/web/templates/homepage.html +++ b/swh/web/templates/homepage.html @@ -115,5 +115,6 @@ $('.swh-coverage-list').iFrameResize({heightCalculationMethod: 'taggedElement'}); }); swh.webapp.initPage('home'); + {% endblock %}