diff --git a/swh/web/admin/adminurls.py b/swh/web/admin/adminurls.py index 6a70cc9a..752cdc3e 100644 --- a/swh/web/admin/adminurls.py +++ b/swh/web/admin/adminurls.py @@ -1,35 +1,35 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common.urlsindex import UrlsIndex class AdminUrls(UrlsIndex): """ Class to manage swh-web admin urls. """ scope = 'admin' def admin_route(*url_patterns, view_name=None): """ Decorator to ease the registration of a swh-web admin endpoint Args: url_patterns: list of url patterns used by Django to identify the admin routes view_name: the name of the Django view associated to the routes used to reverse the url """ url_patterns = ['^' + url_pattern + '$' for url_pattern in url_patterns] def decorator(f): # register the route and its view in the browse endpoints index for url_pattern in url_patterns: AdminUrls.add_url_pattern(url_pattern, f, view_name) return f return decorator diff --git a/swh/web/admin/origin_save.py b/swh/web/admin/origin_save.py index 78a95eb7..a03c0fa6 100644 --- a/swh/web/admin/origin_save.py +++ b/swh/web/admin/origin_save.py @@ -1,194 +1,194 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from django.conf import settings from django.contrib.admin.views.decorators import staff_member_required from django.core.exceptions import ObjectDoesNotExist from django.core.paginator import Paginator from django.http import HttpResponse from django.shortcuts import render from django.views.decorators.http import require_POST from swh.web.admin.adminurls import admin_route from swh.web.common.models import ( SaveAuthorizedOrigin, SaveUnauthorizedOrigin, SaveOriginRequest ) from swh.web.common.origin_save import ( create_save_origin_request, SAVE_REQUEST_PENDING, SAVE_REQUEST_REJECTED ) @admin_route(r'origin/save/', view_name='admin-origin-save') @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save(request): return render(request, 'admin/origin-save.html') def _datatables_origin_urls_response(request, urls_query_set): search_value = request.GET['search[value]'] if search_value: urls_query_set = urls_query_set.filter(url__icontains=search_value) column_order = request.GET['order[0][column]'] field_order = request.GET['columns[%s][name]' % column_order] order_dir = request.GET['order[0][dir]'] if order_dir == 'desc': field_order = '-' + field_order urls_query_set = urls_query_set.order_by(field_order) table_data = {} table_data['draw'] = int(request.GET['draw']) table_data['recordsTotal'] = urls_query_set.count() table_data['recordsFiltered'] = urls_query_set.count() length = int(request.GET['length']) page = int(request.GET['start']) / length + 1 paginator = Paginator(urls_query_set, length) urls_query_set = paginator.page(page).object_list table_data['data'] = [{'url': u.url} for u in urls_query_set] table_data_json = json.dumps(table_data, separators=(',', ': ')) return HttpResponse(table_data_json, content_type='application/json') @admin_route(r'origin/save/authorized_urls/list/', view_name='admin-origin-save-authorized-urls-list') @staff_member_required def _admin_origin_save_authorized_urls_list(request): authorized_urls = SaveAuthorizedOrigin.objects.all() return _datatables_origin_urls_response(request, authorized_urls) @admin_route(r'origin/save/authorized_urls/add/(?P.+)/', view_name='admin-origin-save-add-authorized-url') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_add_authorized_url(request, origin_url): try: SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: # add the new authorized url SaveAuthorizedOrigin.objects.create(url=origin_url) # check if pending save requests with that url prefix exist pending_save_requests = \ SaveOriginRequest.objects.filter(origin_url__startswith=origin_url, status=SAVE_REQUEST_PENDING) # create origin save tasks for previously pending requests for psr in pending_save_requests: create_save_origin_request(psr.origin_type, psr.origin_url) status_code = 200 else: status_code = 400 return HttpResponse(status=status_code) @admin_route(r'origin/save/authorized_urls/remove/(?P.+)/', view_name='admin-origin-save-remove-authorized-url') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_remove_authorized_url(request, origin_url): try: entry = SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: status_code = 404 else: entry.delete() status_code = 200 return HttpResponse(status=status_code) @admin_route(r'origin/save/unauthorized_urls/list/', view_name='admin-origin-save-unauthorized-urls-list') @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_unauthorized_urls_list(request): unauthorized_urls = SaveUnauthorizedOrigin.objects.all() return _datatables_origin_urls_response(request, unauthorized_urls) @admin_route(r'origin/save/unauthorized_urls/add/(?P.+)/', view_name='admin-origin-save-add-unauthorized-url') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_add_unauthorized_url(request, origin_url): try: SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveUnauthorizedOrigin.objects.create(url=origin_url) # check if pending save requests with that url prefix exist pending_save_requests = \ SaveOriginRequest.objects.filter(origin_url__startswith=origin_url, status=SAVE_REQUEST_PENDING) # mark pending requests as rejected for psr in pending_save_requests: psr.status = SAVE_REQUEST_REJECTED psr.save() status_code = 200 else: status_code = 400 return HttpResponse(status=status_code) @admin_route(r'origin/save/unauthorized_urls/remove/(?P.+)/', view_name='admin-origin-save-remove-unauthorized-url') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_remove_unauthorized_url(request, origin_url): try: entry = SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: status_code = 404 else: entry.delete() status_code = 200 return HttpResponse(status=status_code) @admin_route(r'origin/save/request/accept/(?P.+)/url/(?P.+)/', # noqa view_name='admin-origin-save-request-accept') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_request_accept(request, origin_type, origin_url): try: SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveAuthorizedOrigin.objects.create(url=origin_url) create_save_origin_request(origin_type, origin_url) return HttpResponse(status=200) @admin_route(r'origin/save/request/reject/(?P.+)/url/(?P.+)/', # noqa view_name='admin-origin-save-request-reject') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_request_reject(request, origin_type, origin_url): try: SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveUnauthorizedOrigin.objects.create(url=origin_url) sor = SaveOriginRequest.objects.get(origin_type=origin_type, origin_url=origin_url, status=SAVE_REQUEST_PENDING) sor.status = SAVE_REQUEST_REJECTED sor.save() return HttpResponse(status=200) @admin_route(r'origin/save/request/remove/(?P.+)/', view_name='admin-origin-save-request-remove') @require_POST @staff_member_required(login_url=settings.LOGIN_URL) def _admin_origin_save_request_remove(request, sor_id): try: entry = SaveOriginRequest.objects.get(id=sor_id) except ObjectDoesNotExist: status_code = 404 else: entry.delete() status_code = 200 return HttpResponse(status=status_code) diff --git a/swh/web/api/apidoc.py b/swh/web/api/apidoc.py index 96d859ae..3523c12c 100644 --- a/swh/web/api/apidoc.py +++ b/swh/web/api/apidoc.py @@ -1,367 +1,367 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import docutils.nodes import docutils.parsers.rst import docutils.utils import functools import os import re import textwrap from functools import wraps from rest_framework.decorators import api_view from swh.web.common.utils import parse_rst from swh.web.api.apiurls import APIUrls from swh.web.api.apiresponse import make_api_response, error_response class _HTTPDomainDocVisitor(docutils.nodes.NodeVisitor): """ docutils visitor for walking on a parsed rst document containing sphinx httpdomain roles. Its purpose is to extract relevant info regarding swh api endpoints (for instance url arguments) from their docstring written using sphinx httpdomain. """ # httpdomain roles we want to parse (based on sphinxcontrib.httpdomain 1.6) parameter_roles = ('param', 'parameter', 'arg', 'argument') response_json_object_roles = ('resjsonobj', 'resjson', '>jsonobj', '>json') response_json_array_roles = ('resjsonarr', '>jsonarr') query_parameter_roles = ('queryparameter', 'queryparam', 'qparam', 'query') request_header_roles = ('header', 'resheader', 'responseheader') status_code_roles = ('statuscode', 'status', 'code') def __init__(self, document, urls, data): super().__init__(document) self.urls = urls self.url_idx = 0 self.data = data self.args_set = set() self.params_set = set() self.returns_set = set() self.status_codes_set = set() self.reqheaders_set = set() self.resheaders_set = set() self.field_list_visited = False def process_paragraph(self, par): """ Process extracted paragraph text before display. Cleanup document model markups and transform the paragraph into a valid raw rst string (as the apidoc documentation transform rst to html when rendering). """ par = par.replace('\n', ' ') # keep emphasized, strong and literal text par = par.replace('', '*') par = par.replace('', '*') par = par.replace('', '**') par = par.replace('', '**') par = par.replace('', '``') par = par.replace('', '``') # remove parsed document markups par = re.sub('<[^<]+?>', '', par) # api urls cleanup to generate valid links afterwards par = re.sub('\(\w+\)', '', par) # noqa par = re.sub('\[.*\]', '', par) # noqa par = par.replace('//', '/') # transform references to api endpoints into valid rst links par = re.sub(':http:get:`(.*)`', r'`<\1>`_', par) # transform references to some elements into bold text par = re.sub(':http:header:`(.*)`', r'**\1**', par) par = re.sub(':func:`(.*)`', r'**\1**', par) return par def visit_field_list(self, node): """ Visit parsed rst field lists to extract relevant info regarding api endpoint. """ self.field_list_visited = True for child in node.traverse(): # get the parsed field name if isinstance(child, docutils.nodes.field_name): field_name = child.astext() # parse field text elif isinstance(child, docutils.nodes.paragraph): text = self.process_paragraph(str(child)) field_data = field_name.split(' ') # Parameters if field_data[0] in self.parameter_roles: if field_data[2] not in self.args_set: self.data['args'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.args_set.add(field_data[2]) # Query Parameters if field_data[0] in self.query_parameter_roles: if field_data[2] not in self.params_set: self.data['params'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.params_set.add(field_data[2]) # Response type if field_data[0] in self.response_json_array_roles or \ field_data[0] in self.response_json_object_roles: # array if field_data[0] in self.response_json_array_roles: self.data['return_type'] = 'array' # object else: self.data['return_type'] = 'object' # returned object field if field_data[2] not in self.returns_set: self.data['returns'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.returns_set.add(field_data[2]) # Status Codes if field_data[0] in self.status_code_roles: if field_data[1] not in self.status_codes_set: self.data['status_codes'].append({'code': field_data[1], # noqa 'doc': text}) self.status_codes_set.add(field_data[1]) # Request Headers if field_data[0] in self.request_header_roles: if field_data[1] not in self.reqheaders_set: self.data['reqheaders'].append({'name': field_data[1], 'doc': text}) self.reqheaders_set.add(field_data[1]) # Response Headers if field_data[0] in self.response_header_roles: if field_data[1] not in self.resheaders_set: resheader = {'name': field_data[1], 'doc': text} self.data['resheaders'].append(resheader) self.resheaders_set.add(field_data[1]) if resheader['name'] == 'Content-Type' and \ resheader['doc'] == 'application/octet-stream': self.data['return_type'] = 'octet stream' def visit_paragraph(self, node): """ Visit relevant paragraphs to parse """ # only parsed top level paragraphs if isinstance(node.parent, docutils.nodes.block_quote): text = self.process_paragraph(str(node)) # endpoint description if not text.startswith('**') and self.data['description'] != text: self.data['description'] += '\n\n' if self.data['description'] else '' # noqa self.data['description'] += text # http methods elif text.startswith('**Allowed HTTP Methods:**'): text = text.replace('**Allowed HTTP Methods:**', '') http_methods = text.strip().split(',') http_methods = [m[m.find('`')+1:-1].upper() for m in http_methods] self.data['urls'].append({'rule': self.urls[self.url_idx], 'methods': http_methods}) self.url_idx += 1 def visit_literal_block(self, node): """ Visit literal blocks """ text = node.astext() # literal block in endpoint description if not self.field_list_visited: self.data['description'] += \ ':\n\n%s\n' % textwrap.indent(text, '\t') # extract example url if ':swh_web_api:' in text: self.data['examples'].append( '/api/1/' + re.sub('.*`(.*)`.*', r'\1', text)) def visit_bullet_list(self, node): # bullet list in endpoint description if not self.field_list_visited: self.data['description'] += '\n\n' for child in node.traverse(): # process list item if isinstance(child, docutils.nodes.paragraph): line_text = self.process_paragraph(str(child)) self.data['description'] += '\t* %s\n' % line_text def unknown_visit(self, node): pass def depart_document(self, node): """ End of parsing extra processing """ default_methods = ['GET', 'HEAD', 'OPTIONS'] # ensure urls info is present and set default http methods if not self.data['urls']: for url in self.urls: self.data['urls'].append({'rule': url, 'methods': default_methods}) def unknown_departure(self, node): pass def _parse_httpdomain_doc(doc, data): doc_lines = doc.split('\n') doc_lines_filtered = [] urls = [] # httpdomain is a sphinx extension that is unknown to docutils but # fortunately we can still parse its directives' content, # so remove lines with httpdomain directives before executing the # rst parser from docutils for doc_line in doc_lines: if '.. http' not in doc_line: doc_lines_filtered.append(doc_line) else: url = doc_line[doc_line.find('/'):] # emphasize url arguments for html rendering url = re.sub(r'\((\w+)\)', r' **\(\1\)** ', url) urls.append(url) # parse the rst docstring and do not print system messages about # unknown httpdomain roles document = parse_rst('\n'.join(doc_lines_filtered), report_level=5) # remove the system_message nodes from the parsed document for node in document.traverse(docutils.nodes.system_message): node.parent.remove(node) # visit the document nodes to extract relevant endpoint info visitor = _HTTPDomainDocVisitor(document, urls, data) document.walkabout(visitor) class APIDocException(Exception): """ Custom exception to signal errors in the use of the APIDoc decorators """ def api_doc(route, noargs=False, need_params=False, tags=[], handle_response=False, api_version='1'): """ Decorate an API function to register it in the API doc route index and create the corresponding DRF route. Args: route (str): documentation page's route noargs (boolean): set to True if the route has no arguments, and its result should be displayed anytime its documentation is requested. Default to False need_params (boolean): specify the route requires query parameters otherwise errors will occur. It enables to avoid displaying the invalid response in its HTML documentation. Default to False. tags (list): Further information on api endpoints. Two values are possibly expected: * hidden: remove the entry points from the listing * upcoming: display the entry point but it is not followable handle_response (boolean): indicate if the decorated function takes care of creating the HTTP response or delegates that task to the apiresponse module api_version (str): api version string """ urlpattern = '^' + api_version + route + '$' tags = set(tags) # @api_doc() Decorator call def decorator(f): # If the route is not hidden, add it to the index if 'hidden' not in tags: doc_data = get_doc_data(f, route, noargs) doc_desc = doc_data['description'] first_dot_pos = doc_desc.find('.') APIUrls.add_route(route, doc_desc[:first_dot_pos+1], tags=tags) # If the decorated route has arguments, we create a specific # documentation view if not noargs: @api_view(['GET', 'HEAD']) @wraps(f) def doc_view(request): doc_data = get_doc_data(f, route, noargs) return make_api_response(request, None, doc_data) view_name = 'api-%s-%s' % \ (api_version, route[1:-1].replace('/', '-')) APIUrls.add_url_pattern(urlpattern, doc_view, view_name) @wraps(f) def documented_view(request, **kwargs): doc_data = get_doc_data(f, route, noargs) try: response = f(request, **kwargs) except Exception as exc: if request.accepted_media_type == 'text/html' and \ need_params and not request.query_params: response = None else: return error_response(request, exc, doc_data) if handle_response: return response else: return make_api_response(request, response, doc_data) return documented_view return decorator @functools.lru_cache(maxsize=32) def get_doc_data(f, route, noargs): """ Build documentation data for the decorated api endpoint function """ data = { 'description': '', 'response_data': None, 'urls': [], 'args': [], 'params': [], 'resheaders': [], 'reqheaders': [], 'return_type': '', 'returns': [], 'status_codes': [], 'examples': [], 'route': route, 'noargs': noargs } if not f.__doc__: raise APIDocException('apidoc: expected a docstring' ' for function %s' % (f.__name__,)) # use raw docstring as endpoint documentation if sphinx # httpdomain is not used if '.. http' not in f.__doc__: data['description'] = f.__doc__ # else parse the sphinx httpdomain docstring with docutils # (except when building the swh-web documentation through autodoc # sphinx extension, not needed and raise errors with sphinx >= 1.7) elif 'SWH_WEB_DOC_BUILD' not in os.environ: _parse_httpdomain_doc(f.__doc__, data) # process returned object info for nicer html display returns_list = '' for ret in data['returns']: returns_list += '\t* **%s (%s)**: %s\n' %\ (ret['name'], ret['type'], ret['doc']) data['returns_list'] = returns_list return data diff --git a/swh/web/api/apiresponse.py b/swh/web/api/apiresponse.py index cb39f51b..734c25d9 100644 --- a/swh/web/api/apiresponse.py +++ b/swh/web/api/apiresponse.py @@ -1,190 +1,190 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import traceback from django.utils.html import escape from rest_framework.response import Response from swh.storage.exc import StorageDBError, StorageAPIError from swh.web.api import utils from swh.web.common.exc import NotFoundExc, ForbiddenExc from swh.web.common.utils import shorten_path, gen_path_info from swh.web.config import get_config def compute_link_header(rv, options): """Add Link header in returned value results. Args: rv (dict): dictionary with keys: - headers: potential headers with 'link-next' and 'link-prev' keys - results: containing the result to return options (dict): the initial dict to update with result if any Returns: dict: dictionary with optional keys 'link-next' and 'link-prev' """ link_headers = [] if 'headers' not in rv: return {} rv_headers = rv['headers'] if 'link-next' in rv_headers: link_headers.append('<%s>; rel="next"' % ( rv_headers['link-next'])) if 'link-prev' in rv_headers: link_headers.append('<%s>; rel="previous"' % ( rv_headers['link-prev'])) if link_headers: link_header_str = ','.join(link_headers) headers = options.get('headers', {}) headers.update({ 'Link': link_header_str }) return headers return {} def filter_by_fields(request, data): """Extract a request parameter 'fields' if it exists to permit the filtering on the data dict's keys. If such field is not provided, returns the data as is. """ fields = request.query_params.get('fields') if fields: fields = set(fields.split(',')) data = utils.filter_field_keys(data, fields) return data def transform(rv): """Transform an eventual returned value with multiple layer of information with only what's necessary. If the returned value rv contains the 'results' key, this is the associated value which is returned. Otherwise, return the initial dict without the potential 'headers' key. """ if 'results' in rv: return rv['results'] if 'headers' in rv: rv.pop('headers') return rv def make_api_response(request, data, doc_data={}, options={}): """Generates an API response based on the requested mimetype. Args: request: a DRF Request object data: raw data to return in the API response doc_data: documentation data for HTML response options: optional data that can be used to generate the response Returns: a DRF Response a object """ if data: options['headers'] = compute_link_header(data, options) data = transform(data) data = filter_by_fields(request, data) doc_env = doc_data headers = {} if 'headers' in options: doc_env['headers_data'] = options['headers'] headers = options['headers'] # get request status code doc_env['status_code'] = options.get('status', 200) response_args = {'status': doc_env['status_code'], 'headers': headers, 'content_type': request.accepted_media_type} # when requesting HTML, typically when browsing the API through its # documented views, we need to enrich the input data with documentation # related ones and inform DRF that we request HTML template rendering if request.accepted_media_type == 'text/html': if data: data = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')) doc_env['response_data'] = data doc_env['request'] = { 'path': request.path, 'method': request.method, 'absolute_uri': request.build_absolute_uri(), } doc_env['heading'] = shorten_path(str(request.path)) if 'route' in doc_env: doc_env['endpoint_path'] = gen_path_info(doc_env['route']) response_args['data'] = doc_env response_args['template_name'] = 'api/apidoc.html' # otherwise simply return the raw data and let DRF picks # the correct renderer (JSON or YAML) else: response_args['data'] = data return Response(**response_args) def error_response(request, error, doc_data): """Private function to create a custom error response. Args: request: a DRF Request object error: the exception that caused the error doc_data: documentation data for HTML response """ error_code = 400 if isinstance(error, NotFoundExc): error_code = 404 elif isinstance(error, ForbiddenExc): error_code = 403 elif isinstance(error, StorageDBError): error_code = 503 elif isinstance(error, StorageAPIError): error_code = 503 error_opts = {'status': error_code} error_data = { 'exception': error.__class__.__name__, 'reason': str(error), } if request.accepted_media_type == 'text/html': error_data['reason'] = escape(error_data['reason']) if get_config()['debug']: error_data['traceback'] = traceback.format_exc() return make_api_response(request, error_data, doc_data, options=error_opts) diff --git a/swh/web/api/apiurls.py b/swh/web/api/apiurls.py index 60634fa6..c356c9cd 100644 --- a/swh/web/api/apiurls.py +++ b/swh/web/api/apiurls.py @@ -1,85 +1,85 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools from rest_framework.decorators import api_view from swh.web.common.urlsindex import UrlsIndex from swh.web.common import throttling class APIUrls(UrlsIndex): """ Class to manage API documentation URLs. - Indexes all routes documented using apidoc's decorators. - Tracks endpoint/request processing method relationships for use in generating related urls in API documentation """ _apidoc_routes = {} _method_endpoints = {} scope = 'api' @classmethod def get_app_endpoints(cls): return cls._apidoc_routes @classmethod def add_route(cls, route, docstring, **kwargs): """ Add a route to the self-documenting API reference """ route_view_name = 'api-1-%s' % route[1:-1].replace('/', '-') if route not in cls._apidoc_routes: d = {'docstring': docstring, 'route_view_name': route_view_name} for k, v in kwargs.items(): d[k] = v cls._apidoc_routes[route] = d def api_route(url_pattern=None, view_name=None, methods=['GET', 'HEAD', 'OPTIONS'], throttle_scope='swh_api', api_version='1', checksum_args=None): """ Decorator to ease the registration of an API endpoint using the Django REST Framework. Args: url_pattern: the url pattern used by DRF to identify the API route view_name: the name of the API view associated to the route used to reverse the url methods: array of HTTP methods supported by the API route """ url_pattern = '^' + api_version + url_pattern + '$' def decorator(f): # create a DRF view from the wrapped function @api_view(methods) @throttling.throttle_scope(throttle_scope) @functools.wraps(f) def api_view_f(*args, **kwargs): return f(*args, **kwargs) # small hacks for correctly generating API endpoints index doc api_view_f.__name__ = f.__name__ api_view_f.http_method_names = methods # register the route and its view in the endpoints index APIUrls.add_url_pattern(url_pattern, api_view_f, view_name) if checksum_args: APIUrls.add_redirect_for_checksum_args(view_name, [url_pattern], checksum_args) return f return decorator diff --git a/swh/web/api/utils.py b/swh/web/api/utils.py index b84a1049..d4f2d49b 100644 --- a/swh/web/api/utils.py +++ b/swh/web/api/utils.py @@ -1,211 +1,211 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common.utils import reverse from swh.web.common.query import parse_hash def filter_field_keys(data, field_keys): """Given an object instance (directory or list), and a csv field keys to filter on. Return the object instance with filtered keys. Note: Returns obj as is if it's an instance of types not in (dictionary, list) Args: - data: one object (dictionary, list...) to filter. - field_keys: csv or set of keys to filter the object on Returns: obj filtered on field_keys """ if isinstance(data, map): return map(lambda x: filter_field_keys(x, field_keys), data) if isinstance(data, list): return [filter_field_keys(x, field_keys) for x in data] if isinstance(data, dict): return {k: v for (k, v) in data.items() if k in field_keys} return data def person_to_string(person): """Map a person (person, committer, tagger, etc...) to a string. """ return ''.join([person['name'], ' <', person['email'], '>']) def enrich_object(object): """Enrich an object (revision, release) with link to the 'target' of type 'target_type'. Args: object: An object with target and target_type keys (e.g. release, revision) Returns: Object enriched with target_url pointing to the right swh.web.ui.api urls for the pointing object (revision, release, content, directory) """ obj = object.copy() if 'target' in obj and 'target_type' in obj: if obj['target_type'] in ('revision', 'release', 'directory'): obj['target_url'] = \ reverse('api-1-%s' % obj['target_type'], url_args={'sha1_git': obj['target']}) elif obj['target_type'] == 'content': obj['target_url'] = \ reverse('api-1-content', url_args={'q': 'sha1_git:' + obj['target']}) elif obj['target_type'] == 'snapshot': obj['target_url'] = \ reverse('api-1-snapshot', url_args={'snapshot_id': obj['target']}) if 'author' in obj: author = obj['author'] obj['author_url'] = reverse('api-1-person', url_args={'person_id': author['id']}) return obj enrich_release = enrich_object def enrich_directory(directory, context_url=None): """Enrich directory with url to content or directory. """ if 'type' in directory: target_type = directory['type'] target = directory['target'] if target_type == 'file': directory['target_url'] = reverse( 'api-1-content', url_args={'q': 'sha1_git:%s' % target}) if context_url: directory['file_url'] = context_url + directory['name'] + '/' elif target_type == 'dir': directory['target_url'] = reverse( 'api-1-directory', url_args={'sha1_git': target}) if context_url: directory['dir_url'] = context_url + directory['name'] + '/' else: directory['target_url'] = reverse( 'api-1-revision', url_args={'sha1_git': target}) if context_url: directory['rev_url'] = context_url + directory['name'] + '/' return directory def enrich_metadata_endpoint(content): """Enrich metadata endpoint with link to the upper metadata endpoint. """ c = content.copy() c['content_url'] = reverse('api-1-content', url_args={'q': 'sha1:%s' % c['id']}) return c def enrich_content(content, top_url=False, query_string=None): """Enrich content with links to: - data_url: its raw data - filetype_url: its filetype information - language_url: its programming language information - license_url: its licensing information Args: content: dict of data associated to a swh content object top_url: whether or not to include the content url in the enriched data query_string: optional query string of type ':' used when requesting the content, it acts as a hint for picking the same hash method when computing the url listed above Returns: An enriched content dict filled with additional urls """ checksums = content if 'checksums' in content: checksums = content['checksums'] hash_algo = 'sha1' if query_string: hash_algo = parse_hash(query_string)[0] if hash_algo in checksums: q = '%s:%s' % (hash_algo, checksums[hash_algo]) if top_url: content['content_url'] = reverse( 'api-1-content', url_args={'q': q}) content['data_url'] = reverse('api-1-content-raw', url_args={'q': q}) content['filetype_url'] = reverse( 'api-1-content-filetype', url_args={'q': q}) content['language_url'] = reverse( 'api-1-content-language', url_args={'q': q}) content['license_url'] = reverse( 'api-1-content-license', url_args={'q': q}) return content def enrich_revision(revision): """Enrich revision with links where it makes sense (directory, parents). Keep track of the navigation breadcrumbs if they are specified. Args: revision: the revision as a dict """ revision['url'] = reverse('api-1-revision', url_args={'sha1_git': revision['id']}) revision['history_url'] = reverse('api-1-revision-log', url_args={'sha1_git': revision['id']}) if 'author' in revision: author = revision['author'] revision['author_url'] = reverse('api-1-person', url_args={'person_id': author['id']}) if 'committer' in revision: committer = revision['committer'] revision['committer_url'] = reverse( 'api-1-person', url_args={'person_id': committer['id']}) if 'directory' in revision: revision['directory_url'] = reverse( 'api-1-directory', url_args={'sha1_git': revision['directory']}) if 'parents' in revision: parents = [] for parent in revision['parents']: parents.append({ 'id': parent, 'url': reverse('api-1-revision', url_args={'sha1_git': parent}) }) revision['parents'] = parents if 'children' in revision: children = [] for child in revision['children']: children.append(reverse( 'api-1-revision', url_args={'sha1_git': child})) revision['children_urls'] = children if 'message_decoding_failed' in revision: revision['message_url'] = \ reverse('api-1-revision-raw-message', url_args={'sha1_git': revision['id']}) return revision diff --git a/swh/web/api/views/content.py b/swh/web/api/views/content.py index c632991e..6d37ed4a 100644 --- a/swh/web/api/views/content.py +++ b/swh/web/api/views/content.py @@ -1,382 +1,382 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools from django.http import HttpResponse from swh.web.common import service from swh.web.common.utils import reverse from swh.web.common.exc import NotFoundExc from swh.web.api.apidoc import api_doc from swh.web.api import utils from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup @api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/filetype/', 'api-1-content-filetype', checksum_args=['q']) @api_doc('/content/filetype/') def api_content_filetype(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/filetype/ Get information about the detected MIME type of a content object. :param string hash_type: optional parameter specifying which hashing algorithm has been used to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not provided, it is assumed that the hashing algorithm used is `sha1`. :param string hash: hexadecimal representation of the checksum value computed with the specified hashing algorithm. :>json object content_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/` for getting information about the content :>json string encoding: the detected content encoding :>json string id: the **sha1** identifier of the content :>json string mimetype: the detected MIME type of the content :>json object tool: information about the tool used to detect the content filetype :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`content/sha1:dc2830a9e72f23c1dfebef4413003221baa5fb62/filetype/` """ # noqa return api_lookup( service.lookup_content_filetype, q, notfound_msg='No filetype information found for content {}.'.format(q), enrich_fn=utils.enrich_metadata_endpoint) @api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/language/', 'api-1-content-language', checksum_args=['q']) @api_doc('/content/language/') def api_content_language(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/language/ Get information about the programming language used in a content object. Note: this endpoint currently returns no data. :param string hash_type: optional parameter specifying which hashing algorithm has been used to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not provided, it is assumed that the hashing algorithm used is ``sha1``. :param string hash: hexadecimal representation of the checksum value computed with the specified hashing algorithm. :>json object content_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/` for getting information about the content :>json string id: the **sha1** identifier of the content :>json string lang: the detected programming language if any :>json object tool: information about the tool used to detect the programming language :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`content/sha1:dc2830a9e72f23c1dfebef4413003221baa5fb62/language/` """ # noqa return api_lookup( service.lookup_content_language, q, notfound_msg='No language information found for content {}.'.format(q), enrich_fn=utils.enrich_metadata_endpoint) @api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/license/', 'api-1-content-license', checksum_args=['q']) @api_doc('/content/license/') def api_content_license(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/license/ Get information about the license of a content object. :param string hash_type: optional parameter specifying which hashing algorithm has been used to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not provided, it is assumed that the hashing algorithm used is ``sha1``. :param string hash: hexadecimal representation of the checksum value computed with the specified hashing algorithm. :>json object content_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/` for getting information about the content :>json string id: the **sha1** identifier of the content :>json array licenses: array of strings containing the detected license names if any :>json object tool: information about the tool used to detect the license :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`content/sha1:dc2830a9e72f23c1dfebef4413003221baa5fb62/license/` """ # noqa return api_lookup( service.lookup_content_license, q, notfound_msg='No license information found for content {}.'.format(q), enrich_fn=utils.enrich_metadata_endpoint) @api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/ctags/', 'api-1-content-ctags') @api_doc('/content/ctags/', tags=['hidden']) def api_content_ctags(request, q): """ Get information about all `Ctags `_-style symbols defined in a content object. """ return api_lookup( service.lookup_content_ctags, q, notfound_msg='No ctags symbol found for content {}.'.format(q), enrich_fn=utils.enrich_metadata_endpoint) @api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/raw/', 'api-1-content-raw', checksum_args=['q']) @api_doc('/content/raw/', handle_response=True) def api_content_raw(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/raw/ Get the raw content of a content object (aka a "blob"), as a byte sequence. :param string hash_type: optional parameter specifying which hashing algorithm has been used to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not provided, it is assumed that the hashing algorithm used is ``sha1``. :param string hash: hexadecimal representation of the checksum value computed with the specified hashing algorithm. :query string filename: if provided, the downloaded content will get that filename :resheader Content-Type: application/octet-stream **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`content/sha1:dc2830a9e72f23c1dfebef4413003221baa5fb62/raw/` """ # noqa def generate(content): yield content['data'] content_raw = service.lookup_content_raw(q) if not content_raw: raise NotFoundExc('Content %s is not found.' % q) filename = request.query_params.get('filename') if not filename: filename = 'content_%s_raw' % q.replace(':', '_') response = HttpResponse(generate(content_raw), content_type='application/octet-stream') response['Content-disposition'] = 'attachment; filename=%s' % filename return response @api_route(r'/content/symbol/(?P.+)/', 'api-1-content-symbol') @api_doc('/content/symbol/', tags=['hidden']) def api_content_symbol(request, q=None): """Search content objects by `Ctags `_-style symbol (e.g., function name, data type, method, ...). """ result = {} last_sha1 = request.query_params.get('last_sha1', None) per_page = int(request.query_params.get('per_page', '10')) def lookup_exp(exp, last_sha1=last_sha1, per_page=per_page): exp = list(service.lookup_expression(exp, last_sha1, per_page)) return exp if exp else None symbols = api_lookup( lookup_exp, q, notfound_msg="No indexed raw content match expression '{}'.".format(q), enrich_fn=functools.partial(utils.enrich_content, top_url=True)) if symbols: nb_symbols = len(symbols) if nb_symbols == per_page: query_params = {} new_last_sha1 = symbols[-1]['sha1'] query_params['last_sha1'] = new_last_sha1 if request.query_params.get('per_page'): query_params['per_page'] = per_page result['headers'] = { 'link-next': reverse('api-1-content-symbol', url_args={'q': q}, query_params=query_params) } result.update({ 'results': symbols }) return result @api_route(r'/content/known/search/', 'api-1-content-known', methods=['POST']) @api_route(r'/content/known/(?P(?!search).*)/', 'api-1-content-known') @api_doc('/content/known/', tags=['hidden']) def api_check_content_known(request, q=None): """ .. http:get:: /api/1/content/known/(sha1)[,(sha1), ...,(sha1)]/ Check whether some content(s) (aka "blob(s)") is present in the archive based on its **sha1** checksum. :param string sha1: hexadecimal representation of the **sha1** checksum value for the content to check existence. Multiple values can be provided separated by ','. :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>json array search_res: array holding the search result for each provided **sha1** :>json object search_stats: some statistics regarding the number of **sha1** provided and the percentage of those found in the archive **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid **sha1** has been provided **Example:** .. parsed-literal:: :swh_web_api:`content/known/dc2830a9e72f23c1dfebef4413003221baa5fb62,0c3f19cb47ebfbe643fb19fa94c874d18fa62d12/` """ # noqa response = {'search_res': None, 'search_stats': None} search_stats = {'nbfiles': 0, 'pct': 0} search_res = None queries = [] # GET: Many hash separated values request if q: hashes = q.split(',') for v in hashes: queries.append({'filename': None, 'sha1': v}) # POST: Many hash requests in post form submission elif request.method == 'POST': data = request.data # Remove potential inputs with no associated value for k, v in data.items(): if v is not None: if k == 'q' and len(v) > 0: queries.append({'filename': None, 'sha1': v}) elif v != '': queries.append({'filename': k, 'sha1': v}) if queries: lookup = service.lookup_multiple_hashes(queries) result = [] nb_queries = len(queries) for el in lookup: res_d = {'sha1': el['sha1'], 'found': el['found']} if 'filename' in el and el['filename']: res_d['filename'] = el['filename'] result.append(res_d) search_res = result nbfound = len([x for x in lookup if x['found']]) search_stats['nbfiles'] = nb_queries search_stats['pct'] = (nbfound / nb_queries) * 100 response['search_res'] = search_res response['search_stats'] = search_stats return response @api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/', 'api-1-content', checksum_args=['q']) @api_doc('/content/') def api_content_metadata(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/ Get information about a content (aka a "blob") object. In the archive, a content object is identified based on checksum values computed using various hashing algorithms. :param string hash_type: optional parameter specifying which hashing algorithm has been used to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not provided, it is assumed that the hashing algorithm used is ``sha1``. :param string hash: hexadecimal representation of the checksum value computed with the specified hashing algorithm. :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>json object checksums: object holding the computed checksum values for the requested content :>json string data_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/raw/` for downloading the content raw bytes :>json string filetype_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/filetype/` for getting information about the content MIME type :>json string language_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/language/` for getting information about the programming language used in the content :>json number length: length of the content in bytes :>json string license_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/license/` for getting information about the license of the content **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: curl -i :swh_web_api:`content/sha1_git:fe95a46679d128ff167b7c55df5d02356c5a1ae1/` """ # noqa return api_lookup( service.lookup_content, q, notfound_msg='Content with {} not found.'.format(q), enrich_fn=functools.partial(utils.enrich_content, query_string=q)) diff --git a/swh/web/api/views/directory.py b/swh/web/api/views/directory.py index 682fd099..ae912bc3 100644 --- a/swh/web/api/views/directory.py +++ b/swh/web/api/views/directory.py @@ -1,77 +1,77 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api import utils from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup @api_route(r'/directory/(?P[0-9a-f]+)/', 'api-1-directory', checksum_args=['sha1_git']) @api_route(r'/directory/(?P[0-9a-f]+)/(?P.+)/', 'api-1-directory', checksum_args=['sha1_git']) @api_doc('/directory/') def api_directory(request, sha1_git, path=None): """ .. http:get:: /api/1/directory/(sha1_git)/[(path)/] Get information about directory objects. Directories are identified by **sha1** checksums, compatible with Git directory identifiers. See :func:`swh.model.identifiers.directory_identifier` in our data model module for details about how they are computed. When given only a directory identifier, this endpoint returns information about the directory itself, returning its content (usually a list of directory entries). When given a directory identifier and a path, this endpoint returns information about the directory entry pointed by the relative path, starting path resolution from the given directory. :param string sha1_git: hexadecimal representation of the directory **sha1_git** identifier :param string path: optional parameter to get information about the directory entry pointed by that relative path :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>jsonarr object checksums: object holding the computed checksum values for a directory entry (only for file entries) :>jsonarr string dir_id: **sha1_git** identifier of the requested directory :>jsonarr number length: length of a directory entry in bytes (only for file entries) for getting information about the content MIME type :>jsonarr string name: the directory entry name :>jsonarr number perms: permissions for the directory entry :>jsonarr string target: **sha1_git** identifier of the directory entry :>jsonarr string target_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/` or :http:get:`/api/1/directory/(sha1_git)/[(path)/]` depending on the directory entry type :>jsonarr string type: the type of the directory entry, can be either ``dir``, ``file`` or ``rev`` **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested directory can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`directory/977fc4b98c0e85816348cebd3b12026407c368b6/` """ # noqa if path: error_msg_path = ('Entry with path %s relative to directory ' 'with sha1_git %s not found.') % (path, sha1_git) return api_lookup( service.lookup_directory_with_path, sha1_git, path, notfound_msg=error_msg_path, enrich_fn=utils.enrich_directory) else: error_msg_nopath = 'Directory with sha1_git %s not found.' % sha1_git return api_lookup( service.lookup_directory, sha1_git, notfound_msg=error_msg_nopath, enrich_fn=utils.enrich_directory) diff --git a/swh/web/api/views/identifiers.py b/swh/web/api/views/identifiers.py index f6cf790b..d3e5cc7e 100644 --- a/swh/web/api/views/identifiers.py +++ b/swh/web/api/views/identifiers.py @@ -1,77 +1,77 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.model.identifiers import ( CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT ) from swh.web.common import service from swh.web.common.utils import resolve_swh_persistent_id from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route @api_route(r'/resolve/(?P.*)/', 'api-1-resolve-swh-pid') @api_doc('/resolve/') def api_resolve_swh_pid(request, swh_id): """ .. http:get:: /api/1/resolve/(swh_id)/ Resolve a Software Heritage persistent identifier. Try to resolve a provided `persistent identifier `_ into an url for browsing the pointed archive object. If the provided identifier is valid, the existence of the object in the archive will also be checked. :param string swh_id: a Software Heritage presistent identifier :>json string browse_url: the url for browsing the pointed object :>json object metadata: object holding optional parts of the persistent identifier :>json string namespace: the persistent identifier namespace :>json string object_id: the hash identifier of the pointed object :>json string object_type: the type of the pointed object :>json number scheme_version: the scheme version of the persistent identifier :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid persistent identifier has been provided :statuscode 404: the pointed object does not exist in the archive **Example:** .. parsed-literal:: :swh_web_api:`resolve/swh:1:rev:96db9023b881d7cd9f379b0c154650d6c108e9a3;origin=https://github.com/openssl/openssl/` """ # noqa # try to resolve the provided pid swh_id_resolved = resolve_swh_persistent_id(swh_id) # id is well-formed, now check that the pointed # object is present in the archive, NotFoundExc # will be raised otherwise swh_id_parsed = swh_id_resolved['swh_id_parsed'] object_type = swh_id_parsed.object_type object_id = swh_id_parsed.object_id if object_type == CONTENT: service.lookup_content('sha1_git:%s' % object_id) elif object_type == DIRECTORY: service.lookup_directory(object_id) elif object_type == RELEASE: service.lookup_release(object_id) elif object_type == REVISION: service.lookup_revision(object_id) elif object_type == SNAPSHOT: service.lookup_snapshot(object_id) # id is well-formed and the pointed object exists swh_id_data = swh_id_parsed._asdict() swh_id_data['browse_url'] = swh_id_resolved['browse_url'] return swh_id_data diff --git a/swh/web/api/views/origin.py b/swh/web/api/views/origin.py index 51b3a7c1..eeb93a43 100644 --- a/swh/web/api/views/origin.py +++ b/swh/web/api/views/origin.py @@ -1,435 +1,435 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from distutils.util import strtobool from functools import partial from swh.web.common import service from swh.web.common.exc import BadInputExc from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import reverse from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup def _enrich_origin(origin): if 'id' in origin: o = origin.copy() o['origin_visits_url'] = reverse( 'api-1-origin-visits', url_args={'origin_id': origin['id']}) return o return origin def _enrich_origin_visit(origin_visit, *, with_origin_url, with_origin_visit_url): ov = origin_visit.copy() if with_origin_url: ov['origin_url'] = reverse('api-1-origin', url_args={'origin_id': ov['origin']}) if with_origin_visit_url: ov['origin_visit_url'] = reverse('api-1-origin-visit', url_args={'origin_id': ov['origin'], 'visit_id': ov['visit']}) snapshot = ov['snapshot'] if snapshot: ov['snapshot_url'] = reverse('api-1-snapshot', url_args={'snapshot_id': snapshot}) else: ov['snapshot_url'] = None return ov @api_route(r'/origins/', 'api-1-origins') @api_doc('/origins/', noargs=True) def api_origins(request): """ .. http:get:: /api/1/origins/ Get list of archived software origins. Origins are sorted by ids before returning them. :query int origin_from: The first origin id that will be included in returned results (default to 1) :query int origin_count: The maximum number of origins to return (default to 100, can not exceed 10000) :>jsonarr number id: the origin unique identifier :>jsonarr string origin_visits_url: link to in order to get information about the visits for that origin :>jsonarr string type: the type of software origin (possible values are ``git``, ``svn``, ``hg``, ``deb``, ``pypi``, ``ftp`` or ``deposit``) :>jsonarr string url: the origin canonical url :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :resheader Link: indicates that a subsequent or previous result page are available and contains the urls pointing to them **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`origins?origin_from=50000&origin_count=500` """ # noqa origin_from = int(request.query_params.get('origin_from', '1')) origin_count = int(request.query_params.get('origin_count', '100')) origin_count = min(origin_count, 10000) results = api_lookup( service.lookup_origins, origin_from, origin_count+1, enrich_fn=_enrich_origin) response = {'results': results, 'headers': {}} if len(results) > origin_count: origin_from = results.pop()['id'] response['headers']['link-next'] = reverse( 'api-1-origins', query_params={'origin_from': origin_from, 'origin_count': origin_count}) return response @api_route(r'/origin/(?P[0-9]+)/', 'api-1-origin') @api_route(r'/origin/(?P[a-z]+)/url/(?P.+)/', 'api-1-origin') @api_doc('/origin/') def api_origin(request, origin_id=None, origin_type=None, origin_url=None): """ .. http:get:: /api/1/origin/(origin_id)/ Get information about a software origin. :param int origin_id: a software origin identifier :>json number id: the origin unique identifier :>json string origin_visits_url: link to in order to get information about the visits for that origin :>json string type: the type of software origin (possible values are ``git``, ``svn``, ``hg``, ``deb``, ``pypi``, ``ftp`` or ``deposit``) :>json string url: the origin canonical url :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/1/` .. http:get:: /api/1/origin/(origin_type)/url/(origin_url)/ Get information about a software origin. :param string origin_type: the origin type (possible values are ``git``, ``svn``, ``hg``, ``deb``, ``pypi``, ``ftp`` or ``deposit``) :param string origin_url: the origin url :>json number id: the origin unique identifier :>json string origin_visits_url: link to in order to get information about the visits for that origin :>json string type: the type of software origin :>json string url: the origin canonical url :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/git/url/https://github.com/python/cpython/` """ # noqa ori_dict = { 'id': int(origin_id) if origin_id else None, 'type': origin_type, 'url': origin_url } ori_dict = {k: v for k, v in ori_dict.items() if ori_dict[k]} if 'id' in ori_dict: error_msg = 'Origin with id %s not found.' % ori_dict['id'] else: error_msg = 'Origin with type %s and URL %s not found' % ( ori_dict['type'], ori_dict['url']) return api_lookup( service.lookup_origin, ori_dict, notfound_msg=error_msg, enrich_fn=_enrich_origin) @api_route(r'/origin/search/(?P.+)/', 'api-1-origin-search') @api_doc('/origin/search/') def api_origin_search(request, url_pattern): """ .. http:get:: /api/1/origin/search/(url_pattern)/ Search for software origins whose urls contain a provided string pattern or match a provided regular expression. The search is performed in a case insensitive way. :param string url_pattern: a string pattern or a regular expression :query int offset: the number of found origins to skip before returning results :query int limit: the maximum number of found origins to return :query boolean regexp: if true, consider provided pattern as a regular expression and search origins whose urls match it :query boolean with_visit: if true, only return origins with at least one visit by Software heritage :>jsonarr number id: the origin unique identifier :>jsonarr string origin_visits_url: link to in order to get information about the visits for that origin :>jsonarr string type: the type of software origin :>jsonarr string url: the origin canonical url :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`origin/search/python/?limit=2` """ # noqa result = {} offset = int(request.query_params.get('offset', '0')) limit = int(request.query_params.get('limit', '70')) regexp = request.query_params.get('regexp', 'false') with_visit = request.query_params.get('with_visit', 'false') results = api_lookup(service.search_origin, url_pattern, offset, limit, bool(strtobool(regexp)), bool(strtobool(with_visit)), enrich_fn=_enrich_origin) nb_results = len(results) if nb_results == limit: query_params = {} query_params['offset'] = offset + limit query_params['limit'] = limit query_params['regexp'] = regexp result['headers'] = { 'link-next': reverse('api-1-origin-search', url_args={'url_pattern': url_pattern}, query_params=query_params) } result.update({ 'results': results }) return result @api_route(r'/origin/metadata-search/', 'api-1-origin-metadata-search') @api_doc('/origin/metadata-search/', noargs=True, need_params=True) def api_origin_metadata_search(request): """ .. http:get:: /api/1/origin/metadata-search/ Search for software origins whose metadata (expressed as a JSON-LD/CodeMeta dictionary) match the provided criteria. For now, only full-text search on this dictionary is supported. :query str fulltext: a string that will be matched against origin metadata; results are ranked and ordered starting with the best ones. :query int limit: the maximum number of found origins to return (bounded to 100) :>jsonarr number origin_id: the origin unique identifier :>jsonarr dict metadata: metadata of the origin (as a JSON-LD/CodeMeta dictionary) :>jsonarr string from_revision: the revision used to extract these metadata (the current HEAD or one of the former HEADs) :>jsonarr dict tool: the tool used to extract these metadata :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`origin/metadata-search/?limit=2&fulltext=Jane%20Doe` """ # noqa fulltext = request.query_params.get('fulltext', None) limit = min(int(request.query_params.get('limit', '70')), 100) if not fulltext: content = '"fulltext" must be provided and non-empty.' raise BadInputExc(content) results = api_lookup(service.search_origin_metadata, fulltext, limit) return { 'results': results, } @api_route(r'/origin/(?P[0-9]+)/visits/', 'api-1-origin-visits') @api_doc('/origin/visits/') def api_origin_visits(request, origin_id): """ .. http:get:: /api/1/origin/(origin_id)/visits/ Get information about all visits of a software origin. Visits are returned sorted in descending order according to their date. :param int origin_id: a software origin identifier :query int per_page: specify the number of visits to list, for pagination purposes :query int last_visit: visit to start listing from, for pagination purposes :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :resheader Link: indicates that a subsequent result page is available and contains the url pointing to it :>jsonarr string date: ISO representation of the visit date (in UTC) :>jsonarr number id: the unique identifier of the origin :>jsonarr string origin_visit_url: link to :http:get:`/api/1/origin/(origin_id)/visit/(visit_id)/` in order to get information about the visit :>jsonarr string snapshot: the snapshot identifier of the visit :>jsonarr string snapshot_url: link to :http:get:`/api/1/snapshot/(snapshot_id)/` in order to get information about the snapshot of the visit :>jsonarr string status: status of the visit (either **full**, **partial** or **ongoing**) :>jsonarr number visit: the unique identifier of the visit **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/1/visits/` """ # noqa result = {} origin_id = int(origin_id) per_page = int(request.query_params.get('per_page', '10')) last_visit = request.query_params.get('last_visit') if last_visit: last_visit = int(last_visit) def _lookup_origin_visits( origin_id, last_visit=last_visit, per_page=per_page): all_visits = get_origin_visits({'id': origin_id}) all_visits.reverse() visits = [] if not last_visit: visits = all_visits[:per_page] else: for i, v in enumerate(all_visits): if v['visit'] == last_visit: visits = all_visits[i+1:i+1+per_page] break for v in visits: yield v results = api_lookup(_lookup_origin_visits, origin_id, notfound_msg='No origin {} found'.format(origin_id), enrich_fn=partial(_enrich_origin_visit, with_origin_url=False, with_origin_visit_url=True)) if results: nb_results = len(results) if nb_results == per_page: new_last_visit = results[-1]['visit'] query_params = {} query_params['last_visit'] = new_last_visit if request.query_params.get('per_page'): query_params['per_page'] = per_page result['headers'] = { 'link-next': reverse('api-1-origin-visits', url_args={'origin_id': origin_id}, query_params=query_params) } result.update({ 'results': results }) return result @api_route(r'/origin/(?P[0-9]+)/visit/(?P[0-9]+)/', 'api-1-origin-visit') @api_doc('/origin/visit/') def api_origin_visit(request, origin_id, visit_id): """ .. http:get:: /api/1/origin/(origin_id)/visit/(visit_id)/ Get information about a specific visit of a software origin. :param int origin_id: a software origin identifier :param int visit_id: a visit identifier :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>json string date: ISO representation of the visit date (in UTC) :>json number origin: the origin unique identifier :>json string origin_url: link to get information about the origin :>jsonarr string snapshot: the snapshot identifier of the visit :>jsonarr string snapshot_url: link to :http:get:`/api/1/snapshot/(snapshot_id)/` in order to get information about the snapshot of the visit :>json string status: status of the visit (either **full**, **partial** or **ongoing**) :>json number visit: the unique identifier of the visit **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested origin or visit can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/1500/visit/1/` """ # noqa return api_lookup( service.lookup_origin_visit, int(origin_id), int(visit_id), notfound_msg=('No visit {} for origin {} found' .format(visit_id, origin_id)), enrich_fn=partial(_enrich_origin_visit, with_origin_url=True, with_origin_visit_url=False)) diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py index ecc7eb06..2dfe5154 100644 --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -1,84 +1,84 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.views.decorators.cache import never_cache from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route from swh.web.common.origin_save import ( create_save_origin_request, get_save_origin_requests ) @api_route(r'/origin/save/(?P.+)/url/(?P.+)/', 'api-1-save-origin', methods=['GET', 'POST'], throttle_scope='swh_save_origin') @never_cache @api_doc('/origin/save/') def api_save_origin(request, origin_type, origin_url): """ .. http:get:: /api/1/origin/save/(origin_type)/url/(origin_url)/ .. http:post:: /api/1/origin/save/(origin_type)/url/(origin_url)/ Request the saving of a software origin into the archive or check the status of previously created save requests. That endpoint enables to create a saving task for a software origin through a POST request. Depending of the provided origin url, the save request can either be: * immediately **accepted**, for well known code hosting providers like for instance GitHub or GitLab * **rejected**, in case the url is blacklisted by Software Heritage * **put in pending state** until a manual check is done in order to determine if it can be loaded or not Once a saving request has been accepted, its associated saving task status can then be checked through a GET request on the same url. Returned status can either be: * **not created**: no saving task has been created * **not yet scheduled**: saving task has been created but its execution has not yet been scheduled * **scheduled**: the task execution has been scheduled * **succeed**: the saving task has been successfully executed * **failed**: the saving task has been executed but it failed When issuing a POST request an object will be returned while a GET request will return an array of objects (as multiple save requests might have been submitted for the same origin). :param string origin_type: the type of origin to save (currently the supported types are ``git``, ``hg`` and ``svn``) :param string origin_url: the url of the origin to save :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>json string origin_url: the url of the origin to save :>json string origin_type: the type of the origin to save :>json string save_request_date: the date (in iso format) the save request was issued :>json string save_request_status: the status of the save request, either **accepted**, **rejected** or **pending** :>json string save_task_status: the status of the origin saving task, either **not created**, **not yet scheduled**, **scheduled**, **succeed** or **failed** **Allowed HTTP Methods:** :http:method:`get`, :http:method:`post`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid origin type or url has been provided :statuscode 403: the provided origin url is blacklisted :statuscode 404: no save requests have been found for a given origin """ # noqa if request.method == 'POST': sor = create_save_origin_request(origin_type, origin_url) del sor['id'] else: sor = get_save_origin_requests(origin_type, origin_url) for s in sor: del s['id'] # noqa return sor diff --git a/swh/web/api/views/person.py b/swh/web/api/views/person.py index 1aa79a55..1db95582 100644 --- a/swh/web/api/views/person.py +++ b/swh/web/api/views/person.py @@ -1,44 +1,44 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup @api_route(r'/person/(?P[0-9]+)/', 'api-1-person') @api_doc('/person/') def api_person(request, person_id): """ .. http:get:: /api/1/person/(person_id)/ Get information about a person in the archive. :param int person_id: a person identifier :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>json string email: the email of the person :>json string fullname: the full name of the person: combination of its name and email :>json number id: the unique identifier of the person :>json string name: the name of the person **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 404: requested person can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`person/8275/` """ # noqa return api_lookup( service.lookup_person, person_id, notfound_msg='Person with id {} not found.'.format(person_id)) diff --git a/swh/web/api/views/release.py b/swh/web/api/views/release.py index b8f0d1b1..bb1ef1c2 100644 --- a/swh/web/api/views/release.py +++ b/swh/web/api/views/release.py @@ -1,59 +1,59 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api import utils from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup @api_route(r'/release/(?P[0-9a-f]+)/', 'api-1-release', checksum_args=['sha1_git']) @api_doc('/release/') def api_release(request, sha1_git): """ .. http:get:: /api/1/release/(sha1_git)/ Get information about a release in the archive. Releases are identified by **sha1** checksums, compatible with Git tag identifiers. See :func:`swh.model.identifiers.release_identifier` in our data model module for details about how they are computed. :param string sha1_git: hexadecimal representation of the release **sha1_git** identifier :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>json object author: information about the author of the release :>json string author_url: link to :http:get:`/api/1/person/(person_id)/` to get information about the author of the release :>json string date: ISO representation of the release date (in UTC) :>json string id: the release unique identifier :>json string message: the message associated to the release :>json string name: the name of the release :>json string target: the target identifier of the release :>json string target_type: the type of the target, can be either **release**, **revision**, **content**, **directory** :>json string target_url: a link to the adequate api url based on the target type **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid **sha1_git** value has been provided :statuscode 404: requested release can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`release/208f61cc7a5dbc9879ae6e5c2f95891e270f09ef/` """ # noqa error_msg = 'Release with sha1_git %s not found.' % sha1_git return api_lookup( service.lookup_release, sha1_git, notfound_msg=error_msg, enrich_fn=utils.enrich_release) diff --git a/swh/web/api/views/snapshot.py b/swh/web/api/views/snapshot.py index 2979da4a..fee3e756 100644 --- a/swh/web/api/views/snapshot.py +++ b/swh/web/api/views/snapshot.py @@ -1,118 +1,118 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.common.utils import reverse from swh.web.config import get_config from swh.web.api.apidoc import api_doc from swh.web.api import utils from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup @api_route(r'/snapshot/(?P[0-9a-f]+)/', 'api-1-snapshot', checksum_args=['snapshot_id']) @api_doc('/snapshot/') def api_snapshot(request, snapshot_id): """ .. http:get:: /api/1/snapshot/(snapshot_id)/ Get information about a snapshot in the archive. A snapshot is a set of named branches, which are pointers to objects at any level of the Software Heritage DAG. It represents a full picture of an origin at a given time. As well as pointing to other objects in the Software Heritage DAG, branches can also be aliases, in which case their target is the name of another branch in the same snapshot, or dangling, in which case the target is unknown. A snapshot identifier is a salted sha1. See :func:`swh.model.identifiers.snapshot_identifier` in our data model module for details about how they are computed. :param sha1 snapshot_id: a snapshot identifier :query str branches_from: optional parameter used to skip branches whose name is lesser than it before returning them :query int branches_count: optional parameter used to restrain the amount of returned branches (default to 1000) :query str target_types: optional comma separated list parameter used to filter the target types of branch to return (possible values that can be contained in that list are ``content``, ``directory``, ``revision``, ``release``, ``snapshot`` or ``alias``) :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :resheader Link: indicates that a subsequent result page is available and contains the url pointing to it :>json object branches: object containing all branches associated to the snapshot, for each of them the associated target type and id are given but also a link to get information about that target :>json string id: the unique identifier of the snapshot **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid snapshot identifier has been provided :statuscode 404: requested snapshot can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`snapshot/6a3a2cf0b2b90ce7ae1cf0a221ed68035b686f5a/` """ # noqa def _enrich_snapshot(snapshot): s = snapshot.copy() if 'branches' in s: s['branches'] = { k: utils.enrich_object(v) if v else None for k, v in s['branches'].items() } for k, v in s['branches'].items(): if v and v['target_type'] == 'alias': if v['target'] in s['branches']: branch_alias = s['branches'][v['target']] if branch_alias: v['target_url'] = branch_alias['target_url'] else: snp = \ service.lookup_snapshot(s['id'], branches_from=v['target'], branches_count=1) if snp and v['target'] in snp['branches']: branch = snp['branches'][v['target']] branch = utils.enrich_object(branch) v['target_url'] = branch['target_url'] return s snapshot_content_max_size = get_config()['snapshot_content_max_size'] branches_from = request.GET.get('branches_from', '') branches_count = int(request.GET.get('branches_count', snapshot_content_max_size)) target_types = request.GET.get('target_types', None) target_types = target_types.split(',') if target_types else None results = api_lookup( service.lookup_snapshot, snapshot_id, branches_from, branches_count, target_types, notfound_msg='Snapshot with id {} not found.'.format(snapshot_id), enrich_fn=_enrich_snapshot) response = {'results': results, 'headers': {}} if results['next_branch'] is not None: response['headers']['link-next'] = \ reverse('api-1-snapshot', url_args={'snapshot_id': snapshot_id}, query_params={'branches_from': results['next_branch'], 'branches_count': branches_count, 'target_types': target_types}) return response diff --git a/swh/web/api/views/stat.py b/swh/web/api/views/stat.py index 5ff85de2..876e6397 100644 --- a/swh/web/api/views/stat.py +++ b/swh/web/api/views/stat.py @@ -1,47 +1,47 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route @api_route(r'/stat/counters/', 'api-1-stat-counters') @api_doc('/stat/counters/', noargs=True) def api_stats(request): """ .. http:get:: /api/1/stat/counters/ Get statistics about the content of the archive. :>json number content: current number of content objects (aka files) in the archive :>json number directory: current number of directory objects in the archive :>json number origin: current number of software origins (an origin is a "place" where code source can be found, e.g. a git repository, a tarball, ...) in the archive :>json number origin_visit: current number of visits on software origins to fill the archive :>json number person: current number of persons (code source authors or committers) in the archive :>json number release: current number of releases objects in the archive :>json number revision: current number of revision objects (aka commits) in the archive :>json number skipped_content: current number of content objects (aka files) which where not inserted in the archive :>json number snapshot: current number of snapshot objects (aka set of named branches) in the archive :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`stat/counters/` """ # noqa return service.stat_counters() diff --git a/swh/web/api/views/utils.py b/swh/web/api/views/utils.py index 00645b67..acba8ae2 100644 --- a/swh/web/api/views/utils.py +++ b/swh/web/api/views/utils.py @@ -1,73 +1,73 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from rest_framework.response import Response from rest_framework.decorators import api_view from types import GeneratorType from swh.web.common.exc import NotFoundExc from swh.web.api.apiurls import APIUrls, api_route def api_lookup(lookup_fn, *args, notfound_msg='Object not found', enrich_fn=None): """ Capture a redundant behavior of: - looking up the backend with a criteria (be it an identifier or checksum) passed to the function lookup_fn - if nothing is found, raise an NotFoundExc exception with error message notfound_msg. - Otherwise if something is returned: - either as list, map or generator, map the enrich_fn function to it and return the resulting data structure as list. - either as dict and pass to enrich_fn and return the dict enriched. Args: - lookup_fn: function expects one criteria and optional supplementary \*args. - notfound_msg: if nothing matching the criteria is found, raise NotFoundExc with this error message. - enrich_fn: Function to use to enrich the result returned by lookup_fn. Default to the identity function if not provided. - \*args: supplementary arguments to pass to lookup_fn. Raises: NotFoundExp or whatever `lookup_fn` raises. """ # noqa if enrich_fn is None: enrich_fn = (lambda x: x) res = lookup_fn(*args) if res is None: raise NotFoundExc(notfound_msg) if isinstance(res, (map, list, GeneratorType)): return [enrich_fn(x) for x in res] return enrich_fn(res) @api_view(['GET', 'HEAD']) def api_home(request): return Response({}, template_name='api/api.html') APIUrls.add_url_pattern(r'^$', api_home, view_name='api-1-homepage') @api_route(r'/', 'api-1-endpoints') def api_endpoints(request): """Display the list of opened api endpoints. """ routes = APIUrls.get_app_endpoints().copy() for route, doc in routes.items(): doc['doc_intro'] = doc['docstring'].split('\n\n')[0] # Return a list of routes with consistent ordering env = { 'doc_routes': sorted(routes.items()) } return Response(env, template_name="api/endpoints.html") diff --git a/swh/web/api/views/vault.py b/swh/web/api/views/vault.py index 80f7d565..07735139 100644 --- a/swh/web/api/views/vault.py +++ b/swh/web/api/views/vault.py @@ -1,214 +1,214 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.http import HttpResponse from django.views.decorators.cache import never_cache from swh.model import hashutil from swh.web.common import service, query from swh.web.common.utils import reverse from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup # XXX: a bit spaghetti. Would be better with class-based views. def _dispatch_cook_progress(request, obj_type, obj_id): hex_id = hashutil.hash_to_hex(obj_id) object_name = obj_type.split('_')[0].title() if request.method == 'GET': return api_lookup( service.vault_progress, obj_type, obj_id, notfound_msg=("{} '{}' was never requested." .format(object_name, hex_id))) elif request.method == 'POST': email = request.POST.get('email', request.GET.get('email', None)) return api_lookup( service.vault_cook, obj_type, obj_id, email, notfound_msg=("{} '{}' not found." .format(object_name, hex_id))) @api_route(r'/vault/directory/(?P[0-9a-f]+)/', 'api-1-vault-cook-directory', methods=['GET', 'POST'], checksum_args=['dir_id'], throttle_scope='swh_vault_cooking') @never_cache @api_doc('/vault/directory/') def api_vault_cook_directory(request, dir_id): """ .. http:get:: /api/1/vault/directory/(dir_id)/ .. http:post:: /api/1/vault/directory/(dir_id)/ Request the cooking of an archive for a directory or check its cooking status. That endpoint enables to create a vault cooking task for a directory through a POST request or check the status of a previously created one through a GET request. Once the cooking task has been executed, the resulting archive can be downloaded using the dedicated endpoint :http:get:`/api/1/vault/directory/(dir_id)/raw/`. Then to extract the cooked directory in the current one, use:: $ tar xvf path/to/directory.tar.gz :param string dir_id: the directory's sha1 identifier :query string email: e-mail to notify when the archive is ready :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>json string fetch_url: the url from which to download the archive once it has been cooked (see :http:get:`/api/1/vault/directory/(dir_id)/raw/`) :>json string obj_type: the type of object to cook (directory or revision) :>json string progress_message: message describing the cooking task progress :>json number id: the cooking task id :>json string status: the cooking task status (either **new**, **pending**, **done** or **failed**) :>json string obj_id: the identifier of the object to cook **Allowed HTTP Methods:** :http:method:`get`, :http:method:`post`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid directory identifier has been provided :statuscode 404: requested directory can not be found in the archive """ # noqa _, obj_id = query.parse_hash_with_algorithms_or_throws( dir_id, ['sha1'], 'Only sha1_git is supported.') res = _dispatch_cook_progress(request, 'directory', obj_id) res['fetch_url'] = reverse('api-1-vault-fetch-directory', url_args={'dir_id': dir_id}) return res @api_route(r'/vault/directory/(?P[0-9a-f]+)/raw/', 'api-1-vault-fetch-directory', checksum_args=['dir_id']) @api_doc('/vault/directory/raw/', handle_response=True) def api_vault_fetch_directory(request, dir_id): """ .. http:get:: /api/1/vault/directory/(dir_id)/raw/ Fetch the cooked archive for a directory. See :http:get:`/api/1/vault/directory/(dir_id)/` to get more details on directory cooking. :param string dir_id: the directory's sha1 identifier :resheader Content-Type: application/octet-stream **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid directory identifier has been provided :statuscode 404: requested directory can not be found in the archive """ # noqa _, obj_id = query.parse_hash_with_algorithms_or_throws( dir_id, ['sha1'], 'Only sha1_git is supported.') res = api_lookup( service.vault_fetch, 'directory', obj_id, notfound_msg="Directory with ID '{}' not found.".format(dir_id)) fname = '{}.tar.gz'.format(dir_id) response = HttpResponse(res, content_type='application/gzip') response['Content-disposition'] = 'attachment; filename={}'.format(fname) return response @api_route(r'/vault/revision/(?P[0-9a-f]+)/gitfast/', 'api-1-vault-cook-revision_gitfast', methods=['GET', 'POST'], checksum_args=['rev_id'], throttle_scope='swh_vault_cooking') @never_cache @api_doc('/vault/revision/gitfast/') def api_vault_cook_revision_gitfast(request, rev_id): """ .. http:get:: /api/1/vault/revision/(rev_id)/gitfast/ .. http:post:: /api/1/vault/revision/(rev_id)/gitfast/ Request the cooking of a gitfast archive for a revision or check its cooking status. That endpoint enables to create a vault cooking task for a revision through a POST request or check the status of a previously created one through a GET request. Once the cooking task has been executed, the resulting gitfast archive can be downloaded using the dedicated endpoint :http:get:`/api/1/vault/revision/(rev_id)/gitfast/raw/`. Then to import the revision in the current directory, use:: $ git init $ zcat path/to/revision.gitfast.gz | git fast-import $ git checkout HEAD :param string rev_id: the revision's sha1 identifier :query string email: e-mail to notify when the gitfast archive is ready :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :>json string fetch_url: the url from which to download the archive once it has been cooked (see :http:get:`/api/1/vault/revision/(rev_id)/gitfast/raw/`) :>json string obj_type: the type of object to cook (directory or revision) :>json string progress_message: message describing the cooking task progress :>json number id: the cooking task id :>json string status: the cooking task status (new/pending/done/failed) :>json string obj_id: the identifier of the object to cook **Allowed HTTP Methods:** :http:method:`get`, :http:method:`post`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid revision identifier has been provided :statuscode 404: requested revision can not be found in the archive """ # noqa _, obj_id = query.parse_hash_with_algorithms_or_throws( rev_id, ['sha1'], 'Only sha1_git is supported.') res = _dispatch_cook_progress(request, 'revision_gitfast', obj_id) res['fetch_url'] = reverse('api-1-vault-fetch-revision_gitfast', url_args={'rev_id': rev_id}) return res @api_route(r'/vault/revision/(?P[0-9a-f]+)/gitfast/raw/', 'api-1-vault-fetch-revision_gitfast', checksum_args=['rev_id']) @api_doc('/vault/revision/gitfast/raw/', handle_response=True) def api_vault_fetch_revision_gitfast(request, rev_id): """ .. http:get:: /api/1/vault/revision/(rev_id)/gitfast/raw/ Fetch the cooked gitfast archive for a revision. See :http:get:`/api/1/vault/revision/(rev_id)/gitfast/` to get more details on directory cooking. :param string rev_id: the revision's sha1 identifier :resheader Content-Type: application/octet-stream **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options` :statuscode 200: no error :statuscode 400: an invalid revision identifier has been provided :statuscode 404: requested revision can not be found in the archive """ # noqa _, obj_id = query.parse_hash_with_algorithms_or_throws( rev_id, ['sha1'], 'Only sha1_git is supported.') res = api_lookup( service.vault_fetch, 'revision_gitfast', obj_id, notfound_msg="Revision with ID '{}' not found.".format(rev_id)) fname = '{}.gitfast.gz'.format(rev_id) response = HttpResponse(res, content_type='application/gzip') response['Content-disposition'] = 'attachment; filename={}'.format(fname) return response diff --git a/swh/web/assets/config/bootstrap-pre-customize.scss b/swh/web/assets/config/bootstrap-pre-customize.scss index 3f52cabf..779e78a9 100644 --- a/swh/web/assets/config/bootstrap-pre-customize.scss +++ b/swh/web/assets/config/bootstrap-pre-customize.scss @@ -1,40 +1,40 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // override some global boostrap sass variables before generating stylesheets // global text colors and fonts $body-color: rgba(0, 0, 0, 0.55); $font-family-sans-serif: "Alegreya Sans", sans-serif !important; $link-color: rgba(0, 0, 0, 0.75); $code-color: #c7254e; // headings $headings-line-height: 1.1; $headings-color: #e20026; $headings-font-family: "Alegreya Sans", sans-serif !important; // remove the ugly box shadow from bootstrap 4.x $input-btn-focus-width: 0; // dropdown menu padding $dropdown-padding-y: 0.25rem; $dropdown-item-padding-x: 0; $dropdown-item-padding-y: 0; // card header padding $card-spacer-y: 0.5rem; // nav pills colors $nav-pills-link-active-color: rgba(0, 0, 0, 0.55); $nav-pills-link-active-bg: #f2f4f5; // table cell padding $table-cell-padding: 0.4rem; // remove container padding $grid-gutter-width: 0; diff --git a/swh/web/assets/src/bundles/admin/origin-save.js b/swh/web/assets/src/bundles/admin/origin-save.js index b33d70a8..3f60c6df 100644 --- a/swh/web/assets/src/bundles/admin/origin-save.js +++ b/swh/web/assets/src/bundles/admin/origin-save.js @@ -1,313 +1,313 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {handleFetchError, csrfPost} from 'utils/functions'; let authorizedOriginTable; let unauthorizedOriginTable; let pendingSaveRequestsTable; let acceptedSaveRequestsTable; let rejectedSaveRequestsTable; function enableRowSelection(tableSel) { $(`${tableSel} tbody`).on('click', 'tr', function() { if ($(this).hasClass('selected')) { $(this).removeClass('selected'); } else { $(`${tableSel} tr.selected`).removeClass('selected'); $(this).addClass('selected'); } }); } export function initOriginSaveAdmin() { $(document).ready(() => { $.fn.dataTable.ext.errMode = 'throw'; authorizedOriginTable = $('#swh-authorized-origin-urls').DataTable({ serverSide: true, ajax: Urls.admin_origin_save_authorized_urls_list(), columns: [{data: 'url', name: 'url'}], scrollY: '50vh', scrollCollapse: true, info: false }); enableRowSelection('#swh-authorized-origin-urls'); unauthorizedOriginTable = $('#swh-unauthorized-origin-urls').DataTable({ serverSide: true, ajax: Urls.admin_origin_save_unauthorized_urls_list(), columns: [{data: 'url', name: 'url'}], scrollY: '50vh', scrollCollapse: true, info: false }); enableRowSelection('#swh-unauthorized-origin-urls'); let columnsData = [ { data: 'id', name: 'id', visible: false, searchable: false }, { data: 'save_request_date', name: 'request_date', render: (data, type, row) => { if (type === 'display') { let date = new Date(data); return date.toLocaleString(); } return data; } }, { data: 'origin_type', name: 'origin_type' }, { data: 'origin_url', name: 'origin_url', render: (data, type, row) => { if (type === 'display') { const sanitizedURL = $.fn.dataTable.render.text().display(data); return `${sanitizedURL}`; } return data; } } ]; pendingSaveRequestsTable = $('#swh-origin-save-pending-requests').DataTable({ serverSide: true, ajax: Urls.browse_origin_save_requests_list('pending'), searchDelay: 1000, columns: columnsData, scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); enableRowSelection('#swh-origin-save-pending-requests'); rejectedSaveRequestsTable = $('#swh-origin-save-rejected-requests').DataTable({ serverSide: true, ajax: Urls.browse_origin_save_requests_list('rejected'), searchDelay: 1000, columns: columnsData, scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); enableRowSelection('#swh-origin-save-rejected-requests'); columnsData.push({ data: 'save_task_status', name: 'save_task_status', render: (data, type, row) => { if (data === 'succeed') { let browseOriginUrl = Urls.browse_origin(row.origin_url); return `${data}`; } return data; } }); acceptedSaveRequestsTable = $('#swh-origin-save-accepted-requests').DataTable({ serverSide: true, ajax: Urls.browse_origin_save_requests_list('accepted'), searchDelay: 1000, columns: columnsData, scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); enableRowSelection('#swh-origin-save-accepted-requests'); $('#swh-origin-save-requests-nav-item').on('shown.bs.tab', () => { pendingSaveRequestsTable.draw(); }); $('#swh-origin-save-url-filters-nav-item').on('shown.bs.tab', () => { authorizedOriginTable.draw(); }); $('#swh-authorized-origins-tab').on('shown.bs.tab', () => { authorizedOriginTable.draw(); }); $('#swh-unauthorized-origins-tab').on('shown.bs.tab', () => { unauthorizedOriginTable.draw(); }); $('#swh-save-requests-pending-tab').on('shown.bs.tab', () => { pendingSaveRequestsTable.draw(); }); $('#swh-save-requests-accepted-tab').on('shown.bs.tab', () => { acceptedSaveRequestsTable.draw(); }); $('#swh-save-requests-rejected-tab').on('shown.bs.tab', () => { rejectedSaveRequestsTable.draw(); }); $('#swh-save-requests-pending-tab').click(() => { pendingSaveRequestsTable.ajax.reload(null, false); }); $('#swh-save-requests-accepted-tab').click(() => { acceptedSaveRequestsTable.ajax.reload(null, false); }); $('#swh-save-requests-rejected-tab').click(() => { rejectedSaveRequestsTable.ajax.reload(null, false); }); }); } export function addAuthorizedOriginUrl() { let originUrl = $('#swh-authorized-url-prefix').val(); let addOriginUrl = Urls.admin_origin_save_add_authorized_url(originUrl); csrfPost(addOriginUrl) .then(handleFetchError) .then(() => { authorizedOriginTable.row.add({'url': originUrl}).draw(); }) .catch(response => { swh.webapp.showModalMessage( 'Duplicated origin url prefix', 'The provided origin url prefix is already registered in the authorized list.'); }); } export function removeAuthorizedOriginUrl() { let originUrl = $('#swh-authorized-origin-urls tr.selected').text(); if (originUrl) { let removeOriginUrl = Urls.admin_origin_save_remove_authorized_url(originUrl); csrfPost(removeOriginUrl) .then(handleFetchError) .then(() => { authorizedOriginTable.row('.selected').remove().draw(); }) .catch(() => {}); } } export function addUnauthorizedOriginUrl() { let originUrl = $('#swh-unauthorized-url-prefix').val(); let addOriginUrl = Urls.admin_origin_save_add_unauthorized_url(originUrl); csrfPost(addOriginUrl) .then(handleFetchError) .then(() => { unauthorizedOriginTable.row.add({'url': originUrl}).draw(); }) .catch(() => { swh.webapp.showModalMessage( 'Duplicated origin url prefix', 'The provided origin url prefix is already registered in the unauthorized list.'); }); } export function removeUnauthorizedOriginUrl() { let originUrl = $('#swh-unauthorized-origin-urls tr.selected').text(); if (originUrl) { let removeOriginUrl = Urls.admin_origin_save_remove_unauthorized_url(originUrl); csrfPost(removeOriginUrl) .then(handleFetchError) .then(() => { unauthorizedOriginTable.row('.selected').remove().draw(); }) .catch(() => {}); } } export function acceptOriginSaveRequest() { let selectedRow = pendingSaveRequestsTable.row('.selected'); if (selectedRow.length) { let acceptOriginSaveRequestCallback = () => { let rowData = selectedRow.data(); let acceptSaveRequestUrl = Urls.admin_origin_save_request_accept(rowData['origin_type'], rowData['origin_url']); csrfPost(acceptSaveRequestUrl) .then(() => { pendingSaveRequestsTable.ajax.reload(null, false); }); }; swh.webapp.showModalConfirm( 'Accept origin save request ?', 'Are you sure to accept this origin save request ?', acceptOriginSaveRequestCallback); } } export function rejectOriginSaveRequest() { let selectedRow = pendingSaveRequestsTable.row('.selected'); if (selectedRow.length) { let rejectOriginSaveRequestCallback = () => { let rowData = selectedRow.data(); let rejectSaveRequestUrl = Urls.admin_origin_save_request_reject(rowData['origin_type'], rowData['origin_url']); csrfPost(rejectSaveRequestUrl) .then(() => { pendingSaveRequestsTable.ajax.reload(null, false); }); }; swh.webapp.showModalConfirm( 'Reject origin save request ?', 'Are you sure to reject this origin save request ?', rejectOriginSaveRequestCallback); } } function removeOriginSaveRequest(requestTable) { let selectedRow = requestTable.row('.selected'); if (selectedRow.length) { let requestId = selectedRow.data()['id']; let removeOriginSaveRequestCallback = () => { let removeSaveRequestUrl = Urls.admin_origin_save_request_remove(requestId); csrfPost(removeSaveRequestUrl) .then(() => { requestTable.ajax.reload(null, false); }); }; swh.webapp.showModalConfirm( 'Remove origin save request ?', 'Are you sure to remove this origin save request ?', removeOriginSaveRequestCallback); } } export function removePendingOriginSaveRequest() { removeOriginSaveRequest(pendingSaveRequestsTable); } export function removeAcceptedOriginSaveRequest() { removeOriginSaveRequest(acceptedSaveRequestsTable); } export function removeRejectedOriginSaveRequest() { removeOriginSaveRequest(rejectedSaveRequestsTable); } diff --git a/swh/web/assets/src/bundles/browse/browse-utils.js b/swh/web/assets/src/bundles/browse/browse-utils.js index 3cce9854..0dd6f8fa 100644 --- a/swh/web/assets/src/bundles/browse/browse-utils.js +++ b/swh/web/assets/src/bundles/browse/browse-utils.js @@ -1,72 +1,72 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {BREAKPOINT_SM} from 'utils/constants'; $(document).ready(() => { $('.dropdown-submenu a.dropdown-item').on('click', e => { $(e.target).next('div').toggle(); if ($(e.target).next('div').css('display') !== 'none') { $(e.target).focus(); } else { $(e.target).blur(); } e.stopPropagation(); e.preventDefault(); }); $('.swh-popover-toggler').popover({ boundary: 'viewport', container: 'body', html: true, placement: function() { const width = $(window).width(); if (width < BREAKPOINT_SM) { return 'top'; } else { return 'right'; } }, template: ``, content: function() { var content = $(this).attr('data-popover-content'); return $(content).children('.popover-body').remove().html(); }, title: function() { var title = $(this).attr('data-popover-content'); return $(title).children('.popover-heading').html(); }, offset: '50vh', sanitize: false }); $('.swh-vault-menu a.dropdown-item').on('click', e => { $('.swh-popover-toggler').popover('hide'); }); $('.swh-popover-toggler').on('show.bs.popover', (e) => { $(`.swh-popover-toggler:not(#${e.currentTarget.id})`).popover('hide'); $('.swh-vault-menu .dropdown-menu').hide(); }); $('.swh-actions-dropdown').on('hide.bs.dropdown', () => { $('.swh-vault-menu .dropdown-menu').hide(); $('.swh-popover-toggler').popover('hide'); }); $('body').on('click', e => { if ($(e.target).parents('.swh-popover').length) { e.stopPropagation(); } }); }); diff --git a/swh/web/assets/src/bundles/browse/browse.css b/swh/web/assets/src/bundles/browse/browse.css index 5ee06ff4..f9ac5df3 100644 --- a/swh/web/assets/src/bundles/browse/browse.css +++ b/swh/web/assets/src/bundles/browse/browse.css @@ -1,145 +1,145 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ .swh-browse-nav li a { border-radius: 4px; } .scrollable-menu { max-height: 180px; overflow-x: hidden; } .swh-corner-ribbon { width: 200px; background: #fecd1b; color: #e20026; position: absolute; text-align: center; line-height: 50px; letter-spacing: 1px; box-shadow: 0 0 3px rgba(0, 0, 0, 0.3); top: 55px; right: -50px; left: auto; transform: rotate(45deg); z-index: 2000; } .swh-loading { display: none; text-align: center; margin-top: 10px; } .swh-loading.show { display: block; } .swh-metadata-table-row { border-top: 1px solid #ddd !important; } .swh-metadata-table-key { min-width: 200px; max-width: 200px; width: 200px; } .swh-metadata-table-value pre { white-space: pre-wrap; } .swh-table-cell-text-overflow { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .swh-directory-table { margin-bottom: 0; } .swh-directory-table td { border-top: 1px solid #ddd !important; } .swh-title-color { color: #e20026; } .swh-log-entry-message { min-width: 440px; max-width: 440px; width: 440px; } .swh-popover { max-height: 50vh; overflow-y: auto; overflow-x: auto; padding: 0; padding-right: 1.4em; } @media screen and (min-width: 768px) { .swh-popover { max-width: 80vh; } } .swh-search-pagination { margin-top: 5px; } .ui-slideouttab-panel { z-index: 30000; } #swh-identifiers { width: 70vw; top: 0; border: 1px solid #e20026; } #swh-identifiers .handle { background-color: #e20026; border: 1px solid #e20026; color: white; padding-top: 0.1em; padding-bottom: 0.1em; } #swh-identifiers-content { height: 100%; overflow: auto; } .swh-empty-snapshot { white-space: pre-line; } td.swh-branch-name { max-width: 300px; } td.swh-branch-message { min-width: 500px; max-width: 500px; } td.swh-branch-date { min-width: 250px; } @media screen and (max-width: 600px) { .swh-corner-ribbon { line-height: 30px; top: 53px; right: -65px; } } diff --git a/swh/web/assets/src/bundles/browse/origin-search.js b/swh/web/assets/src/bundles/browse/origin-search.js index 8f40ee96..093e2fd2 100644 --- a/swh/web/assets/src/bundles/browse/origin-search.js +++ b/swh/web/assets/src/bundles/browse/origin-search.js @@ -1,238 +1,238 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {heapsPermute} from 'utils/heaps-permute'; import {handleFetchError} from 'utils/functions'; let originPatterns; let perPage = 100; let limit = perPage * 2; let offset = 0; let currentData = null; let inSearch = false; function fixTableRowsStyle() { setTimeout(() => { $('#origin-search-results tbody tr').removeAttr('style'); }); } function clearOriginSearchResultsTable() { $('#origin-search-results tbody tr').remove(); } function populateOriginSearchResultsTable(data, offset) { let localOffset = offset % limit; if (data.length > 0) { $('#swh-origin-search-results').show(); $('#swh-no-result').hide(); clearOriginSearchResultsTable(); let table = $('#origin-search-results tbody'); for (let i = localOffset; i < localOffset + perPage && i < data.length; ++i) { let elem = data[i]; let browseUrl = Urls.browse_origin(elem.url); let tableRow = ``; tableRow += `${elem.type}`; tableRow += `${encodeURI(elem.url)}`; tableRow += ``; tableRow += ''; table.append(tableRow); // get async latest visit snapshot and update visit status icon let latestSnapshotUrl = Urls.browse_origin_latest_snapshot(elem.id); fetch(latestSnapshotUrl) .then(response => response.json()) .then(data => { let originId = elem.id; $(`#visit-status-origin-${originId}`).children().remove(); if (data) { $(`#visit-status-origin-${originId}`).append(''); } else { $(`#visit-status-origin-${originId}`).append(''); if ($('#swh-filter-empty-visits').prop('checked')) { $(`#origin-${originId}`).remove(); } } }); } fixTableRowsStyle(); } else { $('#swh-origin-search-results').hide(); $('#swh-no-result').text('No origins matching the search criteria were found.'); $('#swh-no-result').show(); } if (data.length - localOffset < perPage || (data.length < limit && (localOffset + perPage) === data.length)) { $('#origins-next-results-button').addClass('disabled'); } else { $('#origins-next-results-button').removeClass('disabled'); } if (offset > 0) { $('#origins-prev-results-button').removeClass('disabled'); } else { $('#origins-prev-results-button').addClass('disabled'); } inSearch = false; setTimeout(() => { window.scrollTo(0, 0); }); } function escapeStringRegexp(str) { let matchOperatorsRe = /[|\\{}()[\]^$+*?.]/g; return str.replace(matchOperatorsRe, '\\\\\\$&'); } function searchOrigins(patterns, limit, searchOffset, offset) { let baseSearchUrl; let searchMetadata = $('#swh-search-origin-metadata').prop('checked'); if (searchMetadata) { baseSearchUrl = Urls.api_1_origin_metadata_search() + `?fulltext=${patterns}`; } else { originPatterns = patterns; let patternsArray = patterns.trim().replace(/\s+/g, ' ').split(' '); for (let i = 0; i < patternsArray.length; ++i) { patternsArray[i] = escapeStringRegexp(patternsArray[i]); } // url length must be less than 4096 for modern browsers // assuming average word length, 6 is max patternArray.length if (patternsArray.length < 7) { let patternsPermut = []; heapsPermute(patternsArray, p => patternsPermut.push(p.join('.*'))); let regex = patternsPermut.join('|'); baseSearchUrl = Urls.browse_origin_search(regex) + `?regexp=true`; } else { baseSearchUrl = Urls.browse_origin_search(patternsArray.join('.*')) + `?regexp=true`; } } let withVisit = $('#swh-search-origins-with-visit').prop('checked'); let searchUrl = baseSearchUrl + `&limit=${limit}&offset=${searchOffset}&with_visit=${withVisit}`; clearOriginSearchResultsTable(); $('.swh-loading').addClass('show'); fetch(searchUrl) .then(handleFetchError) .then(response => response.json()) .then(data => { currentData = data; $('.swh-loading').removeClass('show'); populateOriginSearchResultsTable(data, offset); }) .catch(response => { $('.swh-loading').removeClass('show'); inSearch = false; $('#swh-origin-search-results').hide(); $('#swh-no-result').text(`Error ${response.status}: ${response.statusText}`); $('#swh-no-result').show(); }); } function doSearch() { $('#swh-no-result').hide(); let patterns = $('#origins-url-patterns').val(); offset = 0; inSearch = true; // first try to resolve a swh persistent identifier let resolvePidUrl = Urls.api_1_resolve_swh_pid(patterns); fetch(resolvePidUrl) .then(handleFetchError) .then(response => response.json()) .then(data => { // pid has been successfully resolved, // so redirect to browse page window.location = data.browse_url; }) .catch(response => { // pid resolving failed if (patterns.startsWith('swh:')) { // display a useful error message if the input // looks like a swh pid response.json().then(data => { $('#swh-origin-search-results').hide(); $('.swh-search-pagination').hide(); $('#swh-no-result').text(data.reason); $('#swh-no-result').show(); }); } else { // otherwise, proceed with origins search $('#swh-origin-search-results').show(); $('.swh-search-pagination').show(); searchOrigins(patterns, limit, offset, offset); } }); } export function initOriginSearch() { $(document).ready(() => { $('#swh-search-origins').submit(event => { event.preventDefault(); let patterns = $('#origins-url-patterns').val().trim(); let withVisit = $('#swh-search-origins-with-visit').prop('checked'); let withContent = $('#swh-filter-empty-visits').prop('checked'); let searchMetadata = $('#swh-search-origin-metadata').prop('checked'); let queryParameters = '?q=' + encodeURIComponent(patterns); if (withVisit) { queryParameters += '&with_visit'; } if (withContent) { queryParameters += '&with_content'; } if (searchMetadata) { queryParameters += '&search_metadata'; } // Update the url, triggering page reload and effective search window.location.search = queryParameters; }); $('#origins-next-results-button').click(event => { if ($('#origins-next-results-button').hasClass('disabled') || inSearch) { return; } inSearch = true; offset += perPage; if (!currentData || (offset >= limit && offset % limit === 0)) { searchOrigins(originPatterns, limit, offset, offset); } else { populateOriginSearchResultsTable(currentData, offset); } event.preventDefault(); }); $('#origins-prev-results-button').click(event => { if ($('#origins-prev-results-button').hasClass('disabled') || inSearch) { return; } inSearch = true; offset -= perPage; if (!currentData || (offset > 0 && (offset + perPage) % limit === 0)) { searchOrigins(originPatterns, limit, (offset + perPage) - limit, offset); } else { populateOriginSearchResultsTable(currentData, offset); } event.preventDefault(); }); $(document).on('shown.bs.tab', 'a[data-toggle="tab"]', e => { if (e.currentTarget.text.trim() === 'Search') { fixTableRowsStyle(); } }); let urlParams = new URLSearchParams(window.location.search); let query = urlParams.get('q'); let withVisit = urlParams.has('with_visit'); let withContent = urlParams.has('with_content'); let searchMetadata = urlParams.has('search_metadata'); if (query) { $('#origins-url-patterns').val(query); $('#swh-search-origins-with-visit').prop('checked', withVisit); $('#swh-filter-empty-visits').prop('checked', withContent); $('#swh-search-origin-metadata').prop('checked', searchMetadata); doSearch(); } }); } diff --git a/swh/web/assets/src/bundles/browse/snapshot-navigation.css b/swh/web/assets/src/bundles/browse/snapshot-navigation.css index 0749135e..a59d1753 100644 --- a/swh/web/assets/src/bundles/browse/snapshot-navigation.css +++ b/swh/web/assets/src/bundles/browse/snapshot-navigation.css @@ -1,50 +1,50 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ .swh-browse-top-navigation { min-height: 43px; padding: 4px 5px 0 5px; } .swh-branches-releases { min-width: 200px; } .swh-branches-switch, .swh-releases-switch { padding: 5px 15px !important; } li.swh-branch:hover, li.swh-release:hover { background-color: #e8e8e8; } .swh-branch a:hover, .swh-release a:hover { text-decoration: none; } .swh-origin-visit-details, .swh-snapshot-details { text-align: center; } .swh-origin-visit-details ul, .swh-snapshot-details ul { list-style: none; margin: 0; padding: 0; } .swh-origin-visit-details li, .swh-snapshot-details li { display: inline-block; vertical-align: middle; margin-left: 10px; margin-right: 10px; } diff --git a/swh/web/assets/src/bundles/browse/swh-ids-utils.js b/swh/web/assets/src/bundles/browse/swh-ids-utils.js index 4e87bda8..7bc3f724 100644 --- a/swh/web/assets/src/bundles/browse/swh-ids-utils.js +++ b/swh/web/assets/src/bundles/browse/swh-ids-utils.js @@ -1,117 +1,117 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import ClipboardJS from 'clipboard'; import 'thirdparty/jquery.tabSlideOut/jquery.tabSlideOut'; import 'thirdparty/jquery.tabSlideOut/jquery.tabSlideOut.css'; import {BREAKPOINT_SM} from 'utils/constants'; export function swhIdObjectTypeToggled(event) { event.preventDefault(); $(event.target).tab('show'); } export function swhIdOptionOriginToggled(event) { event.stopPropagation(); let swhIdElt = $(event.target).closest('.swh-id-ui').find('.swh-id'); let originPart = ';origin=' + $(event.target).data('swh-origin'); let currentSwhId = swhIdElt.text(); if ($(event.target).prop('checked')) { if (currentSwhId.indexOf(originPart) === -1) { currentSwhId += originPart; } } else { currentSwhId = currentSwhId.replace(originPart, ''); } swhIdElt.text(currentSwhId); swhIdElt.attr('href', '/' + currentSwhId + '/'); } function setIdLinesPart(elt) { let swhIdElt = $(elt).closest('.swh-id-ui').find('.swh-id'); let currentSwhId = swhIdElt.text(); let lines = []; let linesPart = ';lines='; let linesRegexp = new RegExp(/L(\d+)/g); let line = linesRegexp.exec(window.location.hash); while (line) { lines.push(parseInt(line[1])); line = linesRegexp.exec(window.location.hash); } if (lines.length > 0) { linesPart += lines[0]; } if (lines.length > 1) { linesPart += '-' + lines[1]; } if ($(elt).prop('checked')) { currentSwhId = currentSwhId.replace(/;lines=\d+-*\d*/g, ''); currentSwhId += linesPart; } else { currentSwhId = currentSwhId.replace(linesPart, ''); } swhIdElt.text(currentSwhId); swhIdElt.attr('href', '/' + currentSwhId + '/'); } export function swhIdOptionLinesToggled(event) { event.stopPropagation(); if (!window.location.hash) { return; } setIdLinesPart(event.target); } $(document).ready(() => { new ClipboardJS('.btn-swh-id-copy', { text: trigger => { let swhId = $(trigger).closest('.swh-id-ui').find('.swh-id').text(); return swhId; } }); new ClipboardJS('.btn-swh-id-url-copy', { text: trigger => { let swhId = $(trigger).closest('.swh-id-ui').find('.swh-id').text(); return window.location.origin + '/' + swhId + '/'; } }); if (window.innerWidth * 0.7 > 1000) { $('#swh-identifiers').css('width', '1000px'); } let tabSlideOptions = { tabLocation: 'right', offset: function() { const width = $(window).width(); if (width < BREAKPOINT_SM) { return '250px'; } else { return '200px'; } } }; // ensure tab scrolling on small screens if (window.innerHeight < 600 || window.innerWidth < 500) { tabSlideOptions['otherOffset'] = '20px'; } // initiate the sliding identifiers tab $('#swh-identifiers').tabSlideOut(tabSlideOptions); // set the tab visible once the close animation is terminated $('#swh-identifiers').css('display', 'block'); $('.swh-id-option-origin').trigger('click'); $('.swh-id-option-lines').trigger('click'); $(window).on('hashchange', () => { setIdLinesPart('.swh-id-option-lines'); }); }); diff --git a/swh/web/assets/src/bundles/origin/visits-calendar.js b/swh/web/assets/src/bundles/origin/visits-calendar.js index 031d49b8..0ff3590a 100644 --- a/swh/web/assets/src/bundles/origin/visits-calendar.js +++ b/swh/web/assets/src/bundles/origin/visits-calendar.js @@ -1,140 +1,140 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import 'bootstrap-year-calendar-bs4'; import 'bootstrap-year-calendar-bs4/css/bootstrap-year-calendar.css'; let minSize = 15; let maxSize = 28; let currentPopover = null; let visitsByDate = {}; function closePopover() { if (currentPopover) { $(currentPopover).popover('hide'); currentPopover = null; } } // function to update the visits calendar view based on the selected year export function updateCalendar(year, filteredVisits, yearClickedCallback) { visitsByDate = {}; let maxNbVisitsByDate = 0; let minDate, maxDate; for (let i = 0; i < filteredVisits.length; ++i) { filteredVisits[i]['startDate'] = filteredVisits[i]['date']; filteredVisits[i]['endDate'] = filteredVisits[i]['startDate']; let date = new Date(filteredVisits[i]['date']); date.setHours(0, 0, 0, 0); let dateStr = date.toDateString(); if (!visitsByDate.hasOwnProperty(dateStr)) { visitsByDate[dateStr] = [filteredVisits[i]]; } else { visitsByDate[dateStr].push(filteredVisits[i]); } maxNbVisitsByDate = Math.max(maxNbVisitsByDate, visitsByDate[dateStr].length); if (i === 0) { minDate = maxDate = date; } else { if (date.getTime() < minDate.getTime()) { minDate = date; } if (date.getTime() > maxDate.getTime()) { maxDate = date; } } } closePopover(); $('#swh-visits-calendar').calendar({ dataSource: filteredVisits, style: 'custom', minDate: minDate, maxDate: maxDate, startYear: year, renderEnd: e => yearClickedCallback(e.currentYear), customDataSourceRenderer: (element, date, events) => { let dateStr = date.toDateString(); let nbVisits = visitsByDate[dateStr].length; let t = nbVisits / maxNbVisitsByDate; if (maxNbVisitsByDate === 1) { t = 0; } let size = minSize + t * (maxSize - minSize); let offsetX = (maxSize - size) / 2 - parseInt($(element).css('padding-left')); let offsetY = (maxSize - size) / 2 - parseInt($(element).css('padding-top')) + 1; let cellWrapper = $('
'); cellWrapper.css('position', 'relative'); let dayNumber = $('
'); dayNumber.text($(element).text()); let circle = $('
'); let r = 0; let g = 0; for (let i = 0; i < nbVisits; ++i) { let visit = visitsByDate[dateStr][i]; if (visit.status === 'full') { g += 255; } else if (visit.status === 'partial') { r += 255; g += 255; } else { r += 255; } } r /= nbVisits; g /= nbVisits; circle.css('background-color', 'rgba(' + r + ', ' + g + ', 0, 0.3)'); circle.css('width', size + 'px'); circle.css('height', size + 'px'); circle.css('border-radius', size + 'px'); circle.css('position', 'absolute'); circle.css('top', offsetY + 'px'); circle.css('left', offsetX + 'px'); cellWrapper.append(dayNumber); cellWrapper.append(circle); $(element)[0].innerHTML = $(cellWrapper)[0].outerHTML; }, mouseOnDay: e => { if (currentPopover !== e.element) { closePopover(); } let dateStr = e.date.toDateString(); if (visitsByDate.hasOwnProperty(dateStr)) { let visits = visitsByDate[dateStr]; let content = '
' + e.date.toDateString() + '
'; content += '
    '; for (let i = 0; i < visits.length; ++i) { let visitTime = visits[i].fmt_date.substr(visits[i].fmt_date.indexOf(',') + 2); content += '
  • ' + visitTime + '
    • '; } content += '
'; $(e.element).popover({ trigger: 'manual', container: 'body', html: true, content: content }); $(e.element).popover('show'); currentPopover = e.element; } } }); $('#swh-visits-timeline').mouseenter(() => { closePopover(); }); $('#swh-visits-list').mouseenter(() => { closePopover(); }); $('#swh-visits-calendar.calendar table td').css('width', maxSize + 'px'); $('#swh-visits-calendar.calendar table td').css('height', maxSize + 'px'); $('#swh-visits-calendar.calendar table td').css('padding', '0px'); } diff --git a/swh/web/assets/src/bundles/origin/visits-histogram.js b/swh/web/assets/src/bundles/origin/visits-histogram.js index 68799fa1..ebc747c1 100644 --- a/swh/web/assets/src/bundles/origin/visits-histogram.js +++ b/swh/web/assets/src/bundles/origin/visits-histogram.js @@ -1,337 +1,337 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // Creation of a stacked histogram with D3.js for software origin visits history // Parameters description: // - container: selector for the div that will contain the histogram // - visitsData: raw swh origin visits data // - currentYear: the visits year to display by default // - yearClickCallback: callback when the user selects a year through the histogram import * as d3 from 'd3'; export function createVisitsHistogram(container, visitsData, currentYear, yearClickCallback) { // remove previously created histogram and tooltip if any d3.select(container).select('svg').remove(); d3.select('div.d3-tooltip').remove(); // histogram size and margins let width = 1000; let height = 300; let margin = {top: 20, right: 80, bottom: 30, left: 50}; // create responsive svg let svg = d3.select(container) .attr('style', 'padding-bottom: ' + Math.ceil(height * 100 / width) + '%') .append('svg') .attr('viewBox', '0 0 ' + width + ' ' + height); // create tooltip div let tooltip = d3.select('body') .append('div') .attr('class', 'd3-tooltip') .style('opacity', 0); // update width and height without margins width = width - margin.left - margin.right; height = height - margin.top - margin.bottom; // create main svg group element let g = svg.append('g').attr('transform', 'translate(' + margin.left + ',' + margin.top + ')'); // create x scale let x = d3.scaleTime().rangeRound([0, width]); // create y scale let y = d3.scaleLinear().range([height, 0]); // create ordinal colorscale mapping visit status let colors = d3.scaleOrdinal() .domain(['full', 'partial', 'failed', 'ongoing']) .range(['#008000', '#edc344', '#ff0000', '#0000ff']); // first swh crawls were made in 2015 let startYear = 2015; // set latest display year as the current one let now = new Date(); let endYear = now.getUTCFullYear() + 1; let monthExtent = [new Date(Date.UTC(startYear, 0, 1)), new Date(Date.UTC(endYear, 0, 1))]; // create months bins based on setup extent let monthBins = d3.timeMonths(d3.timeMonth.offset(monthExtent[0], -1), monthExtent[1]); // create years bins based on setup extent let yearBins = d3.timeYears(monthExtent[0], monthExtent[1]); // set x scale domain x.domain(d3.extent(monthBins)); // use D3 histogram layout to create a function that will bin the visits by month let binByMonth = d3.histogram() .value(d => d.date) .domain(x.domain()) .thresholds(monthBins); // use D3 nest function to group the visits by status let visitsByStatus = d3.nest() .key(d => d['status']) .sortKeys(d3.ascending) .entries(visitsData); // prepare data in order to be able to stack visit statuses by month let statuses = []; let histData = []; for (let i = 0; i < monthBins.length; ++i) { histData[i] = {}; } visitsByStatus.forEach(entry => { statuses.push(entry.key); let monthsData = binByMonth(entry.values); for (let i = 0; i < monthsData.length; ++i) { histData[i]['x0'] = monthsData[i]['x0']; histData[i]['x1'] = monthsData[i]['x1']; histData[i][entry.key] = monthsData[i]; } }); // create function to stack visits statuses by month let stacked = d3.stack() .keys(statuses) .value((d, key) => d[key].length); // compute the maximum amount of visits by month let yMax = d3.max(histData, d => { let total = 0; for (let i = 0; i < statuses.length; ++i) { total += d[statuses[i]].length; } return total; }); // set y scale domain y.domain([0, yMax]); // compute ticks values for the y axis let step = 5; let yTickValues = []; for (let i = 0; i <= yMax / step; ++i) { yTickValues.push(i * step); } if (yTickValues.length === 0) { for (let i = 0; i <= yMax; ++i) { yTickValues.push(i); } } else if (yMax % step !== 0) { yTickValues.push(yMax); } // add histogram background grid g.append('g') .attr('class', 'grid') .call(d3.axisLeft(y) .tickValues(yTickValues) .tickSize(-width) .tickFormat('')); // create one fill only rectangle by displayed year // each rectangle will be made visible when hovering the mouse over a year range // user will then be able to select a year by clicking in the rectangle g.append('g') .selectAll('rect') .data(yearBins) .enter().append('rect') .attr('class', d => 'year' + d.getUTCFullYear()) .attr('fill', 'red') .attr('fill-opacity', d => d.getUTCFullYear() === currentYear ? 0.3 : 0) .attr('stroke', 'none') .attr('x', d => { let date = new Date(Date.UTC(d.getUTCFullYear(), 0, 1)); return x(date); }) .attr('y', 0) .attr('height', height) .attr('width', d => { let date = new Date(Date.UTC(d.getUTCFullYear(), 0, 1)); let yearWidth = x(d3.timeYear.offset(date, 1)) - x(date); return yearWidth; }) // mouse event callbacks used to show rectangle years // when hovering the mouse over the histograms .on('mouseover', d => { svg.selectAll('rect.year' + d.getUTCFullYear()) .attr('fill-opacity', 0.5); }) .on('mouseout', d => { svg.selectAll('rect.year' + d.getUTCFullYear()) .attr('fill-opacity', 0); svg.selectAll('rect.year' + currentYear) .attr('fill-opacity', 0.3); }) // callback to select a year after a mouse click // in a rectangle year .on('click', d => { svg.selectAll('rect.year' + currentYear) .attr('fill-opacity', 0); svg.selectAll('rect.yearoutline' + currentYear) .attr('stroke', 'none'); currentYear = d.getUTCFullYear(); svg.selectAll('rect.year' + currentYear) .attr('fill-opacity', 0.5); svg.selectAll('rect.yearoutline' + currentYear) .attr('stroke', 'black'); yearClickCallback(currentYear); }); // create the stacked histogram of visits g.append('g') .selectAll('g') .data(stacked(histData)) .enter().append('g') .attr('fill', d => colors(d.key)) .selectAll('rect') .data(d => d) .enter().append('rect') .attr('class', d => 'month' + d.data.x1.getMonth()) .attr('x', d => x(d.data.x0)) .attr('y', d => y(d[1])) .attr('height', d => y(d[0]) - y(d[1])) .attr('width', d => x(d.data.x1) - x(d.data.x0) - 1) // mouse event callbacks used to show rectangle years // but also to show tooltip when hovering the mouse // over the histogram bars .on('mouseover', d => { svg.selectAll('rect.year' + d.data.x1.getUTCFullYear()) .attr('fill-opacity', 0.5); tooltip.transition() .duration(200) .style('opacity', 1); let ds = d.data.x1.toISOString().substr(0, 7).split('-'); let tooltipText = '' + ds[1] + ' / ' + ds[0] + ':
'; for (let i = 0; i < statuses.length; ++i) { let visitStatus = statuses[i]; let nbVisits = d.data[visitStatus].length; if (nbVisits === 0) continue; tooltipText += nbVisits + ' ' + visitStatus + ' visits'; if (i !== statuses.length - 1) tooltipText += '
'; } tooltip.html(tooltipText) .style('left', d3.event.pageX + 15 + 'px') .style('top', d3.event.pageY + 'px'); }) .on('mouseout', d => { svg.selectAll('rect.year' + d.data.x1.getUTCFullYear()) .attr('fill-opacity', 0); svg.selectAll('rect.year' + currentYear) .attr('fill-opacity', 0.3); tooltip.transition() .duration(500) .style('opacity', 0); }) .on('mousemove', () => { tooltip.style('left', d3.event.pageX + 15 + 'px') .style('top', d3.event.pageY + 'px'); }) // callback to select a year after a mouse click // inside a histogram bar .on('click', d => { svg.selectAll('rect.year' + currentYear) .attr('fill-opacity', 0); svg.selectAll('rect.yearoutline' + currentYear) .attr('stroke', 'none'); currentYear = d.data.x1.getUTCFullYear(); svg.selectAll('rect.year' + currentYear) .attr('fill-opacity', 0.5); svg.selectAll('rect.yearoutline' + currentYear) .attr('stroke', 'black'); yearClickCallback(currentYear); }); // create one stroke only rectangle by displayed year // that will be displayed on top of the histogram when the user has selected a year g.append('g') .selectAll('rect') .data(yearBins) .enter().append('rect') .attr('class', d => 'yearoutline' + d.getUTCFullYear()) .attr('fill', 'none') .attr('stroke', d => d.getUTCFullYear() === currentYear ? 'black' : 'none') .attr('x', d => { let date = new Date(Date.UTC(d.getUTCFullYear(), 0, 1)); return x(date); }) .attr('y', 0) .attr('height', height) .attr('width', d => { let date = new Date(Date.UTC(d.getUTCFullYear(), 0, 1)); let yearWidth = x(d3.timeYear.offset(date, 1)) - x(date); return yearWidth; }); // add x axis with a tick for every 1st day of each year let xAxis = g.append('g') .attr('class', 'axis') .attr('transform', 'translate(0,' + height + ')') .call( d3.axisBottom(x) .ticks(d3.timeYear.every(1)) .tickFormat(d => d.getUTCFullYear()) ); // shift tick labels in order to display them at the middle // of each year range xAxis.selectAll('text') .attr('transform', d => { let year = d.getUTCFullYear(); let date = new Date(Date.UTC(year, 0, 1)); let yearWidth = x(d3.timeYear.offset(date, 1)) - x(date); return 'translate(' + -yearWidth / 2 + ', 0)'; }); // add y axis for the number of visits g.append('g') .attr('class', 'axis') .call(d3.axisLeft(y).tickValues(yTickValues)); // add legend for visit statuses let legendGroup = g.append('g') .attr('font-family', 'sans-serif') .attr('font-size', 10) .attr('text-anchor', 'end'); legendGroup.append('text') .attr('x', width + margin.right - 5) .attr('y', 9.5) .attr('dy', '0.32em') .text('visit status:'); let legend = legendGroup.selectAll('g') .data(statuses.slice().reverse()) .enter().append('g') .attr('transform', (d, i) => 'translate(0,' + (i + 1) * 20 + ')'); legend.append('rect') .attr('x', width + 2 * margin.right / 3) .attr('width', 19) .attr('height', 19) .attr('fill', colors); legend.append('text') .attr('x', width + 2 * margin.right / 3 - 5) .attr('y', 9.5) .attr('dy', '0.32em') .text(d => d); // add text label for the y axis g.append('text') .attr('transform', 'rotate(-90)') .attr('y', -margin.left) .attr('x', -(height / 2)) .attr('dy', '1em') .style('text-anchor', 'middle') .text('Number of visits'); } diff --git a/swh/web/assets/src/bundles/revision/log-utils.js b/swh/web/assets/src/bundles/revision/log-utils.js index d97c3ba9..a2b06b95 100644 --- a/swh/web/assets/src/bundles/revision/log-utils.js +++ b/swh/web/assets/src/bundles/revision/log-utils.js @@ -1,20 +1,27 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + export function revsOrderingTypeClicked(event) { let urlParams = new URLSearchParams(window.location.search); let orderingType = $(event.target).val(); if (orderingType) { urlParams.set('revs_ordering', $(event.target).val()); } else if (urlParams.has('revs_ordering')) { urlParams.delete('revs_ordering'); } window.location.search = urlParams.toString(); } export function initRevisionsLog() { $(document).ready(() => { let urlParams = new URLSearchParams(window.location.search); let revsOrderingType = urlParams.get('revs_ordering'); if (revsOrderingType) { $(`:input[value="${revsOrderingType}"]`).prop('checked', true); } }); } diff --git a/swh/web/assets/src/bundles/vault/vault-ui.js b/swh/web/assets/src/bundles/vault/vault-ui.js index 5c1bfc25..d08d2302 100644 --- a/swh/web/assets/src/bundles/vault/vault-ui.js +++ b/swh/web/assets/src/bundles/vault/vault-ui.js @@ -1,252 +1,252 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {handleFetchError, handleFetchErrors, csrfPost} from 'utils/functions'; let progress = `
;`; let pollingInterval = 5000; let checkVaultId; function updateProgressBar(progressBar, cookingTask) { if (cookingTask.status === 'new') { progressBar.css('background-color', 'rgba(128, 128, 128, 0.5)'); } else if (cookingTask.status === 'pending') { progressBar.css('background-color', 'rgba(0, 0, 255, 0.5)'); } else if (cookingTask.status === 'done') { progressBar.css('background-color', '#5cb85c'); } else if (cookingTask.status === 'failed') { progressBar.css('background-color', 'rgba(255, 0, 0, 0.5)'); progressBar.css('background-image', 'none'); } progressBar.text(cookingTask.progress_message || cookingTask.status); if (cookingTask.status === 'new' || cookingTask.status === 'pending') { progressBar.addClass('progress-bar-animated'); } else { progressBar.removeClass('progress-bar-striped'); } } let recookTask; // called when the user wants to download a cooked archive export function fetchCookedObject(fetchUrl) { recookTask = null; // first, check if the link is still available from the vault fetch(fetchUrl) .then(response => { // link is still alive, proceed to download if (response.ok) { $('#vault-fetch-iframe').attr('src', fetchUrl); // link is dead } else { // get the associated cooking task let vaultCookingTasks = JSON.parse(localStorage.getItem('swh-vault-cooking-tasks')); for (let i = 0; i < vaultCookingTasks.length; ++i) { if (vaultCookingTasks[i].fetch_url === fetchUrl) { recookTask = vaultCookingTasks[i]; break; } } // display a modal asking the user if he wants to recook the archive $('#vault-recook-object-modal').modal('show'); } }); } // called when the user wants to recook an archive // for which the download link is not available anymore export function recookObject() { if (recookTask) { // stop cooking tasks status polling clearTimeout(checkVaultId); // build cook request url let cookingUrl; if (recookTask.object_type === 'directory') { cookingUrl = Urls.api_1_vault_cook_directory(recookTask.object_id); } else { cookingUrl = Urls.api_1_vault_cook_revision_gitfast(recookTask.object_id); } if (recookTask.email) { cookingUrl += '?email=' + recookTask.email; } // request archive cooking csrfPost(cookingUrl) .then(handleFetchError) .then(() => { // update task status recookTask.status = 'new'; let vaultCookingTasks = JSON.parse(localStorage.getItem('swh-vault-cooking-tasks')); for (let i = 0; i < vaultCookingTasks.length; ++i) { if (vaultCookingTasks[i].object_id === recookTask.object_id) { vaultCookingTasks[i] = recookTask; break; } } // save updated tasks to local storage localStorage.setItem('swh-vault-cooking-tasks', JSON.stringify(vaultCookingTasks)); // restart cooking tasks status polling checkVaultCookingTasks(); // hide recook archive modal $('#vault-recook-object-modal').modal('hide'); }) // something went wrong .catch(() => { checkVaultCookingTasks(); $('#vault-recook-object-modal').modal('hide'); }); } } function checkVaultCookingTasks() { let vaultCookingTasks = JSON.parse(localStorage.getItem('swh-vault-cooking-tasks')); if (!vaultCookingTasks || vaultCookingTasks.length === 0) { $('.swh-vault-table tbody tr').remove(); checkVaultId = setTimeout(checkVaultCookingTasks, pollingInterval); return; } let cookingTaskRequests = []; let tasks = {}; let currentObjectIds = []; for (let i = 0; i < vaultCookingTasks.length; ++i) { let cookingTask = vaultCookingTasks[i]; currentObjectIds.push(cookingTask.object_id); tasks[cookingTask.object_id] = cookingTask; let cookingUrl; if (cookingTask.object_type === 'directory') { cookingUrl = Urls.api_1_vault_cook_directory(cookingTask.object_id); } else { cookingUrl = Urls.api_1_vault_cook_revision_gitfast(cookingTask.object_id); } if (cookingTask.status !== 'done' && cookingTask.status !== 'failed') { cookingTaskRequests.push(fetch(cookingUrl)); } } $('.swh-vault-table tbody tr').each((i, row) => { let objectId = $(row).find('.vault-object-id').data('object-id'); if ($.inArray(objectId, currentObjectIds) === -1) { $(row).remove(); } }); Promise.all(cookingTaskRequests) .then(handleFetchErrors) .then(responses => Promise.all(responses.map(r => r.json()))) .then(cookingTasks => { let table = $('#vault-cooking-tasks tbody'); for (let i = 0; i < cookingTasks.length; ++i) { let cookingTask = tasks[cookingTasks[i].obj_id]; cookingTask.status = cookingTasks[i].status; cookingTask.fetch_url = cookingTasks[i].fetch_url; cookingTask.progress_message = cookingTasks[i].progress_message; } for (let i = 0; i < vaultCookingTasks.length; ++i) { let cookingTask = vaultCookingTasks[i]; let rowTask = $('#vault-task-' + cookingTask.object_id); let downloadLinkWait = 'Waiting for download link to be available'; if (!rowTask.length) { let browseUrl; if (cookingTask.object_type === 'directory') { browseUrl = Urls.browse_directory(cookingTask.object_id); } else { browseUrl = Urls.browse_revision(cookingTask.object_id); } let progressBar = $.parseHTML(progress)[0]; let progressBarContent = $(progressBar).find('.progress-bar'); updateProgressBar(progressBarContent, cookingTask); let tableRow; if (cookingTask.object_type === 'directory') { tableRow = ``; } else { tableRow = ``; } tableRow += ''; tableRow += `${cookingTask.object_type}`; tableRow += `${cookingTask.object_id}`; tableRow += `${progressBar.outerHTML}`; let downloadLink = downloadLinkWait; if (cookingTask.status === 'done') { downloadLink = `'; } else if (cookingTask.status === 'failed') { downloadLink = ''; } tableRow += `${downloadLink}`; tableRow += ''; table.prepend(tableRow); } else { let progressBar = rowTask.find('.progress-bar'); updateProgressBar(progressBar, cookingTask); let downloadLink = rowTask.find('.vault-dl-link'); if (cookingTask.status === 'done') { downloadLink[0].innerHTML = `'; } else if (cookingTask.status === 'failed') { downloadLink[0].innerHTML = ''; } else if (cookingTask.status === 'new') { downloadLink[0].innerHTML = downloadLinkWait; } } } localStorage.setItem('swh-vault-cooking-tasks', JSON.stringify(vaultCookingTasks)); checkVaultId = setTimeout(checkVaultCookingTasks, pollingInterval); }) .catch(() => {}); } export function initUi() { $('#vault-tasks-toggle-selection').change(event => { $('.vault-task-toggle-selection').prop('checked', event.currentTarget.checked); }); $('#vault-remove-tasks').click(() => { clearTimeout(checkVaultId); let tasksToRemove = []; $('.swh-vault-table tbody tr').each((i, row) => { let taskSelected = $(row).find('.vault-task-toggle-selection').prop('checked'); if (taskSelected) { let objectId = $(row).find('.vault-object-id').data('object-id'); tasksToRemove.push(objectId); $(row).remove(); } }); let vaultCookingTasks = JSON.parse(localStorage.getItem('swh-vault-cooking-tasks')); vaultCookingTasks = $.grep(vaultCookingTasks, task => { return $.inArray(task.object_id, tasksToRemove) === -1; }); localStorage.setItem('swh-vault-cooking-tasks', JSON.stringify(vaultCookingTasks)); $('#vault-tasks-toggle-selection').prop('checked', false); checkVaultId = setTimeout(checkVaultCookingTasks, pollingInterval); }); checkVaultId = setTimeout(checkVaultCookingTasks, pollingInterval); $(document).on('shown.bs.tab', 'a[data-toggle="tab"]', e => { if (e.currentTarget.text.trim() === 'Vault') { clearTimeout(checkVaultId); checkVaultCookingTasks(); } }); window.onfocus = () => { clearTimeout(checkVaultId); checkVaultCookingTasks(); }; } diff --git a/swh/web/assets/src/bundles/webapp/code-highlighting.js b/swh/web/assets/src/bundles/webapp/code-highlighting.js index b5461c3a..1ca0f752 100644 --- a/swh/web/assets/src/bundles/webapp/code-highlighting.js +++ b/swh/web/assets/src/bundles/webapp/code-highlighting.js @@ -1,111 +1,111 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {removeUrlFragment} from 'utils/functions'; export async function highlightCode(showLineNumbers = true) { await import(/* webpackChunkName: "highlightjs" */ 'utils/highlightjs'); // keep track of the first highlighted line let firstHighlightedLine = null; // highlighting color let lineHighlightColor = 'rgb(193, 255, 193)'; // function to highlight a line function highlightLine(i) { let lineTd = $(`.hljs-ln-line[data-line-number="${i}"]`); lineTd.css('background-color', lineHighlightColor); return lineTd; } // function to reset highlighting function resetHighlightedLines() { firstHighlightedLine = null; $('.hljs-ln-line[data-line-number]').css('background-color', 'inherit'); } function scrollToLine(lineDomElt) { if ($(lineDomElt).closest('.swh-content').length > 0) { $('html, body').animate({ scrollTop: $(lineDomElt).offset().top - 70 }, 500); } } // function to highlight lines based on a url fragment // in the form '#Lx' or '#Lx-Ly' function parseUrlFragmentForLinesToHighlight() { let lines = []; let linesRegexp = new RegExp(/L(\d+)/g); let line = linesRegexp.exec(window.location.hash); while (line) { lines.push(parseInt(line[1])); line = linesRegexp.exec(window.location.hash); } resetHighlightedLines(); if (lines.length === 1) { firstHighlightedLine = parseInt(lines[0]); scrollToLine(highlightLine(lines[0])); } else if (lines[0] < lines[lines.length - 1]) { firstHighlightedLine = parseInt(lines[0]); scrollToLine(highlightLine(lines[0])); for (let i = lines[0] + 1; i <= lines[lines.length - 1]; ++i) { highlightLine(i); } } } $(document).ready(() => { // highlight code and add line numbers $('code').each((i, block) => { hljs.highlightBlock(block); if (showLineNumbers) { hljs.lineNumbersBlock(block); } }); if (!showLineNumbers) { return; } // click handler to dynamically highlight line(s) // when the user clicks on a line number (lines range // can also be highlighted while holding the shift key) $('body').click(evt => { if (evt.target.classList.contains('hljs-ln-n')) { let line = parseInt($(evt.target).data('line-number')); if (evt.shiftKey && firstHighlightedLine && line > firstHighlightedLine) { let firstLine = firstHighlightedLine; resetHighlightedLines(); for (let i = firstLine; i <= line; ++i) { highlightLine(i); } firstHighlightedLine = firstLine; window.location.hash = `#L${firstLine}-L${line}`; } else { resetHighlightedLines(); highlightLine(line); window.location.hash = `#L${line}`; scrollToLine(evt.target); } } else if ($(evt.target).closest('.hljs-ln').length) { resetHighlightedLines(); removeUrlFragment(); } }); // update lines highlighting when the url fragment changes $(window).on('hashchange', () => parseUrlFragmentForLinesToHighlight()); // schedule lines highlighting if any as hljs.lineNumbersBlock() is async setTimeout(() => { parseUrlFragmentForLinesToHighlight(); }); }); } diff --git a/swh/web/assets/src/bundles/webapp/pdf-rendering.js b/swh/web/assets/src/bundles/webapp/pdf-rendering.js index 9d10ea50..f99754e0 100644 --- a/swh/web/assets/src/bundles/webapp/pdf-rendering.js +++ b/swh/web/assets/src/bundles/webapp/pdf-rendering.js @@ -1,100 +1,100 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // adapted from pdf.js examples located at http://mozilla.github.io/pdf.js/examples/ import {staticAsset} from 'utils/functions'; export async function renderPdf(pdfUrl) { let pdfDoc = null; let pageNum = 1; let pageRendering = false; let pageNumPending = null; let scale = 1.5; let canvas = $('#pdf-canvas')[0]; let ctx = canvas.getContext('2d'); // Get page info from document, resize canvas accordingly, and render page. function renderPage(num) { pageRendering = true; // Using promise to fetch the page pdfDoc.getPage(num).then(page => { let viewport = page.getViewport(scale); canvas.width = viewport.width; canvas.height = viewport.height; // Render PDF page into canvas context let renderContext = { canvasContext: ctx, viewport: viewport }; let renderTask = page.render(renderContext); // Wait for rendering to finish renderTask.promise.then(() => { pageRendering = false; if (pageNumPending !== null) { // New page rendering is pending renderPage(pageNumPending); pageNumPending = null; } }); }); // Update page counters $('#pdf-page-num').text(num); } // If another page rendering in progress, waits until the rendering is // finished. Otherwise, executes rendering immediately. function queueRenderPage(num) { if (pageRendering) { pageNumPending = num; } else { renderPage(num); } } // Displays previous page. function onPrevPage() { if (pageNum <= 1) { return; } pageNum--; queueRenderPage(pageNum); } // Displays next page. function onNextPage() { if (pageNum >= pdfDoc.numPages) { return; } pageNum++; queueRenderPage(pageNum); } let pdfjs = await import(/* webpackChunkName: "pdfjs" */ 'pdfjs-dist'); pdfjs.GlobalWorkerOptions.workerSrc = staticAsset('js/pdf.worker.min.js'); $(document).ready(() => { $('#pdf-prev').click(onPrevPage); $('#pdf-next').click(onNextPage); let loadingTask = pdfjs.getDocument(pdfUrl); loadingTask.promise.then(pdf => { pdfDoc = pdf; $('#pdf-page-count').text(pdfDoc.numPages); // Initial/first page rendering renderPage(pageNum); }, function(reason) { // PDF loading error console.error(reason); }); }); } diff --git a/swh/web/assets/src/bundles/webapp/webapp-utils.js b/swh/web/assets/src/bundles/webapp/webapp-utils.js index f40d27d0..8fc5ab9d 100644 --- a/swh/web/assets/src/bundles/webapp/webapp-utils.js +++ b/swh/web/assets/src/bundles/webapp/webapp-utils.js @@ -1,203 +1,210 @@ +/** + * Copyright (C) 2018-2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + import objectFitImages from 'object-fit-images'; import {Layout} from 'admin-lte'; import {selectText} from 'utils/functions'; import {BREAKPOINT_MD} from 'utils/constants'; let collapseSidebar = false; let previousSidebarState = localStorage.getItem('swh-sidebar-collapsed'); if (previousSidebarState !== undefined) { collapseSidebar = JSON.parse(previousSidebarState); } // adapt implementation of fixLayoutHeight from admin-lte Layout.prototype.fixLayoutHeight = () => { let heights = { window: $(window).height(), header: $('.main-header').outerHeight(), footer: $('.footer').outerHeight(), sidebar: $('.main-sidebar').height(), topbar: $('.swh-top-bar').height() }; let offset = 10; $('.content-wrapper').css('min-height', heights.window - heights.topbar - heights.header - heights.footer - offset); $('.main-sidebar').css('min-height', heights.window - heights.topbar - heights.header - heights.footer - offset); }; $(document).on('DOMContentLoaded', () => { // set state to collapsed on smaller devices if ($(window).width() < BREAKPOINT_MD) { collapseSidebar = true; } // restore previous sidebar state (collapsed/expanded) if (collapseSidebar) { // hack to avoid animated transition for collapsing sidebar // when loading a page let sidebarTransition = $('.main-sidebar, .main-sidebar:before').css('transition'); let sidebarEltsTransition = $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition'); $('.main-sidebar, .main-sidebar:before').css('transition', 'none'); $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition', 'none'); $('body').addClass('sidebar-collapse'); $('.swh-words-logo-swh').css('visibility', 'visible'); // restore transitions for user navigation setTimeout(() => { $('.main-sidebar, .main-sidebar:before').css('transition', sidebarTransition); $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition', sidebarEltsTransition); }); } }); $(document).on('collapsed.lte.pushmenu', event => { if ($('body').width() >= BREAKPOINT_MD) { $('.swh-words-logo-swh').css('visibility', 'visible'); } }); $(document).on('shown.lte.pushmenu', event => { $('.swh-words-logo-swh').css('visibility', 'hidden'); }); function ensureNoFooterOverflow() { $('body').css('padding-bottom', $('footer').outerHeight() + 'px'); } $(document).ready(() => { // redirect to last browse page if any when clicking on the 'Browse' entry // in the sidebar $(`.swh-browse-link`).click(event => { let lastBrowsePage = sessionStorage.getItem('last-browse-page'); if (lastBrowsePage) { event.preventDefault(); window.location = lastBrowsePage; } }); // ensure footer do not overflow main content for mobile devices // or after resizing the browser window ensureNoFooterOverflow(); $(window).resize(function() { ensureNoFooterOverflow(); if ($('body').hasClass('sidebar-collapse') && $('body').width() >= BREAKPOINT_MD) { $('.swh-words-logo-swh').css('visibility', 'visible'); } }); // activate css polyfill 'object-fit: contain' in old browsers objectFitImages(); // reparent the modals to the top navigation div in order to be able // to display them $('.swh-browse-top-navigation').append($('.modal')); let selectedCode = null; function getCodeOrPreEltUnderPointer(e) { let elts = document.elementsFromPoint(e.clientX, e.clientY); for (let elt of elts) { if (elt.nodeName === 'CODE' || elt.nodeName === 'PRE') { return elt; } } return null; } // click handler to set focus on code block for copy $(document).click(e => { selectedCode = getCodeOrPreEltUnderPointer(e); }); function selectCode(event, selectedCode) { if (selectedCode) { let hljsLnCodeElts = $(selectedCode).find('.hljs-ln-code'); if (hljsLnCodeElts.length) { selectText(hljsLnCodeElts[0], hljsLnCodeElts[hljsLnCodeElts.length - 1]); } else { selectText(selectedCode.firstChild, selectedCode.lastChild); } event.preventDefault(); } } // select the whole text of focused code block when user // double clicks or hits Ctrl+A $(document).dblclick(e => { if ((e.ctrlKey || e.metaKey)) { selectCode(e, getCodeOrPreEltUnderPointer(e)); } }); $(document).keydown(e => { if ((e.ctrlKey || e.metaKey) && e.key === 'a') { selectCode(e, selectedCode); } }); // show/hide back-to-top button let scrollThreshold = 0; scrollThreshold += $('.swh-top-bar').height() || 0; scrollThreshold += $('.navbar').height() || 0; $(window).scroll(() => { if ($(window).scrollTop() > scrollThreshold) { $('#back-to-top').css('display', 'block'); } else { $('#back-to-top').css('display', 'none'); } }); }); export function initPage(page) { $(document).ready(() => { // set relevant sidebar link to page active $(`.swh-${page}-item`).addClass('active'); $(`.swh-${page}-link`).addClass('active'); // triggered when unloading the current page $(window).on('unload', () => { // backup sidebar state (collapsed/expanded) let sidebarCollapsed = $('body').hasClass('sidebar-collapse'); localStorage.setItem('swh-sidebar-collapsed', JSON.stringify(sidebarCollapsed)); // backup current browse page if (page === 'browse') { sessionStorage.setItem('last-browse-page', window.location); } }); }); } export function showModalMessage(title, message) { $('#swh-web-modal-message .modal-title').text(title); $('#swh-web-modal-message .modal-content p').text(message); $('#swh-web-modal-message').modal('show'); } export function showModalConfirm(title, message, callback) { $('#swh-web-modal-confirm .modal-title').text(title); $('#swh-web-modal-confirm .modal-content p').text(message); $('#swh-web-modal-confirm #swh-web-modal-confirm-ok-btn').bind('click', () => { callback(); $('#swh-web-modal-confirm').modal('hide'); $('#swh-web-modal-confirm #swh-web-modal-confirm-ok-btn').unbind('click'); }); $('#swh-web-modal-confirm').modal('show'); } let swhObjectIcons; export function setSwhObjectIcons(icons) { swhObjectIcons = icons; } export function getSwhObjectIcon(swhObjectType) { return swhObjectIcons[swhObjectType]; } let browsedSwhObjectMetadata = {}; export function setBrowsedSwhObjectMetadata(metadata) { browsedSwhObjectMetadata = metadata; } export function getBrowsedSwhObjectMetadata() { return browsedSwhObjectMetadata; } diff --git a/swh/web/assets/src/utils/constants.js b/swh/web/assets/src/utils/constants.js index 166e1dc1..8eca57d7 100644 --- a/swh/web/assets/src/utils/constants.js +++ b/swh/web/assets/src/utils/constants.js @@ -1,4 +1,11 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + // Constants defining Bootstrap Breakpoints export const BREAKPOINT_SM = 768; export const BREAKPOINT_MD = 992; export const BREAKPOINT_LG = 1200; diff --git a/swh/web/assets/src/utils/functions.js b/swh/web/assets/src/utils/functions.js index e5745b3f..626da729 100644 --- a/swh/web/assets/src/utils/functions.js +++ b/swh/web/assets/src/utils/functions.js @@ -1,62 +1,62 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // utility functions export function handleFetchError(response) { if (!response.ok) { throw response; } return response; } export function handleFetchErrors(responses) { for (let i = 0; i < responses.length; ++i) { if (!responses[i].ok) { throw responses[i]; } } return responses; } export function staticAsset(asset) { return `${__STATIC__}${asset}`; } export function csrfPost(url, headers = {}, body = null) { headers['X-CSRFToken'] = Cookies.get('csrftoken'); return fetch(url, { credentials: 'include', headers: headers, method: 'POST', body: body }); } export function isGitRepoUrl(url, domain) { let endOfPattern = '\\/[\\w\\.-]+\\/?(?!=.git)(?:\\.git(?:\\/?|\\#[\\w\\.\\-_]+)?)?$'; let pattern = `(?:git|https?|git@)(?:\\:\\/\\/)?${domain}[/|:][A-Za-z0-9-]+?` + endOfPattern; let re = new RegExp(pattern); return re.test(url); }; export function removeUrlFragment() { history.replaceState('', document.title, window.location.pathname + window.location.search); } export function selectText(startNode, endNode) { let selection = window.getSelection(); selection.removeAllRanges(); let range = document.createRange(); range.setStart(startNode, 0); if (endNode.nodeName !== '#text') { range.setEnd(endNode, endNode.childNodes.length); } else { range.setEnd(endNode, endNode.textContent.length); } selection.addRange(range); } diff --git a/swh/web/assets/src/utils/highlightjs.css b/swh/web/assets/src/utils/highlightjs.css index f3b80131..66ae2f0e 100644 --- a/swh/web/assets/src/utils/highlightjs.css +++ b/swh/web/assets/src/utils/highlightjs.css @@ -1,34 +1,34 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ .highlightjs pre { background-color: transparent; border-radius: 0; border-color: transparent; } .hljs { background-color: transparent; white-space: pre; } /* for block of numbers */ .hljs-ln-numbers { cursor: pointer; user-select: none; text-align: center; color: #aaa; border-right: 1px solid #ccc; vertical-align: top; padding-right: 1px !important; } /* for block of code */ .hljs-ln-code { padding-left: 10px !important; width: 100%; } diff --git a/swh/web/assets/src/utils/highlightjs.js b/swh/web/assets/src/utils/highlightjs.js index 1fda6005..62cc685e 100644 --- a/swh/web/assets/src/utils/highlightjs.js +++ b/swh/web/assets/src/utils/highlightjs.js @@ -1,13 +1,13 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // highlightjs chunk that will be lazily loaded import 'highlight.js'; import 'highlightjs-line-numbers.js'; import 'highlight.js/styles/github.css'; import './highlightjs.css'; diff --git a/swh/web/assets/src/utils/showdown.css b/swh/web/assets/src/utils/showdown.css index 83c9cb39..e0d89f7a 100644 --- a/swh/web/assets/src/utils/showdown.css +++ b/swh/web/assets/src/utils/showdown.css @@ -1,25 +1,25 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ .swh-showdown a { border: none; } .swh-showdown table { border-collapse: collapse; } .swh-showdown table, .swh-showdown table th, .swh-showdown table td { padding: 6px 13px; border: 1px solid #dfe2e5; } .swh-showdown table tr:nth-child(even) { background-color: #f2f2f2; } diff --git a/swh/web/browse/browseurls.py b/swh/web/browse/browseurls.py index cf7a5e20..7330ce4c 100644 --- a/swh/web/browse/browseurls.py +++ b/swh/web/browse/browseurls.py @@ -1,42 +1,42 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common.urlsindex import UrlsIndex class BrowseUrls(UrlsIndex): """ Class to manage swh-web browse application urls. """ scope = 'browse' def browse_route(*url_patterns, view_name=None, checksum_args=None): """ Decorator to ease the registration of a swh-web browse endpoint Args: url_patterns: list of url patterns used by Django to identify the browse routes view_name: the name of the Django view associated to the routes used to reverse the url """ url_patterns = ['^' + url_pattern + '$' for url_pattern in url_patterns] view_name = view_name def decorator(f): # register the route and its view in the browse endpoints index for url_pattern in url_patterns: BrowseUrls.add_url_pattern(url_pattern, f, view_name) if checksum_args: BrowseUrls.add_redirect_for_checksum_args(view_name, url_patterns, checksum_args) return f return decorator diff --git a/swh/web/browse/urls.py b/swh/web/browse/urls.py index 1e04b362..ace5caf4 100644 --- a/swh/web/browse/urls.py +++ b/swh/web/browse/urls.py @@ -1,53 +1,53 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.conf.urls import url from django.shortcuts import render import swh.web.browse.views.directory # noqa import swh.web.browse.views.content # noqa import swh.web.browse.views.origin_save # noqa import swh.web.browse.views.origin # noqa import swh.web.browse.views.person # noqa import swh.web.browse.views.release # noqa import swh.web.browse.views.revision # noqa import swh.web.browse.views.snapshot # noqa from swh.web.browse.browseurls import BrowseUrls from swh.web.browse.identifiers import swh_id_browse def _browse_help_view(request): return render(request, 'browse/help.html', {'heading': 'How to browse the archive ?'}) def _browse_search_view(request): return render(request, 'browse/search.html', {'heading': 'Search software origins to browse'}) def _browse_vault_view(request): return render(request, 'browse/vault-ui.html', {'heading': 'Download archive content from the Vault'}) def _browse_origin_save_view(request): return render(request, 'browse/origin-save.html', {'heading': 'Request the saving of a software origin into the archive'}) # noqa urlpatterns = [ url(r'^$', _browse_search_view), url(r'^help/$', _browse_help_view, name='browse-help'), url(r'^search/$', _browse_search_view, name='browse-search'), url(r'^vault/$', _browse_vault_view, name='browse-vault'), url(r'^origin/save/$', _browse_origin_save_view, name='browse-origin-save'), # for backward compatibility url(r'^(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$', swh_id_browse) ] urlpatterns += BrowseUrls.get_url_patterns() diff --git a/swh/web/browse/views/content.py b/swh/web/browse/views/content.py index e0b8765e..d2e9545e 100644 --- a/swh/web/browse/views/content.py +++ b/swh/web/browse/views/content.py @@ -1,317 +1,317 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import difflib import json from distutils.util import strtobool from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat from swh.model.hashutil import hash_to_hex from swh.web.common import query, service from swh.web.common.utils import ( reverse, gen_path_info, swh_object_icons ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.utils import ( request_content, prepare_content_for_display, content_display_max_size, get_snapshot_context, get_swh_persistent_ids, gen_link, gen_directory_link ) from swh.web.browse.browseurls import browse_route @browse_route(r'content/(?P[0-9a-z_:]*[0-9a-f]+.)/raw/', view_name='browse-content-raw', checksum_args=['query_string']) def content_raw(request, query_string): """Django view that produces a raw display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/raw/` """ # noqa try: reencode = bool(strtobool(request.GET.get('reencode', 'false'))) algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) content_data = request_content(query_string, max_size=None, reencode=reencode) except Exception as exc: return handle_view_exception(request, exc) filename = request.GET.get('filename', None) if not filename: filename = '%s_%s' % (algo, checksum) if content_data['mimetype'].startswith('text/') or \ content_data['mimetype'] == 'inode/x-empty': response = HttpResponse(content_data['raw_data'], content_type="text/plain") response['Content-disposition'] = 'filename=%s' % filename else: response = HttpResponse(content_data['raw_data'], content_type='application/octet-stream') response['Content-disposition'] = 'attachment; filename=%s' % filename return response _auto_diff_size_limit = 20000 @browse_route(r'content/(?P.*)/diff/(?P.*)', # noqa view_name='diff-contents') def _contents_diff(request, from_query_string, to_query_string): """ Browse endpoint used to compute unified diffs between two contents. Diffs are generated only if the two contents are textual. By default, diffs whose size are greater than 20 kB will not be generated. To force the generation of large diffs, the 'force' boolean query parameter must be used. Args: request: input django http request from_query_string: a string of the form "[ALGO_HASH:]HASH" where optional ALGO_HASH can be either ``sha1``, ``sha1_git``, ``sha256``, or ``blake2s256`` (default to ``sha1``) and HASH the hexadecimal representation of the hash value identifying the first content to_query_string: same as above for identifying the second content Returns: A JSON object containing the unified diff. """ diff_data = {} content_from = None content_to = None content_from_size = 0 content_to_size = 0 content_from_lines = [] content_to_lines = [] force = request.GET.get('force', 'false') path = request.GET.get('path', None) language = 'nohighlight' force = bool(strtobool(force)) if from_query_string == to_query_string: diff_str = 'File renamed without changes' else: try: text_diff = True if from_query_string: content_from = \ request_content(from_query_string, max_size=None) content_from_display_data = \ prepare_content_for_display(content_from['raw_data'], content_from['mimetype'], path) language = content_from_display_data['language'] content_from_size = content_from['length'] if not (content_from['mimetype'].startswith('text/') or content_from['mimetype'] == 'inode/x-empty'): text_diff = False if text_diff and to_query_string: content_to = request_content(to_query_string, max_size=None) content_to_display_data = prepare_content_for_display( content_to['raw_data'], content_to['mimetype'], path) language = content_to_display_data['language'] content_to_size = content_to['length'] if not (content_to['mimetype'].startswith('text/') or content_to['mimetype'] == 'inode/x-empty'): text_diff = False diff_size = abs(content_to_size - content_from_size) if not text_diff: diff_str = 'Diffs are not generated for non textual content' language = 'nohighlight' elif not force and diff_size > _auto_diff_size_limit: diff_str = 'Large diffs are not automatically computed' language = 'nohighlight' else: if content_from: content_from_lines = \ content_from['raw_data'].decode('utf-8')\ .splitlines(True) if content_from_lines and \ content_from_lines[-1][-1] != '\n': content_from_lines[-1] += '[swh-no-nl-marker]\n' if content_to: content_to_lines = content_to['raw_data'].decode('utf-8')\ .splitlines(True) if content_to_lines and content_to_lines[-1][-1] != '\n': content_to_lines[-1] += '[swh-no-nl-marker]\n' diff_lines = difflib.unified_diff(content_from_lines, content_to_lines) diff_str = ''.join(list(diff_lines)[2:]) except Exception as e: diff_str = str(e) diff_data['diff_str'] = diff_str diff_data['language'] = language diff_data_json = json.dumps(diff_data, separators=(',', ': ')) return HttpResponse(diff_data_json, content_type='application/json') @browse_route(r'content/(?P[0-9a-z_:]*[0-9a-f]+.)/', view_name='browse-content', checksum_args=['query_string']) def content_display(request, query_string): """Django view that produces an HTML display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/` """ # noqa try: algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) content_data = request_content(query_string, raise_if_unavailable=False) origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) snapshot_context = None if origin_url: try: snapshot_context = get_snapshot_context(None, origin_type, origin_url) except Exception: raw_cnt_url = reverse('browse-content', url_args={'query_string': query_string}) error_message = \ ('The Software Heritage archive has a content ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the content ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_cnt_url))) raise NotFoundExc(error_message) if snapshot_context: snapshot_context['visit_info'] = None except Exception as exc: return handle_view_exception(request, exc) path = request.GET.get('path', None) content = None language = None mimetype = None if content_data['raw_data'] is not None: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] root_dir = None filename = None path_info = None directory_id = None directory_url = None query_params = {'origin': origin_url} breadcrumbs = [] if path: split_path = path.split('/') root_dir = split_path[0] filename = split_path[-1] if root_dir != path: path = path.replace(root_dir + '/', '') path = path[:-len(filename)] path_info = gen_path_info(path) dir_url = reverse('browse-directory', url_args={'sha1_git': root_dir}, query_params=query_params) breadcrumbs.append({'name': root_dir[:7], 'url': dir_url}) for pi in path_info: dir_url = reverse('browse-directory', url_args={'sha1_git': root_dir, 'path': pi['path']}, query_params=query_params) breadcrumbs.append({'name': pi['name'], 'url': dir_url}) breadcrumbs.append({'name': filename, 'url': None}) if path and root_dir != path: dir_info = service.lookup_directory_with_path(root_dir, path) directory_id = dir_info['target'] elif root_dir != path: directory_id = root_dir if directory_id: directory_url = gen_directory_link(directory_id) query_params = {'filename': filename} content_raw_url = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params=query_params) content_metadata = { 'sha1': content_data['checksums']['sha1'], 'sha1_git': content_data['checksums']['sha1_git'], 'sha256': content_data['checksums']['sha256'], 'blake2s256': content_data['checksums']['blake2s256'], 'mimetype': content_data['mimetype'], 'encoding': content_data['encoding'], 'size': filesizeformat(content_data['length']), 'language': content_data['language'], 'licenses': content_data['licenses'], 'filename': filename, 'directory': directory_id, 'context-independent directory': directory_url } if filename: content_metadata['filename'] = filename sha1_git = content_data['checksums']['sha1_git'] swh_ids = get_swh_persistent_ids([{'type': 'content', 'id': sha1_git}]) heading = 'Content - %s' % sha1_git if breadcrumbs: content_path = '/'.join([bc['name'] for bc in breadcrumbs]) heading += ' - %s' % content_path return render(request, 'browse/content.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Content', 'swh_object_metadata': content_metadata, 'content': content, 'content_size': content_data['length'], 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'breadcrumbs': breadcrumbs, 'top_right_link': { 'url': content_raw_url, 'icon': swh_object_icons['content'], 'text': 'Raw File' }, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': content_data['error_code'], 'error_message': content_data['error_message'], 'error_description': content_data['error_description']}, status=content_data['error_code']) diff --git a/swh/web/browse/views/directory.py b/swh/web/browse/views/directory.py index 0cb39567..7b6fc377 100644 --- a/swh/web/browse/views/directory.py +++ b/swh/web/browse/views/directory.py @@ -1,177 +1,177 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from django.http import HttpResponse from django.shortcuts import render, redirect from django.template.defaultfilters import filesizeformat from swh.web.common import service from swh.web.common.utils import ( reverse, gen_path_info ) from swh.web.common.exc import handle_view_exception, NotFoundExc from swh.web.browse.utils import ( get_directory_entries, get_snapshot_context, get_readme_to_display, get_swh_persistent_ids, gen_link ) from swh.web.browse.browseurls import browse_route @browse_route(r'directory/(?P[0-9a-f]+)/', r'directory/(?P[0-9a-f]+)/(?P.+)/', view_name='browse-directory', checksum_args=['sha1_git']) def directory_browse(request, sha1_git, path=None): """Django view for browsing the content of a directory identified by its sha1_git value. The url that points to it is :http:get:`/browse/directory/(sha1_git)/[(path)/]` """ # noqa root_sha1_git = sha1_git try: if path: dir_info = service.lookup_directory_with_path(sha1_git, path) sha1_git = dir_info['target'] dirs, files = get_directory_entries(sha1_git) origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) snapshot_context = None if origin_url: try: snapshot_context = get_snapshot_context(None, origin_type, origin_url) except Exception: raw_dir_url = reverse('browse-directory', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a directory ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the directory ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_dir_url))) raise NotFoundExc(error_message) if snapshot_context: snapshot_context['visit_info'] = None except Exception as exc: return handle_view_exception(request, exc) path_info = gen_path_info(path) query_params = {'origin': origin_url} breadcrumbs = [] breadcrumbs.append({'name': root_sha1_git[:7], 'url': reverse('browse-directory', url_args={'sha1_git': root_sha1_git}, query_params=query_params)}) for pi in path_info: breadcrumbs.append({'name': pi['name'], 'url': reverse('browse-directory', url_args={'sha1_git': root_sha1_git, 'path': pi['path']}, query_params=query_params)}) path = '' if path is None else (path + '/') for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}, query_params=query_params) else: d['url'] = reverse('browse-directory', url_args={'sha1_git': root_sha1_git, 'path': path + d['name']}, query_params=query_params) sum_file_sizes = 0 readmes = {} for f in files: query_string = 'sha1_git:' + f['target'] f['url'] = reverse('browse-content', url_args={'query_string': query_string}, query_params={'path': root_sha1_git + '/' + path + f['name'], 'origin': origin_url}) if f['length'] is not None: sum_file_sizes += f['length'] f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) sum_file_sizes = filesizeformat(sum_file_sizes) dir_metadata = {'directory': sha1_git, 'number of regular files': len(files), 'number of subdirectories': len(dirs), 'sum of regular file sizes': sum_file_sizes} vault_cooking = { 'directory_context': True, 'directory_id': sha1_git, 'revision_context': False, 'revision_id': None } swh_ids = get_swh_persistent_ids([{'type': 'directory', 'id': sha1_git}]) heading = 'Directory - %s' % sha1_git if breadcrumbs: dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' heading += ' - %s' % dir_path return render(request, 'browse/directory.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Directory', 'swh_object_metadata': dir_metadata, 'dirs': dirs, 'files': files, 'breadcrumbs': breadcrumbs, 'top_right_link': None, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'snapshot_context': snapshot_context, 'vault_cooking': vault_cooking, 'show_actions_menu': True, 'swh_ids': swh_ids}) @browse_route(r'directory/resolve/content-path/(?P[0-9a-f]+)/(?P.+)/', # noqa view_name='browse-directory-resolve-content-path', checksum_args=['sha1_git']) def _directory_resolve_content_path(request, sha1_git, path): """ Internal endpoint redirecting to data url for a specific file path relative to a root directory. """ try: path = os.path.normpath(path) if not path.startswith('../'): dir_info = service.lookup_directory_with_path(sha1_git, path) if dir_info['type'] == 'file': sha1 = dir_info['checksums']['sha1'] data_url = reverse('browse-content-raw', url_args={'query_string': sha1}) return redirect(data_url) except Exception: pass return HttpResponse(status=404) diff --git a/swh/web/browse/views/origin.py b/swh/web/browse/views/origin.py index ec2e3368..5e3ca940 100644 --- a/swh/web/browse/views/origin.py +++ b/swh/web/browse/views/origin.py @@ -1,292 +1,292 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from distutils.util import strtobool from django.core.cache import caches from django.http import HttpResponse from django.shortcuts import render, redirect from django.views.decorators.cache import never_cache from swh.web.common import service from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import ( reverse, format_utc_iso_date, parse_timestamp ) from swh.web.common.exc import handle_view_exception from swh.web.browse.utils import ( get_origin_info, get_snapshot_context ) from swh.web.browse.browseurls import browse_route from swh.web.misc.coverage import code_providers from .utils.snapshot_context import ( browse_snapshot_directory, browse_snapshot_content, browse_snapshot_log, browse_snapshot_branches, browse_snapshot_releases ) @browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/directory/', # noqa r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/directory/(?P.+)/', # noqa r'origin/(?P[a-z]+)/url/(?P.+)/directory/', # noqa r'origin/(?P[a-z]+)/url/(?P.+)/directory/(?P.+)/', # noqa r'origin/(?P.+)/visit/(?P.+)/directory/', # noqa r'origin/(?P.+)/visit/(?P.+)/directory/(?P.+)/', # noqa r'origin/(?P.+)/directory/', # noqa r'origin/(?P.+)/directory/(?P.+)/', # noqa view_name='browse-origin-directory') def origin_directory_browse(request, origin_url, origin_type=None, timestamp=None, path=None): """Django view for browsing the content of a directory associated to an origin for a given visit. The url scheme that points to it is the following: * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/directory/[(path)/]` * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/directory/[(path)/]` """ # noqa return browse_snapshot_directory( request, origin_type=origin_type, origin_url=origin_url, timestamp=timestamp, path=path) @browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/content/(?P.+)/', # noqa r'origin/(?P[a-z]+)/url/(?P.+)/content/(?P.+)/', # noqa r'origin/(?P.+)/visit/(?P.+)/content/(?P.+)/', # noqa r'origin/(?P.+)/content/(?P.+)/', # noqa view_name='browse-origin-content') def origin_content_browse(request, origin_url, origin_type=None, path=None, timestamp=None): """Django view that produces an HTML display of a content associated to an origin for a given visit. The url scheme that points to it is the following: * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/content/(path)/` * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/content/(path)/` """ # noqa return browse_snapshot_content(request, origin_type=origin_type, origin_url=origin_url, timestamp=timestamp, path=path) PER_PAGE = 20 @browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/log/', # noqa r'origin/(?P[a-z]+)/url/(?P.+)/log/', r'origin/(?P.+)/visit/(?P.+)/log/', # noqa r'origin/(?P.+)/log/', view_name='browse-origin-log') def origin_log_browse(request, origin_url, origin_type=None, timestamp=None): """Django view that produces an HTML display of revisions history (aka the commit log) associated to a software origin. The url scheme that points to it is the following: * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/log/` * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/log/` """ # noqa return browse_snapshot_log(request, origin_type=origin_type, origin_url=origin_url, timestamp=timestamp) @browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/branches/', # noqa r'origin/(?P[a-z]+)/url/(?P.+)/branches/', # noqa r'origin/(?P.+)/visit/(?P.+)/branches/', # noqa r'origin/(?P.+)/branches/', # noqa view_name='browse-origin-branches') def origin_branches_browse(request, origin_url, origin_type=None, timestamp=None): """Django view that produces an HTML display of the list of branches associated to an origin for a given visit. The url scheme that points to it is the following: * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/branches/` * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/branches/` """ # noqa return browse_snapshot_branches(request, origin_type=origin_type, origin_url=origin_url, timestamp=timestamp) @browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/releases/', # noqa r'origin/(?P[a-z]+)/url/(?P.+)/releases/', # noqa r'origin/(?P.+)/visit/(?P.+)/releases/', # noqa r'origin/(?P.+)/releases/', # noqa view_name='browse-origin-releases') def origin_releases_browse(request, origin_url, origin_type=None, timestamp=None): """Django view that produces an HTML display of the list of releases associated to an origin for a given visit. The url scheme that points to it is the following: * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/releases/` * :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visit/(timestamp)/releases/` """ # noqa return browse_snapshot_releases(request, origin_type=origin_type, origin_url=origin_url, timestamp=timestamp) @browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visits/', r'origin/(?P.+)/visits/', view_name='browse-origin-visits') def origin_visits_browse(request, origin_url, origin_type=None): """Django view that produces an HTML display of visits reporting for a swh origin identified by its id or its url. The url that points to it is :http:get:`/browse/origin/[(origin_type)/url/](origin_url)/visits/`. """ # noqa try: origin_info = get_origin_info(origin_url, origin_type) origin_visits = get_origin_visits(origin_info) snapshot_context = get_snapshot_context(origin_type=origin_type, origin_url=origin_url) except Exception as exc: return handle_view_exception(request, exc) for i, visit in enumerate(origin_visits): url_date = format_utc_iso_date(visit['date'], '%Y-%m-%dT%H:%M:%SZ') visit['fmt_date'] = format_utc_iso_date(visit['date']) query_params = {} if i < len(origin_visits) - 1: if visit['date'] == origin_visits[i+1]['date']: query_params = {'visit_id': visit['visit']} if i > 0: if visit['date'] == origin_visits[i-1]['date']: query_params = {'visit_id': visit['visit']} snapshot = visit['snapshot'] if visit['snapshot'] else '' visit['browse_url'] = reverse('browse-origin-directory', url_args={'origin_type': origin_type, 'origin_url': origin_url, 'timestamp': url_date}, query_params=query_params) if not snapshot: visit['snapshot'] = '' visit['date'] = parse_timestamp(visit['date']).timestamp() heading = 'Origin visits - %s' % origin_url return render(request, 'browse/origin-visits.html', {'heading': heading, 'swh_object_name': 'Visits', 'swh_object_metadata': origin_info, 'origin_visits': origin_visits, 'origin_info': origin_info, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': False}) @browse_route(r'origin/search/(?P.+)/', view_name='browse-origin-search') def _origin_search(request, url_pattern): """Internal browse endpoint to search for origins whose urls contain a provided string pattern or match a provided regular expression. The search is performed in a case insensitive way. """ offset = int(request.GET.get('offset', '0')) limit = int(request.GET.get('limit', '50')) regexp = request.GET.get('regexp', 'false') with_visit = request.GET.get('with_visit', 'false') url_pattern = url_pattern.replace('///', '\\') try: results = service.search_origin(url_pattern, offset, limit, bool(strtobool(regexp)), bool(strtobool(with_visit))) results = json.dumps(list(results), sort_keys=True, indent=4, separators=(',', ': ')) except Exception as exc: return handle_view_exception(request, exc, html_response=False) return HttpResponse(results, content_type='application/json') @browse_route(r'origin/coverage_count/', view_name='browse-origin-coverage-count') @never_cache def _origin_coverage_count(request): """Internal browse endpoint to count the number of origins associated to each code provider declared in the archive coverage list. As this operation takes some times, we execute it once per day and cache its results to database. The cached origin counts are then served. Cache management is handled in the implementation to avoid sending the same count query twice to the storage database. """ try: cache = caches['db_cache'] results = [] for code_provider in code_providers: provider_id = code_provider['provider_id'] url_regexp = code_provider['origin_url_regexp'] cache_key = '%s_origins_count' % provider_id prev_cache_key = '%s_origins_prev_count' % provider_id # get cached origin count origin_count = cache.get(cache_key, -2) # cache entry has expired or does not exist if origin_count == -2: # mark the origin count as processing cache.set(cache_key, -1, timeout=10*60) # execute long count query origin_count = service.storage.origin_count(url_regexp, regexp=True) # cache count result cache.set(cache_key, origin_count, timeout=24*60*60) cache.set(prev_cache_key, origin_count, timeout=None) # origin count is currently processing elif origin_count == -1: # return previous count if it exists origin_count = cache.get(prev_cache_key, -1) results.append({ 'provider_id': provider_id, 'origin_count': origin_count, 'origin_types': code_provider['origin_types'] }) results = json.dumps(results) except Exception as exc: return handle_view_exception(request, exc, html_response=False) return HttpResponse(results, content_type='application/json') @browse_route(r'origin/(?P[0-9]+)/latest_snapshot/', view_name='browse-origin-latest-snapshot') def _origin_latest_snapshot(request, origin_id): """ Internal browse endpoint used to check if an origin has already been visited by Software Heritage and has at least one full visit. """ result = \ service.lookup_latest_origin_snapshot(int(origin_id), allowed_statuses=['full', 'partial']) result = json.dumps(result, sort_keys=True, indent=4, separators=(',', ': ')) return HttpResponse(result, content_type='application/json') @browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/', r'origin/(?P.+)/', view_name='browse-origin') def origin_browse(request, origin_url, origin_type=None): """Django view that redirects to the display of the latest archived snapshot for a given software origin. """ # noqa last_snapshot_url = reverse('browse-origin-directory', url_args={'origin_type': origin_type, 'origin_url': origin_url}) return redirect(last_snapshot_url) diff --git a/swh/web/browse/views/origin_save.py b/swh/web/browse/views/origin_save.py index bdbc0ae3..66058117 100644 --- a/swh/web/browse/views/origin_save.py +++ b/swh/web/browse/views/origin_save.py @@ -1,90 +1,90 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from django.core.paginator import Paginator from django.http import HttpResponse, HttpResponseForbidden from rest_framework.decorators import api_view, authentication_classes from swh.web.browse.browseurls import browse_route from swh.web.common.exc import ForbiddenExc from swh.web.common.models import SaveOriginRequest from swh.web.common.origin_save import ( create_save_origin_request, get_savable_origin_types, get_save_origin_requests_from_queryset ) from swh.web.common.throttling import throttle_scope from swh.web.common.utils import EnforceCSRFAuthentication @browse_route(r'origin/save/(?P.+)/url/(?P.+)/', view_name='browse-origin-save-request') @api_view(['POST']) @authentication_classes((EnforceCSRFAuthentication, )) @throttle_scope('swh_save_origin') def _browse_origin_save_request(request, origin_type, origin_url): """ This view is called through AJAX from the save code now form of swh-web. We use DRF here as we want to rate limit the number of submitted requests per user to avoid being possibly flooded by bots. """ try: response = json.dumps(create_save_origin_request(origin_type, origin_url), separators=(',', ': ')) return HttpResponse(response, content_type='application/json') except ForbiddenExc as exc: return HttpResponseForbidden(str(exc)) @browse_route(r'origin/save/types/list/', view_name='browse-origin-save-types-list') def _browse_origin_save_types_list(request): origin_types = json.dumps(get_savable_origin_types(), separators=(',', ': ')) return HttpResponse(origin_types, content_type='application/json') @browse_route(r'origin/save/requests/list/(?P.+)/', view_name='browse-origin-save-requests-list') def _browse_origin_save_requests_list(request, status): if status != 'all': save_requests = SaveOriginRequest.objects.filter(status=status) else: save_requests = SaveOriginRequest.objects.all() table_data = {} table_data['recordsTotal'] = save_requests.count() table_data['draw'] = int(request.GET['draw']) search_value = request.GET['search[value]'] column_order = request.GET['order[0][column]'] field_order = request.GET['columns[%s][name]' % column_order] order_dir = request.GET['order[0][dir]'] if order_dir == 'desc': field_order = '-' + field_order save_requests = save_requests.order_by(field_order) length = int(request.GET['length']) page = int(request.GET['start']) / length + 1 save_requests = get_save_origin_requests_from_queryset(save_requests) if search_value: save_requests = \ [sr for sr in save_requests if search_value.lower() in sr['save_request_status'].lower() or search_value.lower() in sr['save_task_status'].lower() or search_value.lower() in sr['origin_type'].lower() or search_value.lower() in sr['origin_url'].lower()] table_data['recordsFiltered'] = len(save_requests) paginator = Paginator(save_requests, length) table_data['data'] = paginator.page(page).object_list table_data_json = json.dumps(table_data, separators=(',', ': ')) return HttpResponse(table_data_json, content_type='application/json') diff --git a/swh/web/browse/views/release.py b/swh/web/browse/views/release.py index 74bdd2e7..7e8295f8 100644 --- a/swh/web/browse/views/release.py +++ b/swh/web/browse/views/release.py @@ -1,194 +1,194 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import render from swh.web.common import service from swh.web.common.utils import ( reverse, format_utc_iso_date ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.browseurls import browse_route from swh.web.browse.utils import ( gen_person_link, gen_revision_link, get_snapshot_context, gen_link, gen_snapshot_link, get_swh_persistent_ids, gen_directory_link, gen_content_link, gen_release_link ) @browse_route(r'release/(?P[0-9a-f]+)/', view_name='browse-release', checksum_args=['sha1_git']) def release_browse(request, sha1_git): """ Django view that produces an HTML display of a release identified by its id. The url that points to it is :http:get:`/browse/release/(sha1_git)/`. """ try: release = service.lookup_release(sha1_git) snapshot_context = None origin_info = None snapshot_id = request.GET.get('snapshot_id', None) origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) if origin_url: try: snapshot_context = \ get_snapshot_context(snapshot_id, origin_type, origin_url, timestamp, visit_id) except Exception: raw_rel_url = reverse('browse-release', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a release ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the release ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_rel_url))) raise NotFoundExc(error_message) origin_info = snapshot_context['origin_info'] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) except Exception as exc: return handle_view_exception(request, exc) release_data = {} author_name = 'None' release_data['author'] = 'None' if release['author']: author_name = release['author']['name'] or \ release['author']['fullname'] release_data['author'] = \ gen_person_link(release['author']['id'], author_name, snapshot_context) release_data['date'] = format_utc_iso_date(release['date']) release_data['release'] = sha1_git release_data['name'] = release['name'] release_data['synthetic'] = release['synthetic'] release_data['target'] = release['target'] release_data['target type'] = release['target_type'] if snapshot_context: if release['target_type'] == 'revision': release_data['context-independent target'] = \ gen_revision_link(release['target']) elif release['target_type'] == 'content': release_data['context-independent target'] = \ gen_content_link(release['target']) elif release['target_type'] == 'directory': release_data['context-independent target'] = \ gen_directory_link(release['target']) elif release['target_type'] == 'release': release_data['context-independent target'] = \ gen_release_link(release['target']) release_note_lines = [] if release['message']: release_note_lines = release['message'].split('\n') vault_cooking = None target_link = None if release['target_type'] == 'revision': target_link = gen_revision_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) try: revision = service.lookup_revision(release['target']) vault_cooking = { 'directory_context': True, 'directory_id': revision['directory'], 'revision_context': True, 'revision_id': release['target'] } except Exception: pass elif release['target_type'] == 'directory': target_link = gen_directory_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) try: revision = service.lookup_directory(release['target']) vault_cooking = { 'directory_context': True, 'directory_id': revision['directory'], 'revision_context': False, 'revision_id': None } except Exception: pass elif release['target_type'] == 'content': target_link = gen_content_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) elif release['target_type'] == 'release': target_link = gen_release_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) release['target_link'] = target_link if snapshot_context: release_data['snapshot'] = snapshot_context['snapshot_id'] if origin_info: release_data['context-independent release'] = \ gen_release_link(release['id']) release_data['origin type'] = origin_info['type'] release_data['origin url'] = gen_link(origin_info['url'], origin_info['url']) browse_snapshot_link = \ gen_snapshot_link(snapshot_context['snapshot_id']) release_data['context-independent snapshot'] = browse_snapshot_link swh_objects = [{'type': 'release', 'id': sha1_git}] if snapshot_context: snapshot_id = snapshot_context['snapshot_id'] if snapshot_id: swh_objects.append({'type': 'snapshot', 'id': snapshot_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) note_header = 'None' if len(release_note_lines) > 0: note_header = release_note_lines[0] release['note_header'] = note_header release['note_body'] = '\n'.join(release_note_lines[1:]) heading = 'Release - %s' % release['name'] if snapshot_context: context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading += ' - %s' % context_found return render(request, 'browse/release.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Release', 'swh_object_metadata': release_data, 'release': release, 'snapshot_context': snapshot_context, 'show_actions_menu': True, 'breadcrumbs': None, 'vault_cooking': vault_cooking, 'top_right_link': None, 'swh_ids': swh_ids}) diff --git a/swh/web/browse/views/revision.py b/swh/web/browse/views/revision.py index 7fc14d7f..fd73af13 100644 --- a/swh/web/browse/views/revision.py +++ b/swh/web/browse/views/revision.py @@ -1,536 +1,536 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import hashlib import json import textwrap from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.utils.html import escape from django.utils.safestring import mark_safe from swh.model.identifiers import persistent_identifier from swh.web.common import service from swh.web.common.utils import ( reverse, format_utc_iso_date, gen_path_info, swh_object_icons ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.browseurls import browse_route from swh.web.browse.utils import ( gen_link, gen_person_link, gen_revision_link, gen_revision_url, get_snapshot_context, get_revision_log_url, get_directory_entries, gen_directory_link, request_content, prepare_content_for_display, content_display_max_size, gen_snapshot_link, get_readme_to_display, get_swh_persistent_ids, format_log_entries ) def _gen_content_url(revision, query_string, path, snapshot_context): if snapshot_context: url_args = snapshot_context['url_args'] url_args['path'] = path query_params = snapshot_context['query_params'] query_params['revision'] = revision['id'] content_url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) else: content_path = '%s/%s' % (revision['directory'], path) content_url = reverse('browse-content', url_args={'query_string': query_string}, query_params={'path': content_path}) return content_url def _gen_diff_link(idx, diff_anchor, link_text): if idx < _max_displayed_file_diffs: return gen_link(diff_anchor, link_text) else: return link_text # TODO: put in conf _max_displayed_file_diffs = 1000 def _gen_revision_changes_list(revision, changes, snapshot_context): """ Returns a HTML string describing the file changes introduced in a revision. As this string will be displayed in the browse revision view, links to adequate file diffs are also generated. Args: revision (str): hexadecimal representation of a revision identifier changes (list): list of file changes in the revision snapshot_context (dict): optional origin context used to reverse the content urls Returns: A string to insert in a revision HTML view. """ changes_msg = [] for i, change in enumerate(changes): hasher = hashlib.sha1() from_query_string = '' to_query_string = '' diff_id = 'diff-' if change['from']: from_query_string = 'sha1_git:' + change['from']['target'] diff_id += change['from']['target'] + '-' + change['from_path'] diff_id += '-' if change['to']: to_query_string = 'sha1_git:' + change['to']['target'] diff_id += change['to']['target'] + change['to_path'] change['path'] = change['to_path'] or change['from_path'] url_args = {'from_query_string': from_query_string, 'to_query_string': to_query_string} query_params = {'path': change['path']} change['diff_url'] = reverse('diff-contents', url_args=url_args, query_params=query_params) hasher.update(diff_id.encode('utf-8')) diff_id = hasher.hexdigest() change['id'] = diff_id panel_diff_link = '#panel_' + diff_id if change['type'] == 'modify': change['content_url'] = \ _gen_content_url(revision, to_query_string, change['to_path'], snapshot_context) changes_msg.append('modified: %s' % _gen_diff_link(i, panel_diff_link, change['to_path'])) elif change['type'] == 'insert': change['content_url'] = \ _gen_content_url(revision, to_query_string, change['to_path'], snapshot_context) changes_msg.append('new file: %s' % _gen_diff_link(i, panel_diff_link, change['to_path'])) elif change['type'] == 'delete': parent = service.lookup_revision(revision['parents'][0]) change['content_url'] = \ _gen_content_url(parent, from_query_string, change['from_path'], snapshot_context) changes_msg.append('deleted: %s' % _gen_diff_link(i, panel_diff_link, change['from_path'])) elif change['type'] == 'rename': change['content_url'] = \ _gen_content_url(revision, to_query_string, change['to_path'], snapshot_context) link_text = change['from_path'] + ' → ' + change['to_path'] changes_msg.append('renamed: %s' % _gen_diff_link(i, panel_diff_link, link_text)) if not changes: changes_msg.append('No changes') return mark_safe('\n'.join(changes_msg)) @browse_route(r'revision/(?P[0-9a-f]+)/diff/', view_name='diff-revision', checksum_args=['sha1_git']) def _revision_diff(request, sha1_git): """ Browse internal endpoint to compute revision diff """ try: revision = service.lookup_revision(sha1_git) snapshot_context = None origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) if origin_url: snapshot_context = get_snapshot_context(None, origin_type, origin_url, timestamp, visit_id) except Exception as exc: return handle_view_exception(request, exc) changes = service.diff_revision(sha1_git) changes_msg = _gen_revision_changes_list(revision, changes, snapshot_context) diff_data = { 'total_nb_changes': len(changes), 'changes': changes[:_max_displayed_file_diffs], 'changes_msg': changes_msg } diff_data_json = json.dumps(diff_data, separators=(',', ': ')) return HttpResponse(diff_data_json, content_type='application/json') NB_LOG_ENTRIES = 100 @browse_route(r'revision/(?P[0-9a-f]+)/log/', view_name='browse-revision-log', checksum_args=['sha1_git']) def revision_log_browse(request, sha1_git): """ Django view that produces an HTML display of the history log for a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/log/` """ # noqa try: per_page = int(request.GET.get('per_page', NB_LOG_ENTRIES)) offset = int(request.GET.get('offset', 0)) revs_ordering = request.GET.get('revs_ordering', 'committer_date') session_key = 'rev_%s_log_ordering_%s' % (sha1_git, revs_ordering) rev_log_session = request.session.get(session_key, None) rev_log = [] revs_walker_state = None if rev_log_session: rev_log = rev_log_session['rev_log'] revs_walker_state = rev_log_session['revs_walker_state'] if len(rev_log) < offset+per_page: revs_walker = \ service.get_revisions_walker(revs_ordering, sha1_git, max_revs=offset+per_page+1, state=revs_walker_state) rev_log += [rev['id'] for rev in revs_walker] revs_walker_state = revs_walker.export_state() revs = rev_log[offset:offset+per_page] revision_log = service.lookup_revision_multiple(revs) request.session[session_key] = { 'rev_log': rev_log, 'revs_walker_state': revs_walker_state } except Exception as exc: return handle_view_exception(request, exc) revs_ordering = request.GET.get('revs_ordering', '') prev_log_url = None if len(rev_log) > offset + per_page: prev_log_url = reverse('browse-revision-log', url_args={'sha1_git': sha1_git}, query_params={'per_page': per_page, 'offset': offset + per_page, 'revs_ordering': revs_ordering}) next_log_url = None if offset != 0: next_log_url = reverse('browse-revision-log', url_args={'sha1_git': sha1_git}, query_params={'per_page': per_page, 'offset': offset - per_page, 'revs_ordering': revs_ordering}) revision_log_data = format_log_entries(revision_log, per_page) swh_rev_id = persistent_identifier('revision', sha1_git) return render(request, 'browse/revision-log.html', {'heading': 'Revision history', 'swh_object_id': swh_rev_id, 'swh_object_name': 'Revisions history', 'swh_object_metadata': None, 'revision_log': revision_log_data, 'revs_ordering': revs_ordering, 'next_log_url': next_log_url, 'prev_log_url': prev_log_url, 'breadcrumbs': None, 'top_right_link': None, 'snapshot_context': None, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': None}) @browse_route(r'revision/(?P[0-9a-f]+)/', r'revision/(?P[0-9a-f]+)/(?P.+)/', view_name='browse-revision', checksum_args=['sha1_git']) def revision_browse(request, sha1_git, extra_path=None): """ Django view that produces an HTML display of a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/`. """ try: revision = service.lookup_revision(sha1_git) origin_info = None snapshot_context = None origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) snapshot_id = request.GET.get('snapshot_id', None) path = request.GET.get('path', None) dir_id = None dirs, files = None, None content_data = None if origin_url: try: snapshot_context = get_snapshot_context(None, origin_type, origin_url, timestamp, visit_id) except Exception: raw_rev_url = reverse('browse-revision', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a revision ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the revision ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_rev_url))) raise NotFoundExc(error_message) origin_info = snapshot_context['origin_info'] snapshot_id = snapshot_context['snapshot_id'] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) if path: file_info = \ service.lookup_directory_with_path(revision['directory'], path) if file_info['type'] == 'dir': dir_id = file_info['target'] else: query_string = 'sha1_git:' + file_info['target'] content_data = request_content(query_string, raise_if_unavailable=False) else: dir_id = revision['directory'] if dir_id: path = '' if path is None else (path + '/') dirs, files = get_directory_entries(dir_id) except Exception as exc: return handle_view_exception(request, exc) revision_data = {} author_name = 'None' revision_data['author'] = 'None' if revision['author']: author_name = revision['author']['name'] or \ revision['author']['fullname'] revision_data['author'] = \ gen_person_link(revision['author']['id'], author_name, snapshot_context) revision_data['committer'] = 'None' if revision['committer']: revision_data['committer'] = \ gen_person_link(revision['committer']['id'], revision['committer']['name'], snapshot_context) revision_data['committer date'] = \ format_utc_iso_date(revision['committer_date']) revision_data['date'] = format_utc_iso_date(revision['date']) revision_data['directory'] = revision['directory'] if snapshot_context: revision_data['snapshot'] = snapshot_id browse_snapshot_link = \ gen_snapshot_link(snapshot_id) revision_data['context-independent snapshot'] = browse_snapshot_link revision_data['context-independent directory'] = \ gen_directory_link(revision['directory']) revision_data['revision'] = sha1_git revision_data['merge'] = revision['merge'] revision_data['metadata'] = escape(json.dumps(revision['metadata'], sort_keys=True, indent=4, separators=(',', ': '))) if origin_info: revision_data['origin type'] = origin_info['type'] revision_data['origin url'] = gen_link(origin_info['url'], origin_info['url']) revision_data['context-independent revision'] = \ gen_revision_link(sha1_git) parents = '' for p in revision['parents']: parent_link = gen_revision_link(p, link_text=None, link_attrs=None, snapshot_context=snapshot_context) parents += parent_link + '
' revision_data['parents'] = mark_safe(parents) revision_data['synthetic'] = revision['synthetic'] revision_data['type'] = revision['type'] message_lines = ['None'] if revision['message']: message_lines = revision['message'].split('\n') parents = [] for p in revision['parents']: parent_url = gen_revision_url(p, snapshot_context) parents.append({'id': p, 'url': parent_url}) path_info = gen_path_info(path) query_params = {'snapshot_id': snapshot_id, 'origin_type': origin_type, 'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id} breadcrumbs = [] breadcrumbs.append({'name': revision['directory'][:7], 'url': reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params)}) for pi in path_info: query_params['path'] = pi['path'] breadcrumbs.append({'name': pi['name'], 'url': reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params)}) vault_cooking = { 'directory_context': False, 'directory_id': None, 'revision_context': True, 'revision_id': sha1_git } swh_objects = [{'type': 'revision', 'id': sha1_git}] content = None content_size = None mimetype = None language = None readme_name = None readme_url = None readme_html = None readmes = {} error_code = 200 error_message = '' error_description = '' if content_data: breadcrumbs[-1]['url'] = None content_size = content_data['length'] mimetype = content_data['mimetype'] if content_data['raw_data']: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] query_params = {} if path: filename = path_info[-1]['name'] query_params['filename'] = path_info[-1]['name'] revision_data['filename'] = filename top_right_link = { 'url': reverse('browse-content-raw', url_args={'query_string': query_string}, query_params=query_params), 'icon': swh_object_icons['content'], 'text': 'Raw File' } swh_objects.append({'type': 'content', 'id': file_info['target']}) error_code = content_data['error_code'] error_message = content_data['error_message'] error_description = content_data['error_description'] else: for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: query_params['path'] = path + d['name'] d['url'] = reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) for f in files: query_params['path'] = path + f['name'] f['url'] = reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) if f['length'] is not None: f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) top_right_link = { 'url': get_revision_log_url(sha1_git, snapshot_context), 'icon': swh_object_icons['revisions history'], 'text': 'History' } vault_cooking['directory_context'] = True vault_cooking['directory_id'] = dir_id swh_objects.append({'type': 'directory', 'id': dir_id}) diff_revision_url = reverse('diff-revision', url_args={'sha1_git': sha1_git}, query_params={'origin_type': origin_type, 'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id}) if snapshot_id: swh_objects.append({'type': 'snapshot', 'id': snapshot_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) heading = 'Revision - %s - %s' %\ (sha1_git[:7], textwrap.shorten(message_lines[0], width=70)) if snapshot_context: context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading += ' - %s' % context_found return render(request, 'browse/revision.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Revision', 'swh_object_metadata': revision_data, 'message_header': message_lines[0], 'message_body': '\n'.join(message_lines[1:]), 'parents': parents, 'snapshot_context': snapshot_context, 'dirs': dirs, 'files': files, 'content': content, 'content_size': content_size, 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'breadcrumbs': breadcrumbs, 'top_right_link': top_right_link, 'vault_cooking': vault_cooking, 'diff_revision_url': diff_revision_url, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': error_code, 'error_message': error_message, 'error_description': error_description}, status=error_code) diff --git a/swh/web/browse/views/snapshot.py b/swh/web/browse/views/snapshot.py index 61dc015a..591edb1c 100644 --- a/swh/web/browse/views/snapshot.py +++ b/swh/web/browse/views/snapshot.py @@ -1,97 +1,97 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import redirect from swh.web.browse.browseurls import browse_route from swh.web.common.utils import reverse from .utils.snapshot_context import ( browse_snapshot_directory, browse_snapshot_content, browse_snapshot_log, browse_snapshot_branches, browse_snapshot_releases ) @browse_route(r'snapshot/(?P[0-9a-f]+)/', view_name='browse-snapshot', checksum_args=['snapshot_id']) def snapshot_browse(request, snapshot_id): """Django view for browsing the content of a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/` """ browse_snapshot_url = reverse('browse-snapshot-directory', url_args={'snapshot_id': snapshot_id}, query_params=request.GET) return redirect(browse_snapshot_url) @browse_route(r'snapshot/(?P[0-9a-f]+)/directory/', r'snapshot/(?P[0-9a-f]+)/directory/(?P.+)/', view_name='browse-snapshot-directory', checksum_args=['snapshot_id']) def snapshot_directory_browse(request, snapshot_id, path=None): """Django view for browsing the content of a directory collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/directory/[(path)/]` """ # noqa origin_type = request.GET.get('origin_type', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) return browse_snapshot_directory(request, snapshot_id=snapshot_id, path=path, origin_type=origin_type, origin_url=origin_url) @browse_route(r'snapshot/(?P[0-9a-f]+)/content/(?P.+)/', view_name='browse-snapshot-content', checksum_args=['snapshot_id']) def snapshot_content_browse(request, snapshot_id, path): """Django view that produces an HTML display of a content collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/content/(path)/` """ # noqa return browse_snapshot_content(request, snapshot_id=snapshot_id, path=path) @browse_route(r'snapshot/(?P[0-9a-f]+)/log/', view_name='browse-snapshot-log', checksum_args=['snapshot_id']) def snapshot_log_browse(request, snapshot_id): """Django view that produces an HTML display of revisions history (aka the commit log) collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/log/` """ # noqa return browse_snapshot_log(request, snapshot_id=snapshot_id) @browse_route(r'snapshot/(?P[0-9a-f]+)/branches/', view_name='browse-snapshot-branches', checksum_args=['snapshot_id']) def snapshot_branches_browse(request, snapshot_id): """Django view that produces an HTML display of the list of releases collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/branches/` """ # noqa return browse_snapshot_branches(request, snapshot_id=snapshot_id) @browse_route(r'snapshot/(?P[0-9a-f]+)/releases/', view_name='browse-snapshot-releases', checksum_args=['snapshot_id']) def snapshot_releases_browse(request, snapshot_id): """Django view that produces an HTML display of the list of releases collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/releases/` """ # noqa return browse_snapshot_releases(request, snapshot_id=snapshot_id) diff --git a/swh/web/browse/views/utils/snapshot_context.py b/swh/web/browse/views/utils/snapshot_context.py index 0d385327..60e8180a 100644 --- a/swh/web/browse/views/utils/snapshot_context.py +++ b/swh/web/browse/views/utils/snapshot_context.py @@ -1,910 +1,910 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information # Utility module implementing Django views for browsing the archive # in a snapshot context. # Its purpose is to factorize code for the views reachable from the # /origin/.* and /snapshot/.* endpoints. from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.utils.html import escape from swh.model.identifiers import snapshot_identifier from swh.web.browse.utils import ( get_snapshot_context, get_directory_entries, gen_directory_link, gen_revision_link, request_content, gen_content_link, prepare_content_for_display, content_display_max_size, format_log_entries, gen_revision_log_link, get_readme_to_display, get_swh_persistent_ids, gen_snapshot_link, process_snapshot_branches ) from swh.web.common import service from swh.web.common.exc import ( handle_view_exception, NotFoundExc ) from swh.web.common.utils import ( reverse, gen_path_info, format_utc_iso_date, swh_object_icons ) _empty_snapshot_id = snapshot_identifier({'branches': {}}) def _get_branch(branches, branch_name, snapshot_id): """ Utility function to get a specific branch from a branches list. Its purpose is to get the default HEAD branch as some software origin (e.g those with svn type) does not have it. In that latter case, check if there is a master branch instead and returns it. """ filtered_branches = \ [b for b in branches if b['name'].endswith(branch_name)] if len(filtered_branches) > 0: return filtered_branches[0] elif branch_name == 'HEAD': filtered_branches = \ [b for b in branches if b['name'].endswith('master')] if len(filtered_branches) > 0: return filtered_branches[0] elif len(branches) > 0: return branches[0] else: # case where a large branches list has been truncated snp = service.lookup_snapshot(snapshot_id, branches_from=branch_name, branches_count=1, target_types=['revision', 'alias']) snp_branch, _ = process_snapshot_branches(snp) if snp_branch: branches.append(snp_branch[0]) return snp_branch[0] return None def _get_release(releases, release_name): """ Utility function to get a specific release from a releases list. Returns None if the release can not be found in the list. """ filtered_releases = \ [r for r in releases if r['name'] == release_name] if len(filtered_releases) > 0: return filtered_releases[0] else: return None def _branch_not_found(branch_type, branch, branches, snapshot_id=None, origin_info=None, timestamp=None, visit_id=None): """ Utility function to raise an exception when a specified branch/release can not be found. """ if branch_type == 'branch': branch_type = 'Branch' branch_type_plural = 'branches' else: branch_type = 'Release' branch_type_plural = 'releases' if snapshot_id and len(branches) == 0: msg = 'Snapshot with id %s has an empty list' \ ' of %s!' % (snapshot_id, branch_type_plural) elif snapshot_id: msg = '%s %s for snapshot with id %s' \ ' not found!' % (branch_type, branch, snapshot_id) elif visit_id and len(branches) == 0: msg = 'Origin with type %s and url %s' \ ' for visit with id %s has an empty list' \ ' of %s!' % (origin_info['type'], origin_info['url'], visit_id, branch_type_plural) elif visit_id: msg = '%s %s associated to visit with' \ ' id %s for origin with type %s and url %s' \ ' not found!' % (branch_type, branch, visit_id, origin_info['type'], origin_info['url']) elif len(branches) == 0: msg = 'Origin with type %s and url %s' \ ' for visit with timestamp %s has an empty list' \ ' of %s!' % (origin_info['type'], origin_info['url'], timestamp, branch_type_plural) else: msg = '%s %s associated to visit with' \ ' timestamp %s for origin with type %s' \ ' and url %s not found!' % (branch_type, branch, timestamp, origin_info['type'], origin_info['url']) raise NotFoundExc(escape(msg)) def _process_snapshot_request(request, snapshot_id=None, origin_type=None, origin_url=None, timestamp=None, path=None, browse_context='directory'): """ Utility function to perform common input request processing for snapshot context views. """ visit_id = request.GET.get('visit_id', None) snapshot_context = get_snapshot_context(snapshot_id, origin_type, origin_url, timestamp, visit_id) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] branches = snapshot_context['branches'] releases = snapshot_context['releases'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] if snapshot_context['visit_info']: timestamp = format_utc_iso_date(snapshot_context['visit_info']['date'], '%Y-%m-%dT%H:%M:%SZ') snapshot_context['timestamp'] = \ format_utc_iso_date(snapshot_context['visit_info']['date']) browse_view_name = 'browse-' + swh_type + '-' + browse_context root_sha1_git = None revision_id = request.GET.get('revision', None) release_name = request.GET.get('release', None) release_id = None branch_name = None snapshot_total_size = sum(snapshot_context['snapshot_size'].values()) if snapshot_total_size and revision_id: revision = service.lookup_revision(revision_id) root_sha1_git = revision['directory'] branches.append({'name': revision_id, 'revision': revision_id, 'directory': root_sha1_git, 'url': None}) branch_name = revision_id query_params['revision'] = revision_id elif snapshot_total_size and release_name: release = _get_release(releases, release_name) try: root_sha1_git = release['directory'] revision_id = release['target'] release_id = release['id'] query_params['release'] = release_name except Exception: _branch_not_found("release", release_name, releases, snapshot_id, origin_info, timestamp, visit_id) elif snapshot_total_size: branch_name = request.GET.get('branch', None) if branch_name: query_params['branch'] = branch_name branch = _get_branch(branches, branch_name or 'HEAD', snapshot_context['snapshot_id']) try: branch_name = branch['name'] revision_id = branch['revision'] root_sha1_git = branch['directory'] except Exception: _branch_not_found("branch", branch_name, branches, snapshot_id, origin_info, timestamp, visit_id) for b in branches: branch_url_args = dict(url_args) branch_query_params = dict(query_params) if 'release' in branch_query_params: del branch_query_params['release'] branch_query_params['branch'] = b['name'] if path: b['path'] = path branch_url_args['path'] = path b['url'] = reverse(browse_view_name, url_args=branch_url_args, query_params=branch_query_params) for r in releases: release_url_args = dict(url_args) release_query_params = dict(query_params) if 'branch' in release_query_params: del release_query_params['branch'] release_query_params['release'] = r['name'] if path: r['path'] = path release_url_args['path'] = path r['url'] = reverse(browse_view_name, url_args=release_url_args, query_params=release_query_params) snapshot_context['query_params'] = query_params snapshot_context['root_sha1_git'] = root_sha1_git snapshot_context['revision_id'] = revision_id snapshot_context['branch'] = branch_name snapshot_context['release'] = release_name snapshot_context['release_id'] = release_id return snapshot_context def browse_snapshot_directory(request, snapshot_id=None, origin_type=None, origin_url=None, timestamp=None, path=None): """ Django view implementation for browsing a directory in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, origin_type, origin_url, timestamp, path, browse_context='directory') # noqa root_sha1_git = snapshot_context['root_sha1_git'] sha1_git = root_sha1_git if root_sha1_git and path: dir_info = service.lookup_directory_with_path(root_sha1_git, path) sha1_git = dir_info['target'] dirs = [] files = [] if sha1_git: dirs, files = get_directory_entries(sha1_git) except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] revision_id = snapshot_context['revision_id'] snapshot_id = snapshot_context['snapshot_id'] path_info = gen_path_info(path) browse_view_name = 'browse-' + swh_type + '-directory' breadcrumbs = [] if root_sha1_git: breadcrumbs.append({'name': root_sha1_git[:7], 'url': reverse(browse_view_name, url_args=url_args, query_params=query_params)}) for pi in path_info: bc_url_args = dict(url_args) bc_url_args['path'] = pi['path'] breadcrumbs.append({'name': pi['name'], 'url': reverse(browse_view_name, url_args=bc_url_args, query_params=query_params)}) path = '' if path is None else (path + '/') for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: bc_url_args = dict(url_args) bc_url_args['path'] = path + d['name'] d['url'] = reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) sum_file_sizes = 0 readmes = {} browse_view_name = 'browse-' + swh_type + '-content' for f in files: bc_url_args = dict(url_args) bc_url_args['path'] = path + f['name'] f['url'] = reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) if f['length'] is not None: sum_file_sizes += f['length'] f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) browse_view_name = 'browse-' + swh_type + '-log' history_url = None if snapshot_id != _empty_snapshot_id: history_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) nb_files = None nb_dirs = None dir_path = None if root_sha1_git: nb_files = len(files) nb_dirs = len(dirs) sum_file_sizes = filesizeformat(sum_file_sizes) dir_path = '/' + path browse_dir_link = gen_directory_link(sha1_git) browse_rev_link = gen_revision_link(revision_id) browse_snp_link = gen_snapshot_link(snapshot_id) dir_metadata = {'directory': sha1_git, 'context-independent directory': browse_dir_link, 'number of regular files': nb_files, 'number of subdirectories': nb_dirs, 'sum of regular file sizes': sum_file_sizes, 'path': dir_path, 'revision': revision_id, 'context-independent revision': browse_rev_link, 'snapshot': snapshot_id, 'context-independent snapshot': browse_snp_link} if origin_info: dir_metadata['origin type'] = origin_info['type'] dir_metadata['origin url'] = origin_info['url'] dir_metadata['origin visit date'] = format_utc_iso_date(visit_info['date']) # noqa vault_cooking = { 'directory_context': True, 'directory_id': sha1_git, 'revision_context': True, 'revision_id': revision_id } swh_objects = [{'type': 'directory', 'id': sha1_git}, {'type': 'revision', 'id': revision_id}, {'type': 'snapshot', 'id': snapshot_id}] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = 'Directory - %s - %s - %s' %\ (dir_path, snapshot_context['branch'], context_found) return render(request, 'browse/directory.html', {'heading': heading, 'swh_object_name': 'Directory', 'swh_object_metadata': dir_metadata, 'dirs': dirs, 'files': files, 'breadcrumbs': breadcrumbs if root_sha1_git else [], 'top_right_link': { 'url': history_url, 'icon': swh_object_icons['revisions history'], 'text': 'History' }, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'snapshot_context': snapshot_context, 'vault_cooking': vault_cooking, 'show_actions_menu': True, 'swh_ids': swh_ids}) def browse_snapshot_content(request, snapshot_id=None, origin_type=None, origin_url=None, timestamp=None, path=None): """ Django view implementation for browsing a content in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, origin_type, origin_url, timestamp, path, browse_context='content') root_sha1_git = snapshot_context['root_sha1_git'] sha1_git = None query_string = None content_data = None split_path = path.split('/') filename = split_path[-1] filepath = path[:-len(filename)] if root_sha1_git: content_info = service.lookup_directory_with_path(root_sha1_git, path) sha1_git = content_info['target'] query_string = 'sha1_git:' + sha1_git content_data = request_content(query_string, raise_if_unavailable=False) if filepath: dir_info = service.lookup_directory_with_path(root_sha1_git, filepath) directory_id = dir_info['target'] else: directory_id = root_sha1_git except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] revision_id = snapshot_context['revision_id'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] snapshot_id = snapshot_context['snapshot_id'] content = None language = None mimetype = None if content_data and content_data['raw_data'] is not None: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] browse_view_name = 'browse-' + swh_type + '-directory' breadcrumbs = [] path_info = gen_path_info(filepath) if root_sha1_git: breadcrumbs.append({'name': root_sha1_git[:7], 'url': reverse(browse_view_name, url_args=url_args, query_params=query_params)}) for pi in path_info: bc_url_args = dict(url_args) bc_url_args['path'] = pi['path'] breadcrumbs.append({'name': pi['name'], 'url': reverse(browse_view_name, url_args=bc_url_args, query_params=query_params)}) breadcrumbs.append({'name': filename, 'url': None}) browse_content_link = gen_content_link(sha1_git) content_raw_url = None if query_string: content_raw_url = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params={'filename': filename}) browse_rev_link = gen_revision_link(revision_id) browse_dir_link = gen_directory_link(directory_id) content_metadata = { 'context-independent content': browse_content_link, 'path': None, 'filename': None, 'directory': directory_id, 'context-independent directory': browse_dir_link, 'revision': revision_id, 'context-independent revision': browse_rev_link, 'snapshot': snapshot_id } cnt_sha1_git = None content_size = None error_code = 200 error_description = '' error_message = '' if content_data: content_metadata['sha1'] = \ content_data['checksums']['sha1'] content_metadata['sha1_git'] = \ content_data['checksums']['sha1_git'] content_metadata['sha256'] = \ content_data['checksums']['sha256'] content_metadata['blake2s256'] = \ content_data['checksums']['blake2s256'] content_metadata['mimetype'] = content_data['mimetype'] content_metadata['encoding'] = content_data['encoding'] content_metadata['size'] = filesizeformat(content_data['length']) content_metadata['language'] = content_data['language'] content_metadata['licenses'] = content_data['licenses'] content_metadata['path'] = '/' + filepath content_metadata['filename'] = filename cnt_sha1_git = content_data['checksums']['sha1_git'] content_size = content_data['length'] error_code = content_data['error_code'] error_message = content_data['error_message'] error_description = content_data['error_description'] if origin_info: content_metadata['origin type'] = origin_info['type'] content_metadata['origin url'] = origin_info['url'] content_metadata['origin visit date'] = format_utc_iso_date(visit_info['date']) # noqa browse_snapshot_link = gen_snapshot_link(snapshot_id) content_metadata['context-independent snapshot'] = browse_snapshot_link swh_objects = [{'type': 'content', 'id': cnt_sha1_git}, {'type': 'revision', 'id': revision_id}, {'type': 'snapshot', 'id': snapshot_id}] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) content_path = '/'.join([bc['name'] for bc in breadcrumbs]) context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = 'Content - %s - %s - %s' %\ (content_path, snapshot_context['branch'], context_found) return render(request, 'browse/content.html', {'heading': heading, 'swh_object_name': 'Content', 'swh_object_metadata': content_metadata, 'content': content, 'content_size': content_size, 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'breadcrumbs': breadcrumbs if root_sha1_git else [], 'top_right_link': { 'url': content_raw_url, 'icon': swh_object_icons['content'], 'text': 'Raw File' }, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': error_code, 'error_message': error_message, 'error_description': error_description}, status=error_code) PER_PAGE = 100 def browse_snapshot_log(request, snapshot_id=None, origin_type=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a revision history in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, origin_type, origin_url, timestamp, browse_context='log') # noqa revision_id = snapshot_context['revision_id'] per_page = int(request.GET.get('per_page', PER_PAGE)) offset = int(request.GET.get('offset', 0)) revs_ordering = request.GET.get('revs_ordering', 'committer_date') session_key = 'rev_%s_log_ordering_%s' % (revision_id, revs_ordering) rev_log_session = request.session.get(session_key, None) rev_log = [] revs_walker_state = None if rev_log_session: rev_log = rev_log_session['rev_log'] revs_walker_state = rev_log_session['revs_walker_state'] if len(rev_log) < offset+per_page: revs_walker = \ service.get_revisions_walker(revs_ordering, revision_id, max_revs=offset+per_page+1, state=revs_walker_state) rev_log += [rev['id'] for rev in revs_walker] revs_walker_state = revs_walker.export_state() revs = rev_log[offset:offset+per_page] revision_log = service.lookup_revision_multiple(revs) request.session[session_key] = { 'rev_log': rev_log, 'revs_walker_state': revs_walker_state } except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] snapshot_id = snapshot_context['snapshot_id'] query_params['per_page'] = per_page revs_ordering = request.GET.get('revs_ordering', '') query_params['revs_ordering'] = revs_ordering browse_view_name = 'browse-' + swh_type + '-log' prev_log_url = None if len(rev_log) > offset + per_page: query_params['offset'] = offset + per_page prev_log_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) next_log_url = None if offset != 0: query_params['offset'] = offset - per_page next_log_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) revision_log_data = format_log_entries(revision_log, per_page, snapshot_context) browse_rev_link = gen_revision_link(revision_id) browse_log_link = gen_revision_log_link(revision_id) browse_snp_link = gen_snapshot_link(snapshot_id) revision_metadata = { 'context-independent revision': browse_rev_link, 'context-independent revision history': browse_log_link, 'context-independent snapshot': browse_snp_link, 'snapshot': snapshot_id } if origin_info: revision_metadata['origin type'] = origin_info['type'] revision_metadata['origin url'] = origin_info['url'] revision_metadata['origin visit date'] = format_utc_iso_date(visit_info['date']) # noqa swh_objects = [{'type': 'revision', 'id': revision_id}, {'type': 'snapshot', 'id': snapshot_id}] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = 'Revision history - %s - %s' %\ (snapshot_context['branch'], context_found) return render(request, 'browse/revision-log.html', {'heading': heading, 'swh_object_name': 'Revisions history', 'swh_object_metadata': revision_metadata, 'revision_log': revision_log_data, 'revs_ordering': revs_ordering, 'next_log_url': next_log_url, 'prev_log_url': prev_log_url, 'breadcrumbs': None, 'top_right_link': None, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids}) def browse_snapshot_branches(request, snapshot_id=None, origin_type=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a list of branches in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, origin_type, origin_url, timestamp) branches_bc = request.GET.get('branches_breadcrumbs', '') branches_bc = \ branches_bc.split(',') if branches_bc else [] branches_from = branches_bc[-1] if branches_bc else '' swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] browse_view_name = 'browse-' + swh_type + '-directory' snapshot = \ service.lookup_snapshot(snapshot_context['snapshot_id'], branches_from, PER_PAGE+1, target_types=['revision', 'alias']) displayed_branches, _ = process_snapshot_branches(snapshot) except Exception as exc: return handle_view_exception(request, exc) for branch in displayed_branches: if snapshot_id: revision_url = reverse('browse-revision', url_args={'sha1_git': branch['revision']}, query_params={'snapshot_id': snapshot_id}) else: revision_url = reverse('browse-revision', url_args={'sha1_git': branch['revision']}, query_params={'origin_type': origin_type, 'origin': origin_info['url']}) query_params['branch'] = branch['name'] directory_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) del query_params['branch'] branch['revision_url'] = revision_url branch['directory_url'] = directory_url browse_view_name = 'browse-' + swh_type + '-branches' prev_branches_url = None next_branches_url = None if branches_bc: query_params_prev = dict(query_params) query_params_prev['branches_breadcrumbs'] = \ ','.join(branches_bc[:-1]) prev_branches_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_prev) elif branches_from: prev_branches_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) if len(displayed_branches) > PER_PAGE: query_params_next = dict(query_params) next_branch = displayed_branches[-1]['name'] del displayed_branches[-1] branches_bc.append(next_branch) query_params_next['branches_breadcrumbs'] = \ ','.join(branches_bc) next_branches_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_next) heading = 'Branches - ' if origin_info: heading += 'origin: %s' % origin_info['url'] else: heading += 'snapshot: %s' % snapshot_id return render(request, 'browse/branches.html', {'heading': heading, 'swh_object_name': 'Branches', 'swh_object_metadata': {}, 'top_right_link': None, 'displayed_branches': displayed_branches, 'prev_branches_url': prev_branches_url, 'next_branches_url': next_branches_url, 'snapshot_context': snapshot_context}) def browse_snapshot_releases(request, snapshot_id=None, origin_type=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a list of releases in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, origin_type, origin_url, timestamp) rel_bc = request.GET.get('releases_breadcrumbs', '') rel_bc = \ rel_bc.split(',') if rel_bc else [] rel_from = rel_bc[-1] if rel_bc else '' swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] snapshot = \ service.lookup_snapshot(snapshot_context['snapshot_id'], rel_from, PER_PAGE+1, target_types=['release', 'alias']) _, displayed_releases = process_snapshot_branches(snapshot) except Exception as exc: return handle_view_exception(request, exc) for release in displayed_releases: if snapshot_id: query_params_tgt = {'snapshot_id': snapshot_id} else: query_params_tgt = {'origin': origin_info['url']} release_url = reverse('browse-release', url_args={'sha1_git': release['id']}, query_params=query_params_tgt) target_url = '' if release['target_type'] == 'revision': target_url = reverse('browse-revision', url_args={'sha1_git': release['target']}, query_params=query_params_tgt) elif release['target_type'] == 'directory': target_url = reverse('browse-directory', url_args={'sha1_git': release['target']}, query_params=query_params_tgt) elif release['target_type'] == 'content': target_url = reverse('browse-content', url_args={'query_string': release['target']}, query_params=query_params_tgt) elif release['target_type'] == 'release': target_url = reverse('browse-release', url_args={'sha1_git': release['target']}, query_params=query_params_tgt) release['release_url'] = release_url release['target_url'] = target_url browse_view_name = 'browse-' + swh_type + '-releases' prev_releases_url = None next_releases_url = None if rel_bc: query_params_prev = dict(query_params) query_params_prev['releases_breadcrumbs'] = \ ','.join(rel_bc[:-1]) prev_releases_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_prev) elif rel_from: prev_releases_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) if len(displayed_releases) > PER_PAGE: query_params_next = dict(query_params) next_rel = displayed_releases[-1]['branch_name'] del displayed_releases[-1] rel_bc.append(next_rel) query_params_next['releases_breadcrumbs'] = \ ','.join(rel_bc) next_releases_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_next) heading = 'Releases - ' if origin_info: heading += 'origin: %s' % origin_info['url'] else: heading += 'snapshot: %s' % snapshot_id return render(request, 'browse/releases.html', {'heading': heading, 'top_panel_visible': False, 'top_panel_collapsible': False, 'swh_object_name': 'Releases', 'swh_object_metadata': {}, 'top_right_link': None, 'displayed_releases': displayed_releases, 'prev_releases_url': prev_releases_url, 'next_releases_url': next_releases_url, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': False}) diff --git a/swh/web/common/converters.py b/swh/web/common/converters.py index 30cdb5fc..c09663ed 100644 --- a/swh/web/common/converters.py +++ b/swh/web/common/converters.py @@ -1,374 +1,374 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import json from swh.model import hashutil from swh.core.utils import decode_with_escape def _group_checksums(data): """Groups checksums values computed from hash functions used in swh and stored in data dict under a single entry 'checksums' """ if data: checksums = {} for hash in hashutil.ALGORITHMS: if hash in data and data[hash]: checksums[hash] = data[hash] del data[hash] if len(checksums) > 0: data['checksums'] = checksums def fmap(f, data): """Map f to data at each level. This must keep the origin data structure type: - map -> map - dict -> dict - list -> list - None -> None Args: f: function that expects one argument. data: data to traverse to apply the f function. list, map, dict or bare value. Returns: The same data-structure with modified values by the f function. """ if data is None: return data if isinstance(data, map): return map(lambda y: fmap(f, y), (x for x in data)) if isinstance(data, list): return [fmap(f, x) for x in data] if isinstance(data, dict): return {k: fmap(f, v) for (k, v) in data.items()} return f(data) def from_swh(dict_swh, hashess={}, bytess={}, dates={}, blacklist={}, removables_if_empty={}, empty_dict={}, empty_list={}, convert={}, convert_fn=lambda x: x): """Convert from a swh dictionary to something reasonably json serializable. Args: dict_swh: the origin dictionary needed to be transformed hashess: list/set of keys representing hashes values (sha1, sha256, sha1_git, etc...) as bytes. Those need to be transformed in hexadecimal string bytess: list/set of keys representing bytes values which needs to be decoded blacklist: set of keys to filter out from the conversion convert: set of keys whose associated values need to be converted using convert_fn convert_fn: the conversion function to apply on the value of key in 'convert' The remaining keys are copied as is in the output. Returns: dictionary equivalent as dict_swh only with its keys converted. """ def convert_hashes_bytes(v): """v is supposedly a hash as bytes, returns it converted in hex. """ if isinstance(v, bytes): return hashutil.hash_to_hex(v) return v def convert_bytes(v): """v is supposedly a bytes string, decode as utf-8. FIXME: Improve decoding policy. If not utf-8, break! """ if isinstance(v, bytes): return v.decode('utf-8') return v def convert_date(v): """ Args: v (dict or datatime): either: - a dict with three keys: - timestamp (dict or integer timestamp) - offset - negative_utc - or, a datetime We convert it to a human-readable string """ if not v: return v if isinstance(v, datetime.datetime): return v.isoformat() tz = datetime.timezone(datetime.timedelta(minutes=v['offset'])) swh_timestamp = v['timestamp'] if isinstance(swh_timestamp, dict): date = datetime.datetime.fromtimestamp( swh_timestamp['seconds'], tz=tz) else: date = datetime.datetime.fromtimestamp( swh_timestamp, tz=tz) datestr = date.isoformat() if v['offset'] == 0 and v['negative_utc']: # remove the rightmost + and replace it with a - return '-'.join(datestr.rsplit('+', 1)) return datestr if not dict_swh: return dict_swh new_dict = {} for key, value in dict_swh.items(): if key in blacklist or (key in removables_if_empty and not value): continue if key in dates: new_dict[key] = convert_date(value) elif key in convert: new_dict[key] = convert_fn(value) elif isinstance(value, dict): new_dict[key] = from_swh(value, hashess=hashess, bytess=bytess, dates=dates, blacklist=blacklist, removables_if_empty=removables_if_empty, empty_dict=empty_dict, empty_list=empty_list, convert=convert, convert_fn=convert_fn) elif key in hashess: new_dict[key] = fmap(convert_hashes_bytes, value) elif key in bytess: try: new_dict[key] = fmap(convert_bytes, value) except UnicodeDecodeError: if 'decoding_failures' not in new_dict: new_dict['decoding_failures'] = [key] else: new_dict['decoding_failures'].append(key) new_dict[key] = fmap(decode_with_escape, value) elif key in empty_dict and not value: new_dict[key] = {} elif key in empty_list and not value: new_dict[key] = [] else: new_dict[key] = value _group_checksums(new_dict) return new_dict def from_origin(origin): """Convert from a swh origin to an origin dictionary. """ return from_swh(origin) def from_release(release): """Convert from a swh release to a json serializable release dictionary. Args: release (dict): dictionary with keys: - id: identifier of the revision (sha1 in bytes) - revision: identifier of the revision the release points to (sha1 in bytes) comment: release's comment message (bytes) name: release's name (string) author: release's author identifier (swh's id) synthetic: the synthetic property (boolean) Returns: dict: Release dictionary with the following keys: - id: hexadecimal sha1 (string) - revision: hexadecimal sha1 (string) - comment: release's comment message (string) - name: release's name (string) - author: release's author identifier (swh's id) - synthetic: the synthetic property (boolean) """ return from_swh( release, hashess={'id', 'target'}, bytess={'message', 'name', 'fullname', 'email'}, dates={'date'}, ) class SWHMetadataEncoder(json.JSONEncoder): """Special json encoder for metadata field which can contain bytes encoded value. """ def default(self, obj): if isinstance(obj, bytes): try: return obj.decode('utf-8') except UnicodeDecodeError: # fallback to binary representation to avoid display errors return repr(obj) # Let the base class default method raise the TypeError return json.JSONEncoder.default(self, obj) def convert_revision_metadata(metadata): """Convert json specific dict to a json serializable one. """ if not metadata: return {} return json.loads(json.dumps(metadata, cls=SWHMetadataEncoder)) def from_revision(revision): """Convert from a swh revision to a json serializable revision dictionary. Args: revision (dict): dict with keys: - id: identifier of the revision (sha1 in bytes) - directory: identifier of the directory the revision points to (sha1 in bytes) - author_name, author_email: author's revision name and email - committer_name, committer_email: committer's revision name and email - message: revision's message - date, date_offset: revision's author date - committer_date, committer_date_offset: revision's commit date - parents: list of parents for such revision - synthetic: revision's property nature - type: revision's type (git, tar or dsc at the moment) - metadata: if the revision is synthetic, this can reference dynamic properties. Returns: dict: Revision dictionary with the same keys as inputs, except: - sha1s are in hexadecimal strings (id, directory) - bytes are decoded in string (author_name, committer_name, author_email, committer_email) Remaining keys are left as is """ revision = from_swh(revision, hashess={'id', 'directory', 'parents', 'children'}, bytess={'name', 'fullname', 'email'}, convert={'metadata'}, convert_fn=convert_revision_metadata, dates={'date', 'committer_date'}) if revision: if 'parents' in revision: revision['merge'] = len(revision['parents']) > 1 if 'message' in revision: try: revision['message'] = revision['message'].decode('utf-8') except UnicodeDecodeError: revision['message_decoding_failed'] = True revision['message'] = None return revision def from_content(content): """Convert swh content to serializable content dictionary. """ return from_swh(content, hashess={'sha1', 'sha1_git', 'sha256', 'blake2s256'}, blacklist={'ctime'}, convert={'status'}, convert_fn=lambda v: 'absent' if v == 'hidden' else v) def from_person(person): """Convert swh person to serializable person dictionary. """ return from_swh(person, bytess={'name', 'fullname', 'email'}) def from_origin_visit(visit): """Convert swh origin_visit to serializable origin_visit dictionary. """ ov = from_swh(visit, hashess={'target', 'snapshot'}, bytess={'branch'}, dates={'date'}, empty_dict={'metadata'}) return ov def from_snapshot(snapshot): """Convert swh snapshot to serializable snapshot dictionary. """ sv = from_swh(snapshot, hashess={'id', 'target'}, bytess={'next_branch'}) if sv and 'branches' in sv: sv['branches'] = { decode_with_escape(k): v for k, v in sv['branches'].items() } for k, v in snapshot['branches'].items(): # alias target existing branch names, not a sha1 if v and v['target_type'] == 'alias': branch = decode_with_escape(k) target = decode_with_escape(v['target']) sv['branches'][branch]['target'] = target return sv def from_directory_entry(dir_entry): """Convert swh person to serializable person dictionary. """ return from_swh(dir_entry, hashess={'dir_id', 'sha1_git', 'sha1', 'sha256', 'blake2s256', 'target'}, bytess={'name'}, removables_if_empty={ 'sha1', 'sha1_git', 'sha256', 'blake2s256', 'status'}, convert={'status'}, convert_fn=lambda v: 'absent' if v == 'hidden' else v) def from_filetype(content_entry): """Convert swh person to serializable person dictionary. """ return from_swh(content_entry, hashess={'id'}) diff --git a/swh/web/common/exc.py b/swh/web/common/exc.py index 9fd94202..6be3b09d 100644 --- a/swh/web/common/exc.py +++ b/swh/web/common/exc.py @@ -1,123 +1,123 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import traceback from django.http import HttpResponse from django.shortcuts import render from django.utils.safestring import mark_safe from django.utils.html import escape from swh.web.config import get_config class BadInputExc(ValueError): """Wrong request to the api. Example: Asking a content with the wrong identifier format. """ pass class NotFoundExc(Exception): """Good request to the api but no result were found. Example: Asking a content with the right identifier format but that content does not exist. """ pass class ForbiddenExc(Exception): """Good request to the api, forbidden result to return due to enforce policy. Example: Asking for a raw content which exists but whose mimetype is not text. """ pass http_status_code_message = { 400: 'Bad Request', 401: 'Unauthorized', 403: 'Access Denied', 404: 'Resource not found', 500: 'Internal Server Error', 501: 'Not Implemented', 502: 'Bad Gateway', 503: 'Service unavailable' } def _generate_error_page(request, error_code, error_description): return render(request, 'error.html', {'error_code': error_code, 'error_message': http_status_code_message[error_code], 'error_description': mark_safe(error_description)}, status=error_code) def swh_handle400(request): """ Custom Django HTTP error 400 handler for swh-web. """ error_description = 'The server cannot process the request to %s due to '\ 'something that is perceived to be a client error.' %\ escape(request.META['PATH_INFO']) return _generate_error_page(request, 400, error_description) def swh_handle403(request): """ Custom Django HTTP error 403 handler for swh-web. """ error_description = 'The resource %s requires an authentication.' %\ escape(request.META['PATH_INFO']) return _generate_error_page(request, 403, error_description) def swh_handle404(request): """ Custom Django HTTP error 404 handler for swh-web. """ error_description = 'The resource %s could not be found on the server.' %\ escape(request.META['PATH_INFO']) return _generate_error_page(request, 404, error_description) def swh_handle500(request): """ Custom Django HTTP error 500 handler for swh-web. """ error_description = 'An unexpected condition was encountered when '\ 'requesting resource %s.' %\ escape(request.META['PATH_INFO']) return _generate_error_page(request, 500, error_description) def handle_view_exception(request, exc, html_response=True): """ Function used to generate an error page when an exception was raised inside a swh-web browse view. """ error_code = 500 error_description = '%s: %s' % (type(exc).__name__, str(exc)) if get_config()['debug']: error_description = traceback.format_exc() if isinstance(exc, BadInputExc): error_code = 400 if isinstance(exc, ForbiddenExc): error_code = 403 if isinstance(exc, NotFoundExc): error_code = 404 if html_response: return _generate_error_page(request, error_code, error_description) else: return HttpResponse(error_description, content_type='text/plain', status=error_code) diff --git a/swh/web/common/models.py b/swh/web/common/models.py index 18de0631..cfffb784 100644 --- a/swh/web/common/models.py +++ b/swh/web/common/models.py @@ -1,92 +1,92 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.db import models class SaveAuthorizedOrigin(models.Model): """ Model table holding origin urls authorized to be loaded into the archive. """ url = models.CharField(max_length=200, null=False) class Meta: app_label = 'swh.web.common' db_table = 'save_authorized_origin' def __str__(self): return self.url class SaveUnauthorizedOrigin(models.Model): """ Model table holding origin urls not authorized to be loaded into the archive. """ url = models.CharField(max_length=200, null=False) class Meta: app_label = 'swh.web.common' db_table = 'save_unauthorized_origin' def __str__(self): return self.url SAVE_REQUEST_ACCEPTED = 'accepted' SAVE_REQUEST_REJECTED = 'rejected' SAVE_REQUEST_PENDING = 'pending' SAVE_REQUEST_STATUS = [ (SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_ACCEPTED), (SAVE_REQUEST_REJECTED, SAVE_REQUEST_REJECTED), (SAVE_REQUEST_PENDING, SAVE_REQUEST_PENDING) ] SAVE_TASK_NOT_CREATED = 'not created' SAVE_TASK_NOT_YET_SCHEDULED = 'not yet scheduled' SAVE_TASK_SCHEDULED = 'scheduled' SAVE_TASK_SUCCEED = 'succeed' SAVE_TASK_FAILED = 'failed' SAVE_TASK_RUNNING = 'running' SAVE_TASK_STATUS = [ (SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_CREATED), (SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED), (SAVE_TASK_SCHEDULED, SAVE_TASK_SCHEDULED), (SAVE_TASK_SUCCEED, SAVE_TASK_SUCCEED), (SAVE_TASK_FAILED, SAVE_TASK_FAILED), (SAVE_TASK_RUNNING, SAVE_TASK_RUNNING) ] class SaveOriginRequest(models.Model): """ Model table holding all the save origin requests issued by users. """ id = models.BigAutoField(primary_key=True) request_date = models.DateTimeField(auto_now_add=True) origin_type = models.CharField(max_length=200, null=False) origin_url = models.CharField(max_length=200, null=False) status = models.TextField(choices=SAVE_REQUEST_STATUS, default=SAVE_REQUEST_PENDING) loading_task_id = models.IntegerField(default=-1) visit_date = models.DateTimeField(null=True) loading_task_status = models.TextField(choices=SAVE_TASK_STATUS, default=SAVE_TASK_NOT_CREATED) class Meta: app_label = 'swh.web.common' db_table = 'save_origin_request' ordering = ['-id'] def __str__(self): return str({'id': self.id, 'request_date': self.request_date, 'origin_type': self.origin_type, 'origin_url': self.origin_url, 'status': self.status, 'loading_task_id': self.loading_task_id, 'visit_date': self.visit_date}) diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py index b9e853cb..667e0f2a 100644 --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -1,401 +1,401 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from bisect import bisect_right from datetime import datetime, timezone from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError from django.core.validators import URLValidator from django.utils.html import escape from swh.web import config from swh.web.common import service from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc from swh.web.common.models import ( SaveUnauthorizedOrigin, SaveAuthorizedOrigin, SaveOriginRequest, SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, SAVE_REQUEST_PENDING, SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_SCHEDULED, SAVE_TASK_SUCCEED, SAVE_TASK_FAILED, SAVE_TASK_RUNNING ) from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import parse_timestamp from swh.scheduler.utils import create_oneshot_task_dict scheduler = config.scheduler() def get_origin_save_authorized_urls(): """ Get the list of origin url prefixes authorized to be immediately loaded into the archive (whitelist). Returns: list: The list of authorized origin url prefix """ return [origin.url for origin in SaveAuthorizedOrigin.objects.all()] def get_origin_save_unauthorized_urls(): """ Get the list of origin url prefixes forbidden to be loaded into the archive (blacklist). Returns: list: the list of unauthorized origin url prefix """ return [origin.url for origin in SaveUnauthorizedOrigin.objects.all()] def can_save_origin(origin_url): """ Check if a software origin can be saved into the archive. Based on the origin url, the save request will be either: * immediately accepted if the url is whitelisted * rejected if the url is blacklisted * put in pending state for manual review otherwise Args: origin_url (str): the software origin url to check Returns: str: the origin save request status, either **accepted**, **rejected** or **pending** """ # origin url may be blacklisted for url_prefix in get_origin_save_unauthorized_urls(): if origin_url.startswith(url_prefix): return SAVE_REQUEST_REJECTED # if the origin url is in the white list, it can be immediately saved for url_prefix in get_origin_save_authorized_urls(): if origin_url.startswith(url_prefix): return SAVE_REQUEST_ACCEPTED # otherwise, the origin url needs to be manually verified return SAVE_REQUEST_PENDING # map origin type to scheduler task # TODO: do not hardcode the task name here (T1157) _origin_type_task = { 'git': 'load-git', 'hg': 'load-hg', 'svn': 'load-svn' } # map scheduler task status to origin save status _save_task_status = { 'next_run_not_scheduled': SAVE_TASK_NOT_YET_SCHEDULED, 'next_run_scheduled': SAVE_TASK_SCHEDULED, 'completed': SAVE_TASK_SUCCEED, 'disabled': SAVE_TASK_FAILED } def get_savable_origin_types(): return sorted(list(_origin_type_task.keys())) def _check_origin_type_savable(origin_type): """ Get the list of software origin types that can be loaded through a save request. Returns: list: the list of saveable origin types """ allowed_origin_types = ', '.join(get_savable_origin_types()) if origin_type not in _origin_type_task: raise BadInputExc('Origin of type %s can not be saved! ' 'Allowed types are the following: %s' % (origin_type, allowed_origin_types)) _validate_url = URLValidator(schemes=['http', 'https', 'svn', 'git']) def _check_origin_url_valid(origin_url): try: _validate_url(origin_url) except ValidationError: raise BadInputExc('The provided origin url (%s) is not valid!' % escape(origin_url)) def _get_visit_info_for_save_request(save_request): visit_date = None visit_status = None try: origin = {'type': save_request.origin_type, 'url': save_request.origin_url} origin_info = service.lookup_origin(origin) origin_visits = get_origin_visits(origin_info) visit_dates = [parse_timestamp(v['date']) for v in origin_visits] i = bisect_right(visit_dates, save_request.request_date) if i != len(visit_dates): visit_date = visit_dates[i] visit_status = origin_visits[i]['status'] if origin_visits[i]['status'] == 'ongoing': visit_date = None except Exception: pass return visit_date, visit_status def _check_visit_update_status(save_request, save_task_status): visit_date, visit_status = _get_visit_info_for_save_request(save_request) save_request.visit_date = visit_date # visit has been performed, mark the saving task as succeed if visit_date and visit_status is not None: save_task_status = SAVE_TASK_SUCCEED elif visit_status == 'ongoing': save_task_status = SAVE_TASK_RUNNING else: time_now = datetime.now(tz=timezone.utc) time_delta = time_now - save_request.request_date # consider the task as failed if it is still in scheduled state # 30 days after its submission if time_delta.days > 30: save_task_status = SAVE_TASK_FAILED return visit_date, save_task_status def _save_request_dict(save_request, task=None): must_save = False visit_date = save_request.visit_date # save task still in scheduler db if task: save_task_status = _save_task_status[task['status']] if save_task_status in (SAVE_TASK_FAILED, SAVE_TASK_SUCCEED) \ and not visit_date: visit_date, _ = _get_visit_info_for_save_request(save_request) save_request.visit_date = visit_date must_save = True # Ensure last origin visit is available in database # before reporting the task execution as successful if save_task_status == SAVE_TASK_SUCCEED and not visit_date: save_task_status = SAVE_TASK_SCHEDULED # Check tasks still marked as scheduled / not yet scheduled if save_task_status in (SAVE_TASK_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED): visit_date, save_task_status = _check_visit_update_status( save_request, save_task_status) # save task may have been archived else: save_task_status = save_request.loading_task_status if save_task_status in (SAVE_TASK_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED): visit_date, save_task_status = _check_visit_update_status( save_request, save_task_status) else: save_task_status = save_request.loading_task_status if save_request.loading_task_status != save_task_status: save_request.loading_task_status = save_task_status must_save = True if must_save: save_request.save() return {'id': save_request.id, 'origin_type': save_request.origin_type, 'origin_url': save_request.origin_url, 'save_request_date': save_request.request_date.isoformat(), 'save_request_status': save_request.status, 'save_task_status': save_task_status, 'visit_date': visit_date.isoformat() if visit_date else None} def create_save_origin_request(origin_type, origin_url): """ Create a loading task to save a software origin into the archive. This function aims to create a software origin loading task trough the use of the swh-scheduler component. First, some checks are performed to see if the origin type and url are valid but also if the the save request can be accepted. If those checks passed, the loading task is then created. Otherwise, the save request is put in pending or rejected state. All the submitted save requests are logged into the swh-web database to keep track of them. Args: origin_type (str): the type of origin to save (currently only ``git`` but ``svn`` and ``hg`` will soon be available) origin_url (str): the url of the origin to save Raises: BadInputExc: the origin type or url is invalid ForbiddenExc: the provided origin url is blacklisted Returns: dict: A dict describing the save request with the following keys: * **origin_type**: the type of the origin to save * **origin_url**: the url of the origin * **save_request_date**: the date the request was submitted * **save_request_status**: the request status, either **accepted**, **rejected** or **pending** * **save_task_status**: the origin loading task status, either **not created**, **not yet scheduled**, **scheduled**, **succeed** or **failed** """ _check_origin_type_savable(origin_type) _check_origin_url_valid(origin_url) save_request_status = can_save_origin(origin_url) task = None # if the origin save request is accepted, create a scheduler # task to load it into the archive if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority kwargs = {'priority': 'high'} # set task parameters according to the origin type if origin_type == 'git': kwargs['repo_url'] = origin_url elif origin_type == 'hg': kwargs['origin_url'] = origin_url elif origin_type == 'svn': kwargs['origin_url'] = origin_url kwargs['svn_url'] = origin_url sor = None # get list of previously sumitted save requests current_sors = \ list(SaveOriginRequest.objects.filter(origin_type=origin_type, origin_url=origin_url)) can_create_task = False # if no save requests previously submitted, create the scheduler task if not current_sors: can_create_task = True else: # get the latest submitted save request sor = current_sors[0] # if it was in pending state, we need to create the scheduler task # and update the save request info in the database if sor.status == SAVE_REQUEST_PENDING: can_create_task = True # a task has already been created to load the origin elif sor.loading_task_id != -1: # get the scheduler task and its status tasks = scheduler.get_tasks([sor.loading_task_id]) task = tasks[0] if tasks else None task_status = _save_request_dict(sor, task)['save_task_status'] # create a new scheduler task only if the previous one has been # already executed if task_status == SAVE_TASK_FAILED or \ task_status == SAVE_TASK_SUCCEED: can_create_task = True sor = None else: can_create_task = False if can_create_task: # effectively create the scheduler task task_dict = create_oneshot_task_dict( _origin_type_task[origin_type], **kwargs) task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted if sor: sor.status = SAVE_REQUEST_ACCEPTED sor.loading_task_id = task['id'] sor.save() else: sor = SaveOriginRequest.objects.create(origin_type=origin_type, origin_url=origin_url, status=save_request_status, # noqa loading_task_id=task['id']) # noqa # save request must be manually reviewed for acceptation elif save_request_status == SAVE_REQUEST_PENDING: # check if there is already such a save request already submitted, # no need to add it to the database in that case try: sor = SaveOriginRequest.objects.get(origin_type=origin_type, origin_url=origin_url, status=save_request_status) # if not add it to the database except ObjectDoesNotExist: sor = SaveOriginRequest.objects.create(origin_type=origin_type, origin_url=origin_url, status=save_request_status) # origin can not be saved as its url is blacklisted, # log the request to the database anyway else: sor = SaveOriginRequest.objects.create(origin_type=origin_type, origin_url=origin_url, status=save_request_status) if save_request_status == SAVE_REQUEST_REJECTED: raise ForbiddenExc('The origin url is blacklisted and will not be ' 'loaded into the archive.') return _save_request_dict(sor, task) def get_save_origin_requests_from_queryset(requests_queryset): """ Get all save requests from a SaveOriginRequest queryset. Args: requests_queryset (django.db.models.QuerySet): input SaveOriginRequest queryset Returns: list: A list of save origin requests dict as described in :func:`swh.web.common.origin_save.create_save_origin_request` """ task_ids = [] for sor in requests_queryset: task_ids.append(sor.loading_task_id) requests = [] if task_ids: tasks = scheduler.get_tasks(task_ids) tasks = {task['id']: task for task in tasks} for sor in requests_queryset: sr_dict = _save_request_dict(sor, tasks.get(sor.loading_task_id)) requests.append(sr_dict) return requests def get_save_origin_requests(origin_type, origin_url): """ Get all save requests for a given software origin. Args: origin_type (str): the type of the origin origin_url (str): the url of the origin Raises: BadInputExc: the origin type or url is invalid NotFoundExc: no save requests can be found for the given origin Returns: list: A list of save origin requests dict as described in :func:`swh.web.common.origin_save.create_save_origin_request` """ _check_origin_type_savable(origin_type) _check_origin_url_valid(origin_url) sors = SaveOriginRequest.objects.filter(origin_type=origin_type, origin_url=origin_url) if sors.count() == 0: raise NotFoundExc(('No save requests found for origin with type ' '%s and url %s.') % (origin_type, origin_url)) return get_save_origin_requests_from_queryset(sors) diff --git a/swh/web/common/origin_visits.py b/swh/web/common/origin_visits.py index 34e86230..590df3e2 100644 --- a/swh/web/common/origin_visits.py +++ b/swh/web/common/origin_visits.py @@ -1,188 +1,188 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import math from django.core.cache import cache from swh.web.common.exc import NotFoundExc from swh.web.common.utils import parse_timestamp def get_origin_visits(origin_info): """Function that returns the list of visits for a swh origin. That list is put in cache in order to speedup the navigation in the swh web browse ui. Args: origin_info (dict): dict describing the origin to fetch visits from Returns: list: A list of dict describing the origin visits with the following keys: * **date**: UTC visit date in ISO format, * **origin**: the origin id * **status**: the visit status, either **full**, **partial** or **ongoing** * **visit**: the visit id Raises: NotFoundExc: if the origin is not found """ from swh.web.common import service cache_entry_id = 'origin_%s_visits' % origin_info['id'] cache_entry = cache.get(cache_entry_id) if cache_entry: last_visit = cache_entry[-1]['visit'] new_visits = list(service.lookup_origin_visits(origin_info['id'], last_visit=last_visit)) if not new_visits: last_snp = service.lookup_latest_origin_snapshot(origin_info['id']) if not last_snp or last_snp['id'] == cache_entry[-1]['snapshot']: return cache_entry origin_visits = [] per_page = service.MAX_LIMIT last_visit = None while 1: visits = list(service.lookup_origin_visits(origin_info['id'], last_visit=last_visit, per_page=per_page)) origin_visits += visits if len(visits) < per_page: break else: if not last_visit: last_visit = per_page else: last_visit += per_page def _visit_sort_key(visit): ts = parse_timestamp(visit['date']).timestamp() return ts + (float(visit['visit']) / 10e3) for v in origin_visits: if 'metadata' in v: del v['metadata'] origin_visits = [dict(t) for t in set([tuple(d.items()) for d in origin_visits])] origin_visits = sorted(origin_visits, key=lambda v: _visit_sort_key(v)) cache.set(cache_entry_id, origin_visits) return origin_visits def get_origin_visit(origin_info, visit_ts=None, visit_id=None, snapshot_id=None): """Function that returns information about a visit for a given origin. The visit is retrieved from a provided timestamp. The closest visit from that timestamp is selected. Args: origin_info (dict): a dict filled with origin information (id, url, type) visit_ts (int or str): an ISO date string or Unix timestamp to parse Returns: A dict containing the visit info as described below:: {'origin': 2, 'date': '2017-10-08T11:54:25.582463+00:00', 'metadata': {}, 'visit': 25, 'status': 'full'} """ visits = get_origin_visits(origin_info) if not visits: if 'type' in origin_info and 'url' in origin_info: message = ('No visit associated to origin with' ' type %s and url %s!' % (origin_info['type'], origin_info['url'])) else: message = ('No visit associated to origin with' ' id %s!' % origin_info['id']) raise NotFoundExc(message) if snapshot_id: visit = [v for v in visits if v['snapshot'] == snapshot_id] if len(visit) == 0: if 'type' in origin_info and 'url' in origin_info: message = ('Visit for snapshot with id %s for origin with type' ' %s and url %s not found!' % (snapshot_id, origin_info['type'], origin_info['url'])) else: message = ('Visit for snapshot with id %s for origin with' ' id %s not found!' % (snapshot_id, origin_info['id'])) raise NotFoundExc(message) return visit[0] if visit_id: visit = [v for v in visits if v['visit'] == int(visit_id)] if len(visit) == 0: if 'type' in origin_info and 'url' in origin_info: message = ('Visit with id %s for origin with type %s' ' and url %s not found!' % (visit_id, origin_info['type'], origin_info['url'])) else: message = ('Visit with id %s for origin with id %s' ' not found!' % (visit_id, origin_info['id'])) raise NotFoundExc(message) return visit[0] if not visit_ts: # returns the latest full visit when no timestamp is provided for v in reversed(visits): if v['status'] == 'full': return v return visits[-1] parsed_visit_ts = math.floor(parse_timestamp(visit_ts).timestamp()) visit_idx = None for i, visit in enumerate(visits): ts = math.floor(parse_timestamp(visit['date']).timestamp()) if i == 0 and parsed_visit_ts <= ts: return visit elif i == len(visits) - 1: if parsed_visit_ts >= ts: return visit else: next_ts = math.floor( parse_timestamp(visits[i+1]['date']).timestamp()) if parsed_visit_ts >= ts and parsed_visit_ts < next_ts: if (parsed_visit_ts - ts) < (next_ts - parsed_visit_ts): visit_idx = i break else: visit_idx = i+1 break if visit_idx is not None: visit = visits[visit_idx] while visit_idx < len(visits) - 1 and \ visit['date'] == visits[visit_idx+1]['date']: visit_idx = visit_idx + 1 visit = visits[visit_idx] return visit else: if 'type' in origin_info and 'url' in origin_info: message = ('Visit with timestamp %s for origin with type %s ' 'and url %s not found!' % (visit_ts, origin_info['type'], origin_info['url'])) else: message = ('Visit with timestamp %s for origin with id %s ' 'not found!' % (visit_ts, origin_info['id'])) raise NotFoundExc(message) diff --git a/swh/web/common/urlsindex.py b/swh/web/common/urlsindex.py index 047314e4..0c9649b7 100644 --- a/swh/web/common/urlsindex.py +++ b/swh/web/common/urlsindex.py @@ -1,76 +1,76 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.conf.urls import url from django.shortcuts import redirect class UrlsIndex(object): """ Simple helper class for centralizing url patterns of a Django web application. Derived classes should override the 'scope' class attribute otherwise all declared patterns will be grouped under the default one. """ _urlpatterns = {} scope = 'default' @classmethod def add_url_pattern(cls, url_pattern, view, view_name=None): """ Class method that adds an url pattern to the current scope. Args: url_pattern: regex describing a Django url view: function implementing the Django view view_name: name of the view used to reverse the url """ if cls.scope not in cls._urlpatterns: cls._urlpatterns[cls.scope] = [] if view_name: cls._urlpatterns[cls.scope].append(url(url_pattern, view, name=view_name)) else: cls._urlpatterns[cls.scope].append(url(url_pattern, view)) @classmethod def add_redirect_for_checksum_args(cls, view_name, url_patterns, checksum_args): """ Class method that redirects to view with lowercase checksums when upper/mixed case checksums are passed as url arguments. Args: view_name (str): name of the view to redirect requests url_patterns (List[str]): regexps describing the view urls checksum_args (List[str]): url argument names corresponding to checksum values """ new_view_name = view_name+'-uppercase-checksum' for url_pattern in url_patterns: url_pattern_upper = url_pattern.replace('[0-9a-f]', '[0-9a-fA-F]') def view_redirect(request, *args, **kwargs): for checksum_arg in checksum_args: checksum_upper = kwargs[checksum_arg] kwargs[checksum_arg] = checksum_upper.lower() return redirect(view_name, *args, **kwargs) cls.add_url_pattern(url_pattern_upper, view_redirect, new_view_name) @classmethod def get_url_patterns(cls): """ Class method that returns the list of url pattern associated to the current scope. Returns: The list of url patterns associated to the current scope """ return cls._urlpatterns[cls.scope] diff --git a/swh/web/misc/coverage.py b/swh/web/misc/coverage.py index 1b78f33a..64220611 100644 --- a/swh/web/misc/coverage.py +++ b/swh/web/misc/coverage.py @@ -1,120 +1,120 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import render from django.views.decorators.clickjacking import xframe_options_exempt from swh.web.config import get_config # Current coverage list of the archive # TODO: Retrieve that list dynamically instead of hardcoding it code_providers = [ { 'provider_id': 'debian', 'provider_url': 'https://www.debian.org/', 'provider_logo': 'img/logos/debian.png', 'provider_info': 'source packages from the Debian distribution ' '(continuously archived)', 'origin_url_regexp': '^deb://', 'origin_types': 'packages', }, { 'provider_id': 'framagit', 'provider_url': 'https://framagit.org/', 'provider_logo': 'img/logos/framagit.png', 'provider_info': 'public repositories from Framagit ' '(continuously archived)', 'origin_url_regexp': '^https://framagit.org/', 'origin_types': 'repositories', }, { 'provider_id': 'github', 'provider_url': 'https://github.com', 'provider_logo': 'img/logos/github.png', 'provider_info': 'public repositories from GitHub ' '(continuously archived)', 'origin_url_regexp': '^https://github.com/', 'origin_types': 'repositories', }, { 'provider_id': 'gitlab', 'provider_url': 'https://gitlab.com', 'provider_logo': 'img/logos/gitlab.svg', 'provider_info': 'public repositories from GitLab ' '(continuously archived)', 'origin_url_regexp': '^https://gitlab.com/', 'origin_types': 'repositories', }, { 'provider_id': 'gitorious', 'provider_url': 'https://gitorious.org/', 'provider_logo': 'img/logos/gitorious.png', 'provider_info': 'public repositories from the former Gitorious code ' 'hosting service', 'origin_url_regexp': '^https://gitorious.org/', 'origin_types': 'repositories', }, { 'provider_id': 'googlecode', 'provider_url': 'https://code.google.com/archive/', 'provider_logo': 'img/logos/googlecode.png', 'provider_info': 'public repositories from the former Google Code ' 'project hosting service', 'origin_url_regexp': '^http.*.googlecode.com/', 'origin_types': 'repositories', }, { 'provider_id': 'gnu', 'provider_url': 'https://www.gnu.org', 'provider_logo': 'img/logos/gnu.png', 'provider_info': 'releases from the GNU project (as of August 2015)', 'origin_url_regexp': '^rsync://ftp.gnu.org/', 'origin_types': 'releases', }, { 'provider_id': 'hal', 'provider_url': 'https://hal.archives-ouvertes.fr/', 'provider_logo': 'img/logos/hal.png', 'provider_info': 'scientific software source code deposited in the ' 'open archive HAL', 'origin_url_regexp': '^https://hal.archives-ouvertes.fr/', 'origin_types': 'deposits', }, { 'provider_id': 'inria', 'provider_url': 'https://gitlab.inria.fr', 'provider_logo': 'img/logos/inria.jpg', 'provider_info': 'public repositories from Inria GitLab ' '(continuously archived)', 'origin_url_regexp': '^https://gitlab.inria.fr/', 'origin_types': 'repositories', }, { 'provider_id': 'npm', 'provider_url': 'https://www.npmjs.com/', 'provider_logo': 'img/logos/npm.png', 'provider_info': 'public packages from the package registry for ' 'javascript (continuously archived)', 'origin_url_regexp': '^https://www.npmjs.com/', 'origin_types': 'packages', }, { 'provider_id': 'pypi', 'provider_url': 'https://pypi.org', 'provider_logo': 'img/logos/pypi.svg', 'provider_info': 'source packages from the Python Packaging Index ' '(continuously archived)', 'origin_url_regexp': '^https://pypi.org/', 'origin_types': 'packages', }, ] @xframe_options_exempt def swh_coverage(request): count_origins = get_config()['coverage_count_origins'] return render(request, 'coverage.html', {'providers': code_providers, 'count_origins': count_origins}) diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py index 9b08b7b9..bcfdf186 100644 --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -1,247 +1,247 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django common settings for swh-web. """ import os from swh.web.config import get_config swh_web_config = get_config() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = swh_web_config['secret_key'] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = swh_web_config['debug'] DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config['debug'] ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] + swh_web_config['allowed_hosts'] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', 'swh.web.common', 'swh.web.api', 'swh.web.browse', 'webpack_loader', 'django_js_reverse' ] MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'swh.web.common.middlewares.ThrottlingHeadersMiddleware' ] # Compress all assets (static ones and dynamically generated html) # served by django in a local development environment context. # In a production environment, assets compression will be directly # handled by web servers like apache or nginx. if swh_web_config['serve_assets']: MIDDLEWARE.insert(0, 'django.middleware.gzip.GZipMiddleware') ROOT_URLCONF = 'swh.web.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [os.path.join(PROJECT_DIR, "../templates")], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', 'swh.web.common.utils.context_processor' ], 'libraries': { 'swh_templatetags': 'swh.web.common.swh_templatetags', }, }, }, ] WSGI_APPLICATION = 'swh.web.wsgi.application' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': swh_web_config['development_db'], } } # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa }, ] # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(PROJECT_DIR, "../static") ] INTERNAL_IPS = ['127.0.0.1'] throttle_rates = {} http_requests = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'] throttling = swh_web_config['throttling'] for limiter_scope, limiter_conf in throttling['scopes'].items(): if 'default' in limiter_conf['limiter_rate']: throttle_rates[limiter_scope] = limiter_conf['limiter_rate']['default'] # for backward compatibility else: throttle_rates[limiter_scope] = limiter_conf['limiter_rate'] # register sub scopes specific for HTTP request types for http_request in http_requests: if http_request in limiter_conf['limiter_rate']: throttle_rates[limiter_scope + '_' + http_request.lower()] = \ limiter_conf['limiter_rate'][http_request] REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', 'swh.web.api.renderers.YAMLRenderer', 'rest_framework.renderers.TemplateHTMLRenderer' ), 'DEFAULT_THROTTLE_CLASSES': ( 'swh.web.common.throttling.SwhWebRateThrottle', ), 'DEFAULT_THROTTLE_RATES': throttle_rates } LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse', }, 'require_debug_true': { '()': 'django.utils.log.RequireDebugTrue', }, }, 'formatters': { 'verbose': { 'format': '[%(asctime)s] [%(levelname)s] %(request)s %(status_code)s', # noqa 'datefmt': "%d/%b/%Y %H:%M:%S" }, }, 'handlers': { 'console': { 'level': 'DEBUG', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', }, 'file': { 'level': 'INFO', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), 'formatter': 'verbose' }, 'null': { 'class': 'logging.NullHandler', }, }, 'loggers': { 'django': { 'handlers': ['console', 'file'], 'level': 'DEBUG' if DEBUG else 'INFO', 'propagate': True, }, 'django.request': { 'handlers': ['file'], 'level': 'DEBUG' if DEBUG else 'INFO', 'propagate': False, }, 'django.db.backends': { 'handlers': ['null'], 'propagate': False } }, } WEBPACK_LOADER = { # noqa 'DEFAULT': { 'CACHE': False, 'BUNDLE_DIR_NAME': './', 'STATS_FILE': os.path.join(PROJECT_DIR, '../static/webpack-stats.json'), # noqa 'POLL_INTERVAL': 0.1, 'TIMEOUT': None, 'IGNORE': ['.+\.hot-update.js', '.+\.map'] } } LOGIN_URL = '/admin/login/' LOGIN_REDIRECT_URL = 'admin' SESSION_ENGINE = 'django.contrib.sessions.backends.cache' CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache' }, 'db_cache': { 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'swh_web_cache', } } JS_REVERSE_JS_MINIFY = False diff --git a/swh/web/settings/development.py b/swh/web/settings/development.py index 4f89248e..15d12efb 100644 --- a/swh/web/settings/development.py +++ b/swh/web/settings/development.py @@ -1,19 +1,19 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django development settings for swh-web. """ from django.core.cache import cache from .common import * # noqa from .common import MIDDLEWARE MIDDLEWARE += ['swh.web.common.middlewares.HtmlPrettifyMiddleware'] AUTH_PASSWORD_VALIDATORS = [] # disable any pwd validation mechanism cache.clear() diff --git a/swh/web/settings/production.py b/swh/web/settings/production.py index a98d2fd7..93b49e56 100644 --- a/swh/web/settings/production.py +++ b/swh/web/settings/production.py @@ -1,53 +1,53 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django production settings for swh-web. """ from .common import * # noqa from .common import ( MIDDLEWARE, CACHES, ALLOWED_HOSTS, WEBPACK_LOADER ) from .common import swh_web_config from .common import REST_FRAMEWORK # activate per-site caching if 'GZip' in MIDDLEWARE[0]: MIDDLEWARE.insert(1, 'django.middleware.cache.UpdateCacheMiddleware') else: MIDDLEWARE.insert(0, 'django.middleware.cache.UpdateCacheMiddleware') MIDDLEWARE += ['swh.web.common.middlewares.HtmlMinifyMiddleware', 'django.middleware.cache.FetchFromCacheMiddleware'] CACHES.update({ 'default': { 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': swh_web_config['throttling']['cache_uri'], } }) # Setup support for proxy headers USE_X_FORWARDED_HOST = True SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') # We're going through seven (or, in that case, 2) proxies thanks to Varnish REST_FRAMEWORK['NUM_PROXIES'] = 2 ALLOWED_HOSTS += [ 'archive.softwareheritage.org', 'base.softwareheritage.org', 'archive.internal.softwareheritage.org', ] DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': swh_web_config['production_db'], } } WEBPACK_LOADER['DEFAULT']['CACHE'] = True diff --git a/swh/web/templates/admin/origin-save.html b/swh/web/templates/admin/origin-save.html index fdf71543..e25d13aa 100644 --- a/swh/web/templates/admin/origin-save.html +++ b/swh/web/templates/admin/origin-save.html @@ -1,179 +1,179 @@ {% extends "layout.html" %} {% comment %} -Copyright (C) 2018 The Software Heritage developers +Copyright (C) 2018-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% load render_bundle from webpack_loader %} {% block header %} {{ block.super }} {% render_bundle 'admin' %} {% endblock %} {% block title %} Save origin administration {% endblock %} {% block navbar-content %}

Save origin administration

{% endblock %} {% block content %}
Date Type Url
Date Type Url Status
Date Type Url
Url
Url
{% endblock %} diff --git a/swh/web/templates/api/api.html b/swh/web/templates/api/api.html index 3ceed7ce..e9a4a3fc 100644 --- a/swh/web/templates/api/api.html +++ b/swh/web/templates/api/api.html @@ -1,23 +1,23 @@ {% extends "layout.html" %} {% comment %} -Copyright (C) 2015-2018 The Software Heritage developers +Copyright (C) 2015-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% block title %} Overview – Software Heritage API {% endblock %} {% block navbar-content %}

Web API

{% endblock %} {% block content %}
{% include 'includes/apidoc-header.html' %}
{% endblock %} diff --git a/swh/web/templates/api/apidoc.html b/swh/web/templates/api/apidoc.html index 8dc1cdf2..903d8f5f 100644 --- a/swh/web/templates/api/apidoc.html +++ b/swh/web/templates/api/apidoc.html @@ -1,183 +1,183 @@ {% extends "layout.html" %} {% comment %} -Copyright (C) 2015-2018 The Software Heritage developers +Copyright (C) 2015-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% block title %}{{ heading }} – Software Heritage API {% endblock %} {% block navbar-content %} {% endblock %} {% block content %}
{% if description %}

Description

{{ description | safe_docstring_display | safe }}
{% endif %} {% if response_data %}

Request

{{ request.method }} {{ request.path }}

Response

{% if status_code != 200 %}
Status Code
{{ status_code }}
{% endif %} {% if headers_data %}
Headers
{% for header_name, header_value in headers_data.items %} 
{{ header_name }} {{ header_value | urlize_header_links | safe }}
{% endfor %} {% endif %}
Body
{{ response_data | urlize_links_and_mails | safe }}
{% endif %}
{% if urls and urls|length > 0 %}
{% for url in urls %} {% endfor %}
URL Allowed Methods
{{ url.rule | safe_docstring_display | safe }} {{ url.methods | dictsort:0 | join:', ' }}
{% endif %} {% if args and args|length > 0 %}

Arguments

{% for arg in args %}
{{ arg.name }} ({{ arg.type }})
{{ arg.doc | safe_docstring_display | safe }}
{% endfor %}
{% endif %} {% if params and params|length > 0 %}

Query parameters

{% for param in params %}
{{ param.name }} ({{ param.type }})
{{ param.doc | safe_docstring_display | safe }}
{% endfor %}
{% endif %} {% if reqheaders and reqheaders|length > 0 %}

Request headers

{% for header in reqheaders %}
{{ header.name }}
{{ header.doc | safe_docstring_display | safe }}
{% endfor %}
{% endif %} {% if resheaders and resheaders|length > 0 %}

Response headers

{% for header in resheaders %}
{{ header.name }}
{{ header.doc | safe_docstring_display | safe }}
{% endfor %}
{% endif %} {% if return_type %}

Returns

{{ return_type }}

{% if return_type == 'array' %} an array of objects containing the following keys: {% elif return_type == 'octet stream' %} the raw data as an octet stream {% else %} an object containing the following keys: {% endif %} {{ returns_list | safe_docstring_display | safe }}

{% endif %} {% if status_codes and status_codes|length > 0 %}

HTTP status codes

{% for status in status_codes %}
{{ status.code }}
{{ status.doc | safe_docstring_display | safe }}
{% endfor %}
{% endif %} {% if examples and examples|length > 0 %}

Examples

{% for example in examples %}
{{ example }}
{% endfor %}
{% endif %}
{% endblock %} diff --git a/swh/web/templates/api/endpoints.html b/swh/web/templates/api/endpoints.html index 0938a222..1e4a49f2 100644 --- a/swh/web/templates/api/endpoints.html +++ b/swh/web/templates/api/endpoints.html @@ -1,82 +1,82 @@ {% extends "layout.html" %} {% comment %} -Copyright (C) 2015-2018 The Software Heritage developers +Copyright (C) 2015-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% block title %} Endpoints – Software Heritage API {% endblock %} {% block navbar-content %}
{% endblock %} {% block content %}

Below you can find a list of the available endpoints for version 1 of the Software Heritage API. For a more general introduction please refer to the API overview.

Endpoints marked "available" are considered stable for the current version of the API; endpoints marked "upcoming" are work in progress that will be stabilized in the near future.

{% for route, doc in doc_routes %} {% if doc.tags|length > 0 %} {% else %} {% endif %} {% endfor %}
Endpoint Description
{% url doc.route_view_name %} {% url doc.route_view_name %} {{ doc.doc_intro | safe_docstring_display | safe }}
{% endblock %} diff --git a/swh/web/templates/browse/branches.html b/swh/web/templates/browse/branches.html index aa5901f7..2fc0939c 100644 --- a/swh/web/templates/browse/branches.html +++ b/swh/web/templates/browse/branches.html @@ -1,71 +1,71 @@ {% extends "./browse.html" %} {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% block swh-browse-content %} {% if displayed_branches|length > 0 %}
{% for branch in displayed_branches %} {% endfor %}
Name Revision Message Date
{{ branch.name }} {{ branch.revision|slice:":7" }} {{ branch.message }} {{ branch.date }}
{% else %} The list of branches is empty ! {% endif %} {% endblock %} {% block swh-browse-after-content %} {% if prev_branches_url or next_branches_url %}
    {% if prev_branches_url %}
  • Previous
    • {% else %}
  • Previous
    • {% endif %} {% if next_branches_url %}
  • Next
    • {% else %}
  • Next
    • {% endif %}
{% endif %} {% endblock %} diff --git a/swh/web/templates/browse/layout.html b/swh/web/templates/browse/layout.html index d922de2d..688ed544 100644 --- a/swh/web/templates/browse/layout.html +++ b/swh/web/templates/browse/layout.html @@ -1,23 +1,23 @@ {% extends "layout.html" %} {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% load render_bundle from webpack_loader %} {% block title %}{{ heading }} – Software Heritage archive {% endblock %} {% block header %} {% render_bundle 'browse' %} {% render_bundle 'vault' %} {% endblock %} {% block content %}
Beta version
{% block browse-content %}{% endblock %} {% endblock %} diff --git a/swh/web/templates/browse/release.html b/swh/web/templates/browse/release.html index d1839bbf..54edf7e5 100644 --- a/swh/web/templates/browse/release.html +++ b/swh/web/templates/browse/release.html @@ -1,25 +1,25 @@ {% extends "./browse.html" %} {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% block swh-browse-content %} {% include "includes/top-navigation.html" %}
Release {{ swh_object_metadata.name }} created by {{ swh_object_metadata.author }} on {{ swh_object_metadata.date }} 
 
{{ release.note_header }}
{{ release.note_body }}
Target: {{ release.target_link}}
{% endblock %} diff --git a/swh/web/templates/browse/releases.html b/swh/web/templates/browse/releases.html index 5ee0ffd5..7a49318c 100644 --- a/swh/web/templates/browse/releases.html +++ b/swh/web/templates/browse/releases.html @@ -1,69 +1,69 @@ {% extends "./browse.html" %} {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% block swh-browse-content %} {% if displayed_releases|length > 0 %}
{% for release in displayed_releases %} {% endfor %}
Name Target Message Date
{{ release.name }} {{ release.target|slice:":7" }} {{ release.message }} {{ release.date }}
{% else %} The list of releases is empty ! {% endif %} {% endblock %} {% block swh-browse-after-content %} {% if prev_releases_url or next_releases_url %}
    {% if prev_releases_url %}
  • Previous
    • {% else %}
  • Previous
    • {% endif %} {% if next_releases_url %}
  • Next
    • {% else %}
  • Next
    • {% endif %}
{% endif %} {% endblock %} diff --git a/swh/web/templates/browse/revision-log.html b/swh/web/templates/browse/revision-log.html index ba74625c..66f35c92 100644 --- a/swh/web/templates/browse/revision-log.html +++ b/swh/web/templates/browse/revision-log.html @@ -1,119 +1,119 @@ {% extends "./browse.html" %} {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load render_bundle from webpack_loader %} {% load swh_templatetags %} {% block header %} {{ block.super }} {% render_bundle 'revision' %} {% endblock %} {% block swh-browse-content %} {% if snapshot_context %} {% include "includes/top-navigation.html" %} {% endif %} {% if snapshot_context and snapshot_context.is_empty %} {% include "includes/empty-snapshot.html" %} {% else %}
sort by:
{% for rev in revision_log %} {% endfor %}
Revision Author Date Message Commit Date
{{ rev.id }} {{ rev.author }} {{ rev.date }} {{ rev.message }} {{ rev.commit_date }}
{% endif %} {% endblock %} {% block swh-browse-after-content %} {% if not snapshot_context or not snapshot_context.is_empty %} {% endif %} {% endblock %} diff --git a/swh/web/templates/browse/revision.html b/swh/web/templates/browse/revision.html index 82bf731b..b1971841 100644 --- a/swh/web/templates/browse/revision.html +++ b/swh/web/templates/browse/revision.html @@ -1,111 +1,111 @@ {% extends "./browse.html" %} {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load static %} {% load swh_templatetags %} {% load render_bundle from webpack_loader %} {% block header %} {{ block.super }} {% render_bundle 'revision' %} {% endblock %} {% block swh-browse-content %}
Revision {{ swh_object_metadata.revision }} authored by {{ swh_object_metadata.author }} on {{ swh_object_metadata.date }}, committed by {{ swh_object_metadata.committer }} on {{ swh_object_metadata|key_value:'committer date' }} 
{{ message_header }}
{% if message_body %} 
{{ message_body }}
{% endif %}
{{ parents|length }} parent{% if parents|length > 1 %}s{% endif %} {% for parent in parents %} {{ parent.id|slice:":7" }} {% if not forloop.last %} + {% endif %} {% endfor %}
{% include "includes/top-navigation.html" %} {% if content_size %} {% include "includes/content-display.html" %} {% else %} {% include "includes/directory-display.html" %} {% endif %}
{% endblock %} {% block swh-browse-after-content %} {% include "includes/readme-display.html" %} {% endblock %} diff --git a/swh/web/templates/browse/search.html b/swh/web/templates/browse/search.html index 44efe2cb..f71616a8 100644 --- a/swh/web/templates/browse/search.html +++ b/swh/web/templates/browse/search.html @@ -1,75 +1,75 @@ {% extends "./layout.html" %} {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load static %} {% block navbar-content %}

Search archived software

{% endblock %} {% block browse-content %}

Searching origins ...

{% endblock %} diff --git a/swh/web/templates/homepage.html b/swh/web/templates/homepage.html index 40112cf1..4bd35348 100644 --- a/swh/web/templates/homepage.html +++ b/swh/web/templates/homepage.html @@ -1,121 +1,121 @@ {% extends "layout.html" %} {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load static %} {% block title %}Welcome to the Software Heritage archive{% endblock %} {% block navbar-content %}

Welcome to the Software Heritage archive

{% endblock %} {% block content %}

Overview

The long term goal of the Software Heritage initiative is to collect all publicly available software in source code form together with its development history, replicate it massively to ensure its preservation, and share it with everyone who needs it. The Software Heritage archive is growing over time as we crawl new source code from software projects and development forges. We will incrementally release archive search and browse functionalities — as of now you can check whether source code you care about is already present in the archive or not.

Content

A significant amount of source code has already been ingested in the Software Heritage archive. It currently includes:

Size

As of today the archive already contains and keeps safe for you the following amount of objects:

Source files
Directories
Commits
Authors
Projects
Releases

Access

Browse the Software Heritage archive
Browse

Explore the archive using a dedicated web interface.
Software Heritage Web API
Web API

Query the archive programmatically using a RESTful API.

{% endblock %} diff --git a/swh/web/templates/includes/content-display.html b/swh/web/templates/includes/content-display.html index 5c57711a..bf5ab2d5 100644 --- a/swh/web/templates/includes/content-display.html +++ b/swh/web/templates/includes/content-display.html @@ -1,59 +1,59 @@ {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% if snapshot_context and snapshot_context.is_empty %} {% include "includes/empty-snapshot.html" %} {% else %}
{% if swh_object_metadata.filename %}
{{ swh_object_metadata.filename }}
{% endif %}
{% if content_size > max_content_size %} Content is too large to be displayed (size is greater than {{ max_content_size|filesizeformat }}). {% elif "inode/x-empty" == mimetype %} File is empty {% elif swh_object_metadata.filename and swh_object_metadata.filename|default:""|slice:"-5:" == "ipynb" %}
{% elif "text/" in mimetype %} 
{{ content }}
{% elif "image/" in mimetype and content %} {% elif "application/pdf" == mimetype %}
Page: /
{% elif content %} Content with mime type {{ mimetype }} can not be displayed. {% else %} {% include "includes/http-error.html" %} {% endif %}
{% endif %} diff --git a/swh/web/templates/includes/directory-display.html b/swh/web/templates/includes/directory-display.html index 9d8ac58d..e0d2c9e7 100644 --- a/swh/web/templates/includes/directory-display.html +++ b/swh/web/templates/includes/directory-display.html @@ -1,58 +1,58 @@ {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% if snapshot_context and snapshot_context.is_empty %} {% include "includes/empty-snapshot.html" %} {% elif dirs|length > 0 or files|length > 0 %}
{% for d in dirs %} {% endfor %} {% for f in files %} {% endfor %}
File Mode Size
{{ d.name }} {{ d.perms }}
{{ f.name }} {{ f.perms }} {{ f.length }}
{% elif dirs|length == 0 and files|length == 0 %} Directory is empty -{% endif %} \ No newline at end of file +{% endif %} diff --git a/swh/web/templates/includes/readme-display.html b/swh/web/templates/includes/readme-display.html index b2dce346..4fe8e2dd 100644 --- a/swh/web/templates/includes/readme-display.html +++ b/swh/web/templates/includes/readme-display.html @@ -1,38 +1,38 @@ {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% if readme_name %}

{{ readme_name }}

{% if readme_html %} {% elif readme_name.lower == 'readme' or readme_name.lower == 'readme.txt' %} {% elif readme_name.lower == 'readme.org' %} {% else %} {% endif %} {% endif %} diff --git a/swh/web/templates/includes/show-metadata.html b/swh/web/templates/includes/show-metadata.html index 900b36a5..3bc96b61 100644 --- a/swh/web/templates/includes/show-metadata.html +++ b/swh/web/templates/includes/show-metadata.html @@ -1,36 +1,36 @@ {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %}
Show metadata
diff --git a/swh/web/templates/includes/show-swh-ids.html b/swh/web/templates/includes/show-swh-ids.html index 9cdab5fe..de355574 100644 --- a/swh/web/templates/includes/show-swh-ids.html +++ b/swh/web/templates/includes/show-swh-ids.html @@ -1,96 +1,96 @@ {% comment %} -Copyright (C) 2017-2018 The Software Heritage developers +Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% if swh_ids %}