diff --git a/requirements-swh.txt b/requirements-swh.txt index a2f7aa45..2fefb9d7 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,6 @@ swh.model >= 0.0.32 swh.storage >= 0.0.145 swh.vault >= 0.0.23 swh.indexer >= 0.0.120 swh.scheduler >= 0.0.31 +swh.core >= 0.0.81 diff --git a/requirements-test.txt b/requirements-test.txt index 70056983..4910e54e 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,8 +1,9 @@ hypothesis pytest pytest-django pytest-mock django-stubs < 1.3.0 requests-mock -swh.core[http] >= 0.0.61 +swh.core[http] >= 0.0.81 swh.loader.git >= 0.0.47 +decorator # dependency of swh.core[http] diff --git a/requirements.txt b/requirements.txt index e6408273..364e6201 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,28 +1,29 @@ # Add here external Python modules dependencies, one per line. Module names # should match https://pypi.python.org/pypi names. For the full spec or # dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html # Runtime dependencies beautifulsoup4 Django >= 1.11.0, < 2.0 django-cors-headers djangorestframework >= 3.4.0 django_webpack_loader django_js_reverse docutils python-magic >= 0.4.0 htmlmin lxml pygments pypandoc python-dateutil pyyaml requests python-memcached pybadges +sentry-sdk # Doc dependencies sphinx sphinxcontrib-httpdomain diff --git a/swh/web/admin/deposit.py b/swh/web/admin/deposit.py index 7cb84f98..d7da7bb5 100644 --- a/swh/web/admin/deposit.py +++ b/swh/web/admin/deposit.py @@ -1,93 +1,95 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import requests from django.core.cache import cache from django.conf import settings from django.contrib.admin.views.decorators import staff_member_required from django.core.paginator import Paginator from django.http import HttpResponse from django.shortcuts import render from requests.auth import HTTPBasicAuth +import sentry_sdk from swh.web.admin.adminurls import admin_route from swh.web.config import get_config config = get_config()['deposit'] @admin_route(r'deposit/', view_name='admin-deposit') @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save(request): return render(request, 'admin/deposit.html') @admin_route(r'deposit/list/', view_name='admin-deposit-list') @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_deposit_list(request): table_data = {} table_data['draw'] = int(request.GET['draw']) deposits_list_url = config['private_api_url'] + 'deposits' deposits_list_auth = HTTPBasicAuth(config['private_api_user'], config['private_api_password']) try: nb_deposits = requests.get('%s?page_size=1' % deposits_list_url, auth=deposits_list_auth, timeout=30).json()['count'] deposits_data = cache.get('swh-deposit-list') if not deposits_data or deposits_data['count'] != nb_deposits: deposits_data = requests.get('%s?page_size=%s' % (deposits_list_url, nb_deposits), auth=deposits_list_auth, timeout=30).json() cache.set('swh-deposit-list', deposits_data) deposits = deposits_data['results'] search_value = request.GET['search[value]'] if search_value: deposits = \ [d for d in deposits if any(search_value.lower() in val for val in [str(v).lower() for v in d.values()])] column_order = request.GET['order[0][column]'] field_order = request.GET['columns[%s][name]' % column_order] order_dir = request.GET['order[0][dir]'] deposits = sorted(deposits, key=lambda d: d[field_order] or '') if order_dir == 'desc': deposits = list(reversed(deposits)) length = int(request.GET['length']) page = int(request.GET['start']) / length + 1 paginator = Paginator(deposits, length) data = paginator.page(page).object_list table_data['recordsTotal'] = deposits_data['count'] table_data['recordsFiltered'] = len(deposits) table_data['data'] = [{ 'id': d['id'], 'external_id': d['external_id'], 'reception_date': d['reception_date'], 'status': d['status'], 'status_detail': d['status_detail'], 'swh_anchor_id': d['swh_anchor_id'], 'swh_anchor_id_context': d['swh_anchor_id_context'], 'swh_id': d['swh_id'], 'swh_id_context': d['swh_id_context'] } for d in data] - except Exception: + except Exception as exc: + sentry_sdk.capture_exception(exc) table_data['error'] = ('An error occurred while retrieving ' 'the list of deposits !') return HttpResponse(json.dumps(table_data), content_type='application/json') diff --git a/swh/web/api/apidoc.py b/swh/web/api/apidoc.py index 82fd47ce..970a843a 100644 --- a/swh/web/api/apidoc.py +++ b/swh/web/api/apidoc.py @@ -1,398 +1,400 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import docutils.nodes import docutils.parsers.rst import docutils.utils import functools +from functools import wraps import os import re import textwrap -from functools import wraps from rest_framework.decorators import api_view +import sentry_sdk from swh.web.common.utils import parse_rst from swh.web.api.apiurls import APIUrls from swh.web.api.apiresponse import make_api_response, error_response class _HTTPDomainDocVisitor(docutils.nodes.NodeVisitor): """ docutils visitor for walking on a parsed rst document containing sphinx httpdomain roles. Its purpose is to extract relevant info regarding swh api endpoints (for instance url arguments) from their docstring written using sphinx httpdomain. """ # httpdomain roles we want to parse (based on sphinxcontrib.httpdomain 1.6) parameter_roles = ('param', 'parameter', 'arg', 'argument') response_json_object_roles = ('resjsonobj', 'resjson', '>jsonobj', '>json') response_json_array_roles = ('resjsonarr', '>jsonarr') query_parameter_roles = ('queryparameter', 'queryparam', 'qparam', 'query') request_header_roles = ('header', 'resheader', 'responseheader') status_code_roles = ('statuscode', 'status', 'code') def __init__(self, document, urls, data): super().__init__(document) self.urls = urls self.url_idx = 0 self.data = data self.args_set = set() self.params_set = set() self.returns_set = set() self.status_codes_set = set() self.reqheaders_set = set() self.resheaders_set = set() self.field_list_visited = False def process_paragraph(self, par): """ Process extracted paragraph text before display. Cleanup document model markups and transform the paragraph into a valid raw rst string (as the apidoc documentation transform rst to html when rendering). """ par = par.replace('\n', ' ') # keep emphasized, strong and literal text par = par.replace('', '*') par = par.replace('', '*') par = par.replace('', '**') par = par.replace('', '**') par = par.replace('', '``') par = par.replace('', '``') # remove parsed document markups par = re.sub('<[^<]+?>', '', par) # api urls cleanup to generate valid links afterwards par = re.sub(r'\(\w+\)', '', par) par = re.sub(r'\[.*\]', '', par) par = par.replace('//', '/') # transform references to api endpoints into valid rst links par = re.sub(':http:get:`([^,]*)`', r'`<\1>`_', par) # transform references to some elements into bold text par = re.sub(':http:header:`(.*)`', r'**\1**', par) par = re.sub(':func:`(.*)`', r'**\1**', par) return par def visit_field_list(self, node): """ Visit parsed rst field lists to extract relevant info regarding api endpoint. """ self.field_list_visited = True for child in node.traverse(): # get the parsed field name if isinstance(child, docutils.nodes.field_name): field_name = child.astext() # parse field text elif isinstance(child, docutils.nodes.paragraph): text = self.process_paragraph(str(child)) field_data = field_name.split(' ') # Parameters if field_data[0] in self.parameter_roles: if field_data[2] not in self.args_set: self.data['args'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.args_set.add(field_data[2]) # Query Parameters if field_data[0] in self.query_parameter_roles: if field_data[2] not in self.params_set: self.data['params'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.params_set.add(field_data[2]) # Response type if field_data[0] in self.response_json_array_roles or \ field_data[0] in self.response_json_object_roles: # array if field_data[0] in self.response_json_array_roles: self.data['return_type'] = 'array' # object else: self.data['return_type'] = 'object' # returned object field if field_data[2] not in self.returns_set: self.data['returns'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.returns_set.add(field_data[2]) # Status Codes if field_data[0] in self.status_code_roles: if field_data[1] not in self.status_codes_set: self.data['status_codes'].append({'code': field_data[1], # noqa 'doc': text}) self.status_codes_set.add(field_data[1]) # Request Headers if field_data[0] in self.request_header_roles: if field_data[1] not in self.reqheaders_set: self.data['reqheaders'].append({'name': field_data[1], 'doc': text}) self.reqheaders_set.add(field_data[1]) # Response Headers if field_data[0] in self.response_header_roles: if field_data[1] not in self.resheaders_set: resheader = {'name': field_data[1], 'doc': text} self.data['resheaders'].append(resheader) self.resheaders_set.add(field_data[1]) if resheader['name'] == 'Content-Type' and \ resheader['doc'] == 'application/octet-stream': self.data['return_type'] = 'octet stream' def visit_paragraph(self, node): """ Visit relevant paragraphs to parse """ # only parsed top level paragraphs if isinstance(node.parent, docutils.nodes.block_quote): text = self.process_paragraph(str(node)) # endpoint description if (not text.startswith('**') and text not in self.data['description']): self.data['description'] += '\n\n' if self.data['description'] else '' # noqa self.data['description'] += text # http methods elif text.startswith('**Allowed HTTP Methods:**'): text = text.replace('**Allowed HTTP Methods:**', '') http_methods = text.strip().split(',') http_methods = [m[m.find('`')+1:-1].upper() for m in http_methods] self.data['urls'].append({'rule': self.urls[self.url_idx], 'methods': http_methods}) self.url_idx += 1 def visit_literal_block(self, node): """ Visit literal blocks """ text = node.astext() # literal block in endpoint description if not self.field_list_visited: self.data['description'] += \ ':\n\n%s\n' % textwrap.indent(text, '\t') # extract example url if ':swh_web_api:' in text: self.data['examples'].append( '/api/1/' + re.sub('.*`(.*)`.*', r'\1', text)) def visit_bullet_list(self, node): # bullet list in endpoint description if not self.field_list_visited: self.data['description'] += '\n\n' for child in node.traverse(): # process list item if isinstance(child, docutils.nodes.paragraph): line_text = self.process_paragraph(str(child)) self.data['description'] += '\t* %s\n' % line_text def visit_warning(self, node): text = self.process_paragraph(str(node)) rst_warning = '\n\n.. warning::\n%s\n' % textwrap.indent(text, '\t') if rst_warning not in self.data['description']: self.data['description'] += rst_warning def unknown_visit(self, node): pass def depart_document(self, node): """ End of parsing extra processing """ default_methods = ['GET', 'HEAD', 'OPTIONS'] # ensure urls info is present and set default http methods if not self.data['urls']: for url in self.urls: self.data['urls'].append({'rule': url, 'methods': default_methods}) def unknown_departure(self, node): pass def _parse_httpdomain_doc(doc, data): doc_lines = doc.split('\n') doc_lines_filtered = [] urls = [] # httpdomain is a sphinx extension that is unknown to docutils but # fortunately we can still parse its directives' content, # so remove lines with httpdomain directives before executing the # rst parser from docutils for doc_line in doc_lines: if '.. http' not in doc_line: doc_lines_filtered.append(doc_line) else: url = doc_line[doc_line.find('/'):] # emphasize url arguments for html rendering url = re.sub(r'\((\w+)\)', r' **\(\1\)** ', url) urls.append(url) # parse the rst docstring and do not print system messages about # unknown httpdomain roles document = parse_rst('\n'.join(doc_lines_filtered), report_level=5) # remove the system_message nodes from the parsed document for node in document.traverse(docutils.nodes.system_message): node.parent.remove(node) # visit the document nodes to extract relevant endpoint info visitor = _HTTPDomainDocVisitor(document, urls, data) document.walkabout(visitor) class APIDocException(Exception): """ Custom exception to signal errors in the use of the APIDoc decorators """ def api_doc(route, noargs=False, need_params=False, tags=[], handle_response=False, api_version='1'): """ Decorate an API function to register it in the API doc route index and create the corresponding DRF route. Args: route (str): documentation page's route noargs (boolean): set to True if the route has no arguments, and its result should be displayed anytime its documentation is requested. Default to False need_params (boolean): specify the route requires query parameters otherwise errors will occur. It enables to avoid displaying the invalid response in its HTML documentation. Default to False. tags (list): Further information on api endpoints. Two values are possibly expected: * hidden: remove the entry points from the listing * upcoming: display the entry point but it is not followable handle_response (boolean): indicate if the decorated function takes care of creating the HTTP response or delegates that task to the apiresponse module api_version (str): api version string """ urlpattern = '^' + api_version + route + '$' tags = set(tags) # @api_doc() Decorator call def decorator(f): # If the route is not hidden, add it to the index if 'hidden' not in tags: doc_data = get_doc_data(f, route, noargs) doc_desc = doc_data['description'] first_dot_pos = doc_desc.find('.') APIUrls.add_route(route, doc_desc[:first_dot_pos+1], tags=tags) # If the decorated route has arguments, we create a specific # documentation view if not noargs: @api_view(['GET', 'HEAD']) @wraps(f) def doc_view(request): doc_data = get_doc_data(f, route, noargs) return make_api_response(request, None, doc_data) view_name = 'api-%s-%s' % \ (api_version, route[1:-1].replace('/', '-')) APIUrls.add_url_pattern(urlpattern, doc_view, view_name) @wraps(f) def documented_view(request, **kwargs): doc_data = get_doc_data(f, route, noargs) try: response = f(request, **kwargs) except Exception as exc: + sentry_sdk.capture_exception(exc) if request.accepted_media_type == 'text/html' and \ need_params and not request.query_params: response = None else: return error_response(request, exc, doc_data) if handle_response: return response else: return make_api_response(request, response, doc_data) return documented_view return decorator @functools.lru_cache(maxsize=32) def get_doc_data(f, route, noargs): """ Build documentation data for the decorated api endpoint function """ data = { 'description': '', 'response_data': None, 'urls': [], 'args': [], 'params': [], 'resheaders': [], 'reqheaders': [], 'return_type': '', 'returns': [], 'status_codes': [], 'examples': [], 'route': route, 'noargs': noargs } if not f.__doc__: raise APIDocException('apidoc: expected a docstring' ' for function %s' % (f.__name__,)) # use raw docstring as endpoint documentation if sphinx # httpdomain is not used if '.. http' not in f.__doc__: data['description'] = f.__doc__ # else parse the sphinx httpdomain docstring with docutils # (except when building the swh-web documentation through autodoc # sphinx extension, not needed and raise errors with sphinx >= 1.7) elif 'SWH_WEB_DOC_BUILD' not in os.environ: _parse_httpdomain_doc(f.__doc__, data) # process returned object info for nicer html display returns_list = '' for ret in data['returns']: returns_list += '\t* **%s (%s)**: %s\n' %\ (ret['name'], ret['type'], ret['doc']) data['returns_list'] = returns_list return data DOC_COMMON_HEADERS = ''' :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request''' DOC_RESHEADER_LINK = ''' :resheader Link: indicates that a subsequent result page is available and contains the url pointing to it ''' DEFAULT_SUBSTITUTIONS = { 'common_headers': DOC_COMMON_HEADERS, 'resheader_link': DOC_RESHEADER_LINK, } def format_docstring(**substitutions): def decorator(f): f.__doc__ = f.__doc__.format(**{ **DEFAULT_SUBSTITUTIONS, **substitutions}) return f return decorator diff --git a/swh/web/browse/utils.py b/swh/web/browse/utils.py index d68aa953..5d72f120 100644 --- a/swh/web/browse/utils.py +++ b/swh/web/browse/utils.py @@ -1,1101 +1,1104 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import base64 import magic import pypandoc import stat import textwrap from collections import defaultdict from threading import Lock from django.core.cache import cache from django.utils.safestring import mark_safe from django.utils.html import escape +import sentry_sdk from swh.model.identifiers import persistent_identifier from swh.web.common import highlightjs, service from swh.web.common.exc import NotFoundExc, http_status_code_message from swh.web.common.origin_visits import get_origin_visit from swh.web.common.utils import ( reverse, format_utc_iso_date, get_swh_persistent_id, swh_object_icons ) from swh.web.config import get_config def get_directory_entries(sha1_git): """Function that retrieves the content of a directory from the archive. The directories entries are first sorted in lexicographical order. Sub-directories and regular files are then extracted. Args: sha1_git: sha1_git identifier of the directory Returns: A tuple whose first member corresponds to the sub-directories list and second member the regular files list Raises: NotFoundExc if the directory is not found """ cache_entry_id = 'directory_entries_%s' % sha1_git cache_entry = cache.get(cache_entry_id) if cache_entry: return cache_entry entries = list(service.lookup_directory(sha1_git)) for e in entries: e['perms'] = stat.filemode(e['perms']) if e['type'] == 'rev': # modify dir entry name to explicitly show it points # to a revision e['name'] = '%s @ %s' % (e['name'], e['target'][:7]) dirs = [e for e in entries if e['type'] in ('dir', 'rev')] files = [e for e in entries if e['type'] == 'file'] dirs = sorted(dirs, key=lambda d: d['name']) files = sorted(files, key=lambda f: f['name']) cache.set(cache_entry_id, (dirs, files)) return dirs, files _lock = Lock() def get_mimetype_and_encoding_for_content(content): """Function that returns the mime type and the encoding associated to a content buffer using the magic module under the hood. Args: content (bytes): a content buffer Returns: A tuple (mimetype, encoding), for instance ('text/plain', 'us-ascii'), associated to the provided content. """ # https://pypi.org/project/python-magic/ # packaged as python3-magic in debian buster if hasattr(magic, 'from_buffer'): m = magic.Magic(mime=True, mime_encoding=True) mime_encoding = m.from_buffer(content) mime_type, encoding = mime_encoding.split(';') encoding = encoding.replace(' charset=', '') # https://pypi.org/project/file-magic/ # packaged as python3-magic in debian stretch else: # TODO: Remove that code when production environment is upgraded # to debian buster # calls to the file-magic API are not thread-safe so they must # be protected with a Lock to guarantee they will succeed _lock.acquire() magic_result = magic.detect_from_content(content) _lock.release() mime_type = magic_result.mime_type encoding = magic_result.encoding return mime_type, encoding # maximum authorized content size in bytes for HTML display # with code highlighting content_display_max_size = get_config()['content_display_max_size'] snapshot_content_max_size = get_config()['snapshot_content_max_size'] def _re_encode_content(mimetype, encoding, content_data): # encode textual content to utf-8 if needed if mimetype.startswith('text/'): # probably a malformed UTF-8 content, re-encode it # by replacing invalid chars with a substitution one if encoding == 'unknown-8bit': content_data = content_data.decode('utf-8', 'replace')\ .encode('utf-8') elif encoding not in ['utf-8', 'binary']: content_data = content_data.decode(encoding, 'replace')\ .encode('utf-8') elif mimetype.startswith('application/octet-stream'): # file may detect a text content as binary # so try to decode it for display encodings = ['us-ascii', 'utf-8'] encodings += ['iso-8859-%s' % i for i in range(1, 17)] for enc in encodings: try: content_data = content_data.decode(enc).encode('utf-8') - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) else: # ensure display in content view encoding = enc mimetype = 'text/plain' break return mimetype, encoding, content_data def request_content(query_string, max_size=content_display_max_size, raise_if_unavailable=True, re_encode=True): """Function that retrieves a content from the archive. Raw bytes content is first retrieved, then the content mime type. If the mime type is not stored in the archive, it will be computed using Python magic module. Args: query_string: a string of the form "[ALGO_HASH:]HASH" where optional ALGO_HASH can be either ``sha1``, ``sha1_git``, ``sha256``, or ``blake2s256`` (default to ``sha1``) and HASH the hexadecimal representation of the hash value max_size: the maximum size for a content to retrieve (default to 1MB, no size limit if None) Returns: A tuple whose first member corresponds to the content raw bytes and second member the content mime type Raises: NotFoundExc if the content is not found """ content_data = service.lookup_content(query_string) filetype = None language = None license = None # requests to the indexer db may fail so properly handle # those cases in order to avoid content display errors try: filetype = service.lookup_content_filetype(query_string) language = service.lookup_content_language(query_string) license = service.lookup_content_license(query_string) - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) mimetype = 'unknown' encoding = 'unknown' if filetype: mimetype = filetype['mimetype'] encoding = filetype['encoding'] # workaround when encountering corrupted data due to implicit # conversion from bytea to text in the indexer db (see T818) # TODO: Remove that code when all data have been correctly converted if mimetype.startswith('\\'): filetype = None content_data['error_code'] = 200 content_data['error_message'] = '' content_data['error_description'] = '' if not max_size or content_data['length'] < max_size: try: content_raw = service.lookup_content_raw(query_string) - except Exception as e: + except Exception as exc: if raise_if_unavailable: - raise e + raise exc else: + sentry_sdk.capture_exception(exc) content_data['raw_data'] = None content_data['error_code'] = 404 content_data['error_description'] = \ 'The bytes of the content are currently not available in the archive.' # noqa content_data['error_message'] = \ http_status_code_message[content_data['error_code']] else: content_data['raw_data'] = content_raw['data'] if not filetype: mimetype, encoding = \ get_mimetype_and_encoding_for_content(content_data['raw_data']) # noqa if re_encode: mimetype, encoding, raw_data = _re_encode_content( mimetype, encoding, content_data['raw_data']) content_data['raw_data'] = raw_data else: content_data['raw_data'] = None content_data['mimetype'] = mimetype content_data['encoding'] = encoding if language: content_data['language'] = language['lang'] else: content_data['language'] = 'not detected' if license: content_data['licenses'] = ', '.join(license['facts'][0]['licenses']) else: content_data['licenses'] = 'not detected' return content_data _browsers_supported_image_mimes = set(['image/gif', 'image/png', 'image/jpeg', 'image/bmp', 'image/webp', 'image/svg', 'image/svg+xml']) def prepare_content_for_display(content_data, mime_type, path): """Function that prepares a content for HTML display. The function tries to associate a programming language to a content in order to perform syntax highlighting client-side using highlightjs. The language is determined using either the content filename or its mime type. If the mime type corresponds to an image format supported by web browsers, the content will be encoded in base64 for displaying the image. Args: content_data (bytes): raw bytes of the content mime_type (string): mime type of the content path (string): path of the content including filename Returns: A dict containing the content bytes (possibly different from the one provided as parameter if it is an image) under the key 'content_data and the corresponding highlightjs language class under the key 'language'. """ language = highlightjs.get_hljs_language_from_filename(path) if not language: language = highlightjs.get_hljs_language_from_mime_type(mime_type) if not language: language = 'nohighlight' elif mime_type.startswith('application/'): mime_type = mime_type.replace('application/', 'text/') if mime_type.startswith('image/'): if mime_type in _browsers_supported_image_mimes: content_data = base64.b64encode(content_data) content_data = content_data.decode('utf-8') else: content_data = None if mime_type.startswith('image/svg'): mime_type = 'image/svg+xml' return {'content_data': content_data, 'language': language, 'mimetype': mime_type} def process_snapshot_branches(snapshot): """ Process a dictionary describing snapshot branches: extract those targeting revisions and releases, put them in two different lists, then sort those lists in lexicographical order of the branches' names. Args: snapshot_branches (dict): A dict describing the branches of a snapshot as returned for instance by :func:`swh.web.common.service.lookup_snapshot` Returns: tuple: A tuple whose first member is the sorted list of branches targeting revisions and second member the sorted list of branches targeting releases """ snapshot_branches = snapshot['branches'] branches = {} branch_aliases = {} releases = {} revision_to_branch = defaultdict(set) revision_to_release = defaultdict(set) release_to_branch = defaultdict(set) for branch_name, target in snapshot_branches.items(): if not target: # FIXME: display branches with an unknown target anyway continue target_id = target['target'] target_type = target['target_type'] if target_type == 'revision': branches[branch_name] = { 'name': branch_name, 'revision': target_id, } revision_to_branch[target_id].add(branch_name) elif target_type == 'release': release_to_branch[target_id].add(branch_name) elif target_type == 'alias': branch_aliases[branch_name] = target_id # FIXME: handle pointers to other object types def _enrich_release_branch(branch, release): releases[branch] = { 'name': release['name'], 'branch_name': branch, 'date': format_utc_iso_date(release['date']), 'id': release['id'], 'message': release['message'], 'target_type': release['target_type'], 'target': release['target'], } def _enrich_revision_branch(branch, revision): branches[branch].update({ 'revision': revision['id'], 'directory': revision['directory'], 'date': format_utc_iso_date(revision['date']), 'message': revision['message'] }) releases_info = service.lookup_release_multiple( release_to_branch.keys() ) for release in releases_info: branches_to_update = release_to_branch[release['id']] for branch in branches_to_update: _enrich_release_branch(branch, release) if release['target_type'] == 'revision': revision_to_release[release['target']].update( branches_to_update ) revisions = service.lookup_revision_multiple( set(revision_to_branch.keys()) | set(revision_to_release.keys()) ) for revision in revisions: if not revision: continue for branch in revision_to_branch[revision['id']]: _enrich_revision_branch(branch, revision) for release in revision_to_release[revision['id']]: releases[release]['directory'] = revision['directory'] for branch_alias, branch_target in branch_aliases.items(): if branch_target in branches: branches[branch_alias] = dict(branches[branch_target]) else: snp = service.lookup_snapshot(snapshot['id'], branches_from=branch_target, branches_count=1) if snp and branch_target in snp['branches']: if snp['branches'][branch_target] is None: continue target_type = snp['branches'][branch_target]['target_type'] target = snp['branches'][branch_target]['target'] if target_type == 'revision': branches[branch_alias] = snp['branches'][branch_target] revision = service.lookup_revision(target) _enrich_revision_branch(branch_alias, revision) elif target_type == 'release': release = service.lookup_release(target) _enrich_release_branch(branch_alias, release) if branch_alias in branches: branches[branch_alias]['name'] = branch_alias ret_branches = list(sorted(branches.values(), key=lambda b: b['name'])) ret_releases = list(sorted(releases.values(), key=lambda b: b['name'])) return ret_branches, ret_releases def get_snapshot_content(snapshot_id): """Returns the lists of branches and releases associated to a swh snapshot. That list is put in cache in order to speedup the navigation in the swh-web/browse ui. .. warning:: At most 1000 branches contained in the snapshot will be returned for performance reasons. Args: snapshot_id (str): hexadecimal representation of the snapshot identifier Returns: A tuple with two members. The first one is a list of dict describing the snapshot branches. The second one is a list of dict describing the snapshot releases. Raises: NotFoundExc if the snapshot does not exist """ cache_entry_id = 'swh_snapshot_%s' % snapshot_id cache_entry = cache.get(cache_entry_id) if cache_entry: return cache_entry['branches'], cache_entry['releases'] branches = [] releases = [] if snapshot_id: snapshot = service.lookup_snapshot( snapshot_id, branches_count=snapshot_content_max_size) branches, releases = process_snapshot_branches(snapshot) cache.set(cache_entry_id, { 'branches': branches, 'releases': releases, }) return branches, releases def get_origin_visit_snapshot(origin_info, visit_ts=None, visit_id=None, snapshot_id=None): """Returns the lists of branches and releases associated to a swh origin for a given visit. The visit is expressed by a timestamp. In the latter case, the closest visit from the provided timestamp will be used. If no visit parameter is provided, it returns the list of branches found for the latest visit. That list is put in cache in order to speedup the navigation in the swh-web/browse ui. .. warning:: At most 1000 branches contained in the snapshot will be returned for performance reasons. Args: origin_info (dict): a dict filled with origin information (id, url, type) visit_ts (int or str): an ISO date string or Unix timestamp to parse visit_id (int): optional visit id for disambiguation in case several visits have the same timestamp Returns: A tuple with two members. The first one is a list of dict describing the origin branches for the given visit. The second one is a list of dict describing the origin releases for the given visit. Raises: NotFoundExc if the origin or its visit are not found """ visit_info = get_origin_visit(origin_info, visit_ts, visit_id, snapshot_id) return get_snapshot_content(visit_info['snapshot']) def gen_link(url, link_text=None, link_attrs=None): """ Utility function for generating an HTML link to insert in Django templates. Args: url (str): an url link_text (str): optional text for the produced link, if not provided the url will be used link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ attrs = ' ' if link_attrs: for k, v in link_attrs.items(): attrs += '%s="%s" ' % (k, v) if not link_text: link_text = url link = '%s' \ % (attrs, escape(url), escape(link_text)) return mark_safe(link) def _snapshot_context_query_params(snapshot_context): query_params = None if snapshot_context and snapshot_context['origin_info']: origin_info = snapshot_context['origin_info'] query_params = {'origin': origin_info['url']} if 'timestamp' in snapshot_context['url_args']: query_params['timestamp'] = \ snapshot_context['url_args']['timestamp'] if 'visit_id' in snapshot_context['query_params']: query_params['visit_id'] = \ snapshot_context['query_params']['visit_id'] elif snapshot_context: query_params = {'snapshot_id': snapshot_context['snapshot_id']} return query_params def gen_revision_url(revision_id, snapshot_context=None): """ Utility function for generating an url to a revision. Args: revision_id (str): a revision id snapshot_context (dict): if provided, generate snapshot-dependent browsing url Returns: str: The url to browse the revision """ query_params = _snapshot_context_query_params(snapshot_context) return reverse('browse-revision', url_args={'sha1_git': revision_id}, query_params=query_params) def gen_revision_link(revision_id, shorten_id=False, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a revision HTML view to insert in Django templates. Args: revision_id (str): a revision id shorten_id (boolean): whether to shorten the revision id to 7 characters for the link text snapshot_context (dict): if provided, generate snapshot-dependent browsing link link_text (str): optional text for the generated link (the revision id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: str: An HTML link in the form 'revision_id' """ if not revision_id: return None revision_url = gen_revision_url(revision_id, snapshot_context) if shorten_id: return gen_link(revision_url, revision_id[:7], link_attrs) else: if not link_text: link_text = revision_id return gen_link(revision_url, link_text, link_attrs) def gen_directory_link(sha1_git, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a directory HTML view to insert in Django templates. Args: sha1_git (str): directory identifier link_text (str): optional text for the generated link (the directory id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ if not sha1_git: return None query_params = _snapshot_context_query_params(snapshot_context) directory_url = reverse('browse-directory', url_args={'sha1_git': sha1_git}, query_params=query_params) if not link_text: link_text = sha1_git return gen_link(directory_url, link_text, link_attrs) def gen_snapshot_link(snapshot_id, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a snapshot HTML view to insert in Django templates. Args: snapshot_id (str): snapshot identifier link_text (str): optional text for the generated link (the snapshot id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ query_params = _snapshot_context_query_params(snapshot_context) snapshot_url = reverse('browse-snapshot', url_args={'snapshot_id': snapshot_id}, query_params=query_params) if not link_text: link_text = snapshot_id return gen_link(snapshot_url, link_text, link_attrs) def gen_content_link(sha1_git, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a content HTML view to insert in Django templates. Args: sha1_git (str): content identifier link_text (str): optional text for the generated link (the content sha1_git will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ if not sha1_git: return None query_params = _snapshot_context_query_params(snapshot_context) content_url = reverse('browse-content', url_args={'query_string': 'sha1_git:' + sha1_git}, query_params=query_params) if not link_text: link_text = sha1_git return gen_link(content_url, link_text, link_attrs) def get_revision_log_url(revision_id, snapshot_context=None): """ Utility function for getting the URL for a revision log HTML view (possibly in the context of an origin). Args: revision_id (str): revision identifier the history heads to snapshot_context (dict): if provided, generate snapshot-dependent browsing link Returns: The revision log view URL """ query_params = {'revision': revision_id} if snapshot_context and snapshot_context['origin_info']: origin_info = snapshot_context['origin_info'] url_args = {'origin_url': origin_info['url']} if 'timestamp' in snapshot_context['url_args']: url_args['timestamp'] = \ snapshot_context['url_args']['timestamp'] if 'visit_id' in snapshot_context['query_params']: query_params['visit_id'] = \ snapshot_context['query_params']['visit_id'] revision_log_url = reverse('browse-origin-log', url_args=url_args, query_params=query_params) elif snapshot_context: url_args = {'snapshot_id': snapshot_context['snapshot_id']} revision_log_url = reverse('browse-snapshot-log', url_args=url_args, query_params=query_params) else: revision_log_url = reverse('browse-revision-log', url_args={'sha1_git': revision_id}) return revision_log_url def gen_revision_log_link(revision_id, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a revision log HTML view (possibly in the context of an origin) to insert in Django templates. Args: revision_id (str): revision identifier the history heads to snapshot_context (dict): if provided, generate snapshot-dependent browsing link link_text (str): optional text to use for the generated link (the revision id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ if not revision_id: return None revision_log_url = get_revision_log_url(revision_id, snapshot_context) if not link_text: link_text = revision_id return gen_link(revision_log_url, link_text, link_attrs) def gen_person_mail_link(person, link_text=None): """ Utility function for generating a mail link to a person to insert in Django templates. Args: person (dict): dictionary containing person data (*name*, *email*, *fullname*) link_text (str): optional text to use for the generated mail link (the person name will be used by default) Returns: str: A mail link to the person or the person name if no email is present in person data """ person_name = person['name'] or person['fullname'] or 'None' if link_text is None: link_text = person_name person_email = person['email'] if person['email'] else None if person_email is None and '@' in person_name and ' ' not in person_name: person_email = person_name if person_email: return gen_link(url='mailto:%s' % person_email, link_text=link_text) else: return person_name def gen_release_link(sha1_git, snapshot_context=None, link_text='Browse', link_attrs={'class': 'btn btn-default btn-sm', 'role': 'button'}): """ Utility function for generating a link to a release HTML view to insert in Django templates. Args: sha1_git (str): release identifier link_text (str): optional text for the generated link (the release id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ query_params = _snapshot_context_query_params(snapshot_context) release_url = reverse('browse-release', url_args={'sha1_git': sha1_git}, query_params=query_params) if not link_text: link_text = sha1_git return gen_link(release_url, link_text, link_attrs) def format_log_entries(revision_log, per_page, snapshot_context=None): """ Utility functions that process raw revision log data for HTML display. Its purpose is to: * add links to relevant browse views * format date in human readable format * truncate the message log Args: revision_log (list): raw revision log as returned by the swh-web api per_page (int): number of log entries per page snapshot_context (dict): if provided, generate snapshot-dependent browsing link """ revision_log_data = [] for i, rev in enumerate(revision_log): if i == per_page: break author_name = 'None' author_fullname = 'None' committer_fullname = 'None' if rev['author']: author_name = gen_person_mail_link(rev['author']) author_fullname = rev['author']['fullname'] if rev['committer']: committer_fullname = rev['committer']['fullname'] author_date = format_utc_iso_date(rev['date']) committer_date = format_utc_iso_date(rev['committer_date']) tooltip = 'revision %s\n' % rev['id'] tooltip += 'author: %s\n' % author_fullname tooltip += 'author date: %s\n' % author_date tooltip += 'committer: %s\n' % committer_fullname tooltip += 'committer date: %s\n\n' % committer_date if rev['message']: tooltip += textwrap.indent(rev['message'], ' '*4) revision_log_data.append({ 'author': author_name, 'id': rev['id'][:7], 'message': rev['message'], 'date': author_date, 'commit_date': committer_date, 'url': gen_revision_url(rev['id'], snapshot_context), 'tooltip': tooltip }) return revision_log_data def get_snapshot_context(snapshot_id=None, origin_url=None, timestamp=None, visit_id=None): """ Utility function to compute relevant information when navigating the archive in a snapshot context. The snapshot is either referenced by its id or it will be retrieved from an origin visit. Args: snapshot_id (str): hexadecimal representation of a snapshot identifier, all other parameters will be ignored if it is provided origin_url (str): the origin_url (e.g. https://github.com/(user)/(repo)/) timestamp (str): a datetime string for retrieving the closest visit of the origin visit_id (int): optional visit id for disambiguation in case of several visits with the same timestamp Returns: A dict with the following entries: * origin_info: dict containing origin information * visit_info: dict containing visit information * branches: the list of branches for the origin found during the visit * releases: the list of releases for the origin found during the visit * origin_browse_url: the url to browse the origin * origin_branches_url: the url to browse the origin branches * origin_releases_url': the url to browse the origin releases * origin_visit_url: the url to browse the snapshot of the origin found during the visit * url_args: dict containing url arguments to use when browsing in the context of the origin and its visit Raises: NotFoundExc: if no snapshot is found for the visit of an origin. """ origin_info = None visit_info = None url_args = None query_params = {} branches = [] releases = [] browse_url = None visit_url = None branches_url = None releases_url = None swh_type = 'snapshot' if origin_url: swh_type = 'origin' origin_info = service.lookup_origin({'url': origin_url}) visit_info = get_origin_visit(origin_info, timestamp, visit_id, snapshot_id) fmt_date = format_utc_iso_date(visit_info['date']) visit_info['fmt_date'] = fmt_date snapshot_id = visit_info['snapshot'] if not snapshot_id: raise NotFoundExc('No snapshot associated to the visit of origin ' '%s on %s' % (escape(origin_url), fmt_date)) # provided timestamp is not necessarily equals to the one # of the retrieved visit, so get the exact one in order # use it in the urls generated below if timestamp: timestamp = visit_info['date'] branches, releases = \ get_origin_visit_snapshot(origin_info, timestamp, visit_id, snapshot_id) url_args = {'origin_url': origin_info['url']} query_params = {'visit_id': visit_id} browse_url = reverse('browse-origin-visits', url_args=url_args) if timestamp: url_args['timestamp'] = format_utc_iso_date(timestamp, '%Y-%m-%dT%H:%M:%S') visit_url = reverse('browse-origin-directory', url_args=url_args, query_params=query_params) visit_info['url'] = visit_url branches_url = reverse('browse-origin-branches', url_args=url_args, query_params=query_params) releases_url = reverse('browse-origin-releases', url_args=url_args, query_params=query_params) elif snapshot_id: branches, releases = get_snapshot_content(snapshot_id) url_args = {'snapshot_id': snapshot_id} browse_url = reverse('browse-snapshot', url_args=url_args) branches_url = reverse('browse-snapshot-branches', url_args=url_args) releases_url = reverse('browse-snapshot-releases', url_args=url_args) releases = list(reversed(releases)) snapshot_sizes = service.lookup_snapshot_sizes(snapshot_id) is_empty = sum(snapshot_sizes.values()) == 0 swh_snp_id = persistent_identifier('snapshot', snapshot_id) return { 'swh_type': swh_type, 'swh_object_id': swh_snp_id, 'snapshot_id': snapshot_id, 'snapshot_sizes': snapshot_sizes, 'is_empty': is_empty, 'origin_info': origin_info, 'visit_info': visit_info, 'branches': branches, 'releases': releases, 'branch': None, 'release': None, 'browse_url': browse_url, 'branches_url': branches_url, 'releases_url': releases_url, 'url_args': url_args, 'query_params': query_params } # list of common readme names ordered by preference # (lower indices have higher priority) _common_readme_names = [ "readme.markdown", "readme.md", "readme.rst", "readme.txt", "readme" ] def get_readme_to_display(readmes): """ Process a list of readme files found in a directory in order to find the adequate one to display. Args: readmes: a list of dict where keys are readme file names and values are readme sha1s Returns: A tuple (readme_name, readme_sha1) """ readme_name = None readme_url = None readme_sha1 = None readme_html = None lc_readmes = {k.lower(): {'orig_name': k, 'sha1': v} for k, v in readmes.items()} # look for readme names according to the preference order # defined by the _common_readme_names list for common_readme_name in _common_readme_names: if common_readme_name in lc_readmes: readme_name = lc_readmes[common_readme_name]['orig_name'] readme_sha1 = lc_readmes[common_readme_name]['sha1'] readme_url = reverse('browse-content-raw', url_args={'query_string': readme_sha1}, query_params={'re_encode': 'true'}) break # otherwise pick the first readme like file if any if not readme_name and len(readmes.items()) > 0: readme_name = next(iter(readmes)) readme_sha1 = readmes[readme_name] readme_url = reverse('browse-content-raw', url_args={'query_string': readme_sha1}, query_params={'re_encode': 'true'}) # convert rst README to html server side as there is # no viable solution to perform that task client side if readme_name and readme_name.endswith('.rst'): cache_entry_id = 'readme_%s' % readme_sha1 cache_entry = cache.get(cache_entry_id) if cache_entry: readme_html = cache_entry else: try: rst_doc = request_content(readme_sha1) readme_html = pypandoc.convert_text(rst_doc['raw_data'], 'html', format='rst') cache.set(cache_entry_id, readme_html) - except Exception: + except Exception as exc: + sentry_sdk.capture_exception(exc) readme_html = 'Readme bytes are not available' return readme_name, readme_url, readme_html def get_swh_persistent_ids(swh_objects, snapshot_context=None): """ Returns a list of dict containing info related to persistent identifiers of swh objects. Args: swh_objects (list): a list of dict with the following keys: * type: swh object type (content/directory/release/revision/snapshot) * id: swh object id snapshot_context (dict): optional parameter describing the snapshot in which the object has been found Returns: list: a list of dict with the following keys: * object_type: the swh object type (content/directory/release/revision/snapshot) * object_icon: the swh object icon to use in HTML views * swh_id: the computed swh object persistent identifier * swh_id_url: the url resolving the persistent identifier * show_options: boolean indicating if the persistent id options must be displayed in persistent ids HTML view """ swh_ids = [] for swh_object in swh_objects: if not swh_object['id']: continue swh_id = get_swh_persistent_id(swh_object['type'], swh_object['id']) show_options = swh_object['type'] == 'content' or \ (snapshot_context and snapshot_context['origin_info'] is not None) object_icon = swh_object_icons[swh_object['type']] swh_ids.append({ 'object_type': swh_object['type'], 'object_id': swh_object['id'], 'object_icon': object_icon, 'swh_id': swh_id, 'swh_id_url': reverse('browse-swh-id', url_args={'swh_id': swh_id}), 'show_options': show_options }) return swh_ids diff --git a/swh/web/browse/views/content.py b/swh/web/browse/views/content.py index 75220605..30c15537 100644 --- a/swh/web/browse/views/content.py +++ b/swh/web/browse/views/content.py @@ -1,332 +1,334 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import difflib import json from distutils.util import strtobool from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat +import sentry_sdk from swh.model.hashutil import hash_to_hex from swh.web.common import query, service, highlightjs from swh.web.common.utils import ( reverse, gen_path_info, swh_object_icons ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.utils import ( request_content, prepare_content_for_display, content_display_max_size, get_snapshot_context, get_swh_persistent_ids, gen_link, gen_directory_link ) from swh.web.browse.browseurls import browse_route @browse_route(r'content/(?P[0-9a-z_:]*[0-9a-f]+.)/raw/', view_name='browse-content-raw', checksum_args=['query_string']) def content_raw(request, query_string): """Django view that produces a raw display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/raw/` """ try: re_encode = bool(strtobool(request.GET.get('re_encode', 'false'))) algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) content_data = request_content(query_string, max_size=None, re_encode=re_encode) except Exception as exc: return handle_view_exception(request, exc) filename = request.GET.get('filename', None) if not filename: filename = '%s_%s' % (algo, checksum) if content_data['mimetype'].startswith('text/') or \ content_data['mimetype'] == 'inode/x-empty': response = HttpResponse(content_data['raw_data'], content_type="text/plain") response['Content-disposition'] = 'filename=%s' % filename else: response = HttpResponse(content_data['raw_data'], content_type='application/octet-stream') response['Content-disposition'] = 'attachment; filename=%s' % filename return response _auto_diff_size_limit = 20000 @browse_route(r'content/(?P.*)/diff/(?P.*)', # noqa view_name='diff-contents') def _contents_diff(request, from_query_string, to_query_string): """ Browse endpoint used to compute unified diffs between two contents. Diffs are generated only if the two contents are textual. By default, diffs whose size are greater than 20 kB will not be generated. To force the generation of large diffs, the 'force' boolean query parameter must be used. Args: request: input django http request from_query_string: a string of the form "[ALGO_HASH:]HASH" where optional ALGO_HASH can be either ``sha1``, ``sha1_git``, ``sha256``, or ``blake2s256`` (default to ``sha1``) and HASH the hexadecimal representation of the hash value identifying the first content to_query_string: same as above for identifying the second content Returns: A JSON object containing the unified diff. """ diff_data = {} content_from = None content_to = None content_from_size = 0 content_to_size = 0 content_from_lines = [] content_to_lines = [] force = request.GET.get('force', 'false') path = request.GET.get('path', None) language = 'nohighlight' force = bool(strtobool(force)) if from_query_string == to_query_string: diff_str = 'File renamed without changes' else: try: text_diff = True if from_query_string: content_from = \ request_content(from_query_string, max_size=None) content_from_display_data = \ prepare_content_for_display(content_from['raw_data'], content_from['mimetype'], path) language = content_from_display_data['language'] content_from_size = content_from['length'] if not (content_from['mimetype'].startswith('text/') or content_from['mimetype'] == 'inode/x-empty'): text_diff = False if text_diff and to_query_string: content_to = request_content(to_query_string, max_size=None) content_to_display_data = prepare_content_for_display( content_to['raw_data'], content_to['mimetype'], path) language = content_to_display_data['language'] content_to_size = content_to['length'] if not (content_to['mimetype'].startswith('text/') or content_to['mimetype'] == 'inode/x-empty'): text_diff = False diff_size = abs(content_to_size - content_from_size) if not text_diff: diff_str = 'Diffs are not generated for non textual content' language = 'nohighlight' elif not force and diff_size > _auto_diff_size_limit: diff_str = 'Large diffs are not automatically computed' language = 'nohighlight' else: if content_from: content_from_lines = \ content_from['raw_data'].decode('utf-8')\ .splitlines(True) if content_from_lines and \ content_from_lines[-1][-1] != '\n': content_from_lines[-1] += '[swh-no-nl-marker]\n' if content_to: content_to_lines = content_to['raw_data'].decode('utf-8')\ .splitlines(True) if content_to_lines and content_to_lines[-1][-1] != '\n': content_to_lines[-1] += '[swh-no-nl-marker]\n' diff_lines = difflib.unified_diff(content_from_lines, content_to_lines) diff_str = ''.join(list(diff_lines)[2:]) - except Exception as e: - diff_str = str(e) + except Exception as exc: + sentry_sdk.capture_exception(exc) + diff_str = str(exc) diff_data['diff_str'] = diff_str diff_data['language'] = language diff_data_json = json.dumps(diff_data, separators=(',', ': ')) return HttpResponse(diff_data_json, content_type='application/json') @browse_route(r'content/(?P[0-9a-z_:]*[0-9a-f]+.)/', view_name='browse-content', checksum_args=['query_string']) def content_display(request, query_string): """Django view that produces an HTML display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/` """ try: algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) content_data = request_content(query_string, raise_if_unavailable=False) origin_url = request.GET.get('origin_url', None) selected_language = request.GET.get('language', None) if not origin_url: origin_url = request.GET.get('origin', None) snapshot_context = None if origin_url: try: snapshot_context = get_snapshot_context(origin_url=origin_url) - except Exception: + except NotFoundExc: raw_cnt_url = reverse('browse-content', url_args={'query_string': query_string}) error_message = \ ('The Software Heritage archive has a content ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the content ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_cnt_url))) raise NotFoundExc(error_message) if snapshot_context: snapshot_context['visit_info'] = None except Exception as exc: return handle_view_exception(request, exc) path = request.GET.get('path', None) content = None language = None mimetype = None if content_data['raw_data'] is not None: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] # Override language with user-selected language if selected_language is not None: language = selected_language available_languages = None if mimetype and 'text/' in mimetype: available_languages = highlightjs.get_supported_languages() root_dir = None filename = None path_info = None directory_id = None directory_url = None query_params = {'origin': origin_url} breadcrumbs = [] if path: split_path = path.split('/') root_dir = split_path[0] filename = split_path[-1] if root_dir != path: path = path.replace(root_dir + '/', '') path = path[:-len(filename)] path_info = gen_path_info(path) dir_url = reverse('browse-directory', url_args={'sha1_git': root_dir}, query_params=query_params) breadcrumbs.append({'name': root_dir[:7], 'url': dir_url}) for pi in path_info: dir_url = reverse('browse-directory', url_args={'sha1_git': root_dir, 'path': pi['path']}, query_params=query_params) breadcrumbs.append({'name': pi['name'], 'url': dir_url}) breadcrumbs.append({'name': filename, 'url': None}) if path and root_dir != path: try: dir_info = service.lookup_directory_with_path(root_dir, path) directory_id = dir_info['target'] except Exception as exc: return handle_view_exception(request, exc) elif root_dir != path: directory_id = root_dir if directory_id: directory_url = gen_directory_link(directory_id) query_params = {'filename': filename} content_raw_url = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params=query_params) content_metadata = { 'sha1': content_data['checksums']['sha1'], 'sha1_git': content_data['checksums']['sha1_git'], 'sha256': content_data['checksums']['sha256'], 'blake2s256': content_data['checksums']['blake2s256'], 'mimetype': content_data['mimetype'], 'encoding': content_data['encoding'], 'size': filesizeformat(content_data['length']), 'language': content_data['language'], 'licenses': content_data['licenses'], 'filename': filename, 'directory': directory_id, 'context-independent directory': directory_url } if filename: content_metadata['filename'] = filename sha1_git = content_data['checksums']['sha1_git'] swh_ids = get_swh_persistent_ids([{'type': 'content', 'id': sha1_git}]) heading = 'Content - %s' % sha1_git if breadcrumbs: content_path = '/'.join([bc['name'] for bc in breadcrumbs]) heading += ' - %s' % content_path return render(request, 'browse/content.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Content', 'swh_object_metadata': content_metadata, 'content': content, 'content_size': content_data['length'], 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'available_languages': available_languages, 'breadcrumbs': breadcrumbs, 'top_right_link': { 'url': content_raw_url, 'icon': swh_object_icons['content'], 'text': 'Raw File' }, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': content_data['error_code'], 'error_message': content_data['error_message'], 'error_description': content_data['error_description']}, status=content_data['error_code']) diff --git a/swh/web/browse/views/directory.py b/swh/web/browse/views/directory.py index 3abcc98a..917ba6a4 100644 --- a/swh/web/browse/views/directory.py +++ b/swh/web/browse/views/directory.py @@ -1,176 +1,177 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from django.http import HttpResponse from django.shortcuts import render, redirect from django.template.defaultfilters import filesizeformat +import sentry_sdk from swh.web.common import service from swh.web.common.utils import ( reverse, gen_path_info ) from swh.web.common.exc import handle_view_exception, NotFoundExc from swh.web.browse.utils import ( get_directory_entries, get_snapshot_context, get_readme_to_display, get_swh_persistent_ids, gen_link ) from swh.web.browse.browseurls import browse_route @browse_route(r'directory/(?P[0-9a-f]+)/', r'directory/(?P[0-9a-f]+)/(?P.+)/', view_name='browse-directory', checksum_args=['sha1_git']) def directory_browse(request, sha1_git, path=None): """Django view for browsing the content of a directory identified by its sha1_git value. The url that points to it is :http:get:`/browse/directory/(sha1_git)/[(path)/]` """ root_sha1_git = sha1_git try: if path: dir_info = service.lookup_directory_with_path(sha1_git, path) sha1_git = dir_info['target'] dirs, files = get_directory_entries(sha1_git) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) snapshot_context = None if origin_url: try: snapshot_context = get_snapshot_context(origin_url=origin_url) - except Exception: + except NotFoundExc: raw_dir_url = reverse('browse-directory', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a directory ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the directory ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_dir_url))) raise NotFoundExc(error_message) if snapshot_context: snapshot_context['visit_info'] = None except Exception as exc: return handle_view_exception(request, exc) path_info = gen_path_info(path) query_params = {'origin': origin_url} breadcrumbs = [] breadcrumbs.append({'name': root_sha1_git[:7], 'url': reverse('browse-directory', url_args={'sha1_git': root_sha1_git}, query_params=query_params)}) for pi in path_info: breadcrumbs.append({'name': pi['name'], 'url': reverse('browse-directory', url_args={'sha1_git': root_sha1_git, 'path': pi['path']}, query_params=query_params)}) path = '' if path is None else (path + '/') for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}, query_params=query_params) else: d['url'] = reverse('browse-directory', url_args={'sha1_git': root_sha1_git, 'path': path + d['name']}, query_params=query_params) sum_file_sizes = 0 readmes = {} for f in files: query_string = 'sha1_git:' + f['target'] f['url'] = reverse('browse-content', url_args={'query_string': query_string}, query_params={'path': root_sha1_git + '/' + path + f['name'], 'origin': origin_url}) if f['length'] is not None: sum_file_sizes += f['length'] f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) sum_file_sizes = filesizeformat(sum_file_sizes) dir_metadata = {"directory": sha1_git, "number of regular files": len(files), "number of subdirectories": len(dirs), "sum of regular file sizes": sum_file_sizes} vault_cooking = { 'directory_context': True, 'directory_id': sha1_git, 'revision_context': False, 'revision_id': None } swh_ids = get_swh_persistent_ids([{'type': 'directory', 'id': sha1_git}]) heading = 'Directory - %s' % sha1_git if breadcrumbs: dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' heading += ' - %s' % dir_path return render(request, 'browse/directory.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Directory', 'swh_object_metadata': dir_metadata, 'dirs': dirs, 'files': files, 'breadcrumbs': breadcrumbs, 'top_right_link': None, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'snapshot_context': snapshot_context, 'vault_cooking': vault_cooking, 'show_actions_menu': True, 'swh_ids': swh_ids}) @browse_route(r'directory/resolve/content-path/(?P[0-9a-f]+)/(?P.+)/', # noqa view_name='browse-directory-resolve-content-path', checksum_args=['sha1_git']) def _directory_resolve_content_path(request, sha1_git, path): """ Internal endpoint redirecting to data url for a specific file path relative to a root directory. """ try: path = os.path.normpath(path) if not path.startswith('../'): dir_info = service.lookup_directory_with_path(sha1_git, path) if dir_info['type'] == 'file': sha1 = dir_info['checksums']['sha1'] data_url = reverse('browse-content-raw', url_args={'query_string': sha1}) return redirect(data_url) - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) return HttpResponse(status=404) diff --git a/swh/web/browse/views/release.py b/swh/web/browse/views/release.py index 5a402dbe..c7fe01f6 100644 --- a/swh/web/browse/views/release.py +++ b/swh/web/browse/views/release.py @@ -1,185 +1,186 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import render +import sentry_sdk from swh.web.common import service from swh.web.common.utils import ( reverse, format_utc_iso_date ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.browseurls import browse_route from swh.web.browse.utils import ( gen_revision_link, get_snapshot_context, gen_link, gen_snapshot_link, get_swh_persistent_ids, gen_directory_link, gen_content_link, gen_release_link, gen_person_mail_link ) @browse_route(r'release/(?P[0-9a-f]+)/', view_name='browse-release', checksum_args=['sha1_git']) def release_browse(request, sha1_git): """ Django view that produces an HTML display of a release identified by its id. The url that points to it is :http:get:`/browse/release/(sha1_git)/`. """ try: release = service.lookup_release(sha1_git) snapshot_context = None origin_info = None snapshot_id = request.GET.get('snapshot_id', None) origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) if origin_url: try: snapshot_context = get_snapshot_context( snapshot_id, origin_url, timestamp, visit_id) - except Exception: + except NotFoundExc: raw_rel_url = reverse('browse-release', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a release ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the release ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_rel_url))) raise NotFoundExc(error_message) origin_info = snapshot_context['origin_info'] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) except Exception as exc: return handle_view_exception(request, exc) release_data = {} release_data['author'] = 'None' if release['author']: release_data['author'] = gen_person_mail_link(release['author']) release_data['date'] = format_utc_iso_date(release['date']) release_data['release'] = sha1_git release_data['name'] = release['name'] release_data['synthetic'] = release['synthetic'] release_data['target'] = release['target'] release_data['target type'] = release['target_type'] if snapshot_context: if release['target_type'] == 'revision': release_data['context-independent target'] = \ gen_revision_link(release['target']) elif release['target_type'] == 'content': release_data['context-independent target'] = \ gen_content_link(release['target']) elif release['target_type'] == 'directory': release_data['context-independent target'] = \ gen_directory_link(release['target']) elif release['target_type'] == 'release': release_data['context-independent target'] = \ gen_release_link(release['target']) release_note_lines = [] if release['message']: release_note_lines = release['message'].split('\n') vault_cooking = None target_link = None if release['target_type'] == 'revision': target_link = gen_revision_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) try: revision = service.lookup_revision(release['target']) vault_cooking = { 'directory_context': True, 'directory_id': revision['directory'], 'revision_context': True, 'revision_id': release['target'] } - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) elif release['target_type'] == 'directory': target_link = gen_directory_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) try: revision = service.lookup_directory(release['target']) vault_cooking = { 'directory_context': True, 'directory_id': revision['directory'], 'revision_context': False, 'revision_id': None } - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) elif release['target_type'] == 'content': target_link = gen_content_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) elif release['target_type'] == 'release': target_link = gen_release_link(release['target'], snapshot_context=snapshot_context, link_text=None, link_attrs=None) release['target_link'] = target_link if snapshot_context: release_data['snapshot'] = snapshot_context['snapshot_id'] if origin_info: release_data['context-independent release'] = \ gen_release_link(release['id']) release_data['origin url'] = gen_link(origin_info['url'], origin_info['url']) browse_snapshot_link = \ gen_snapshot_link(snapshot_context['snapshot_id']) release_data['context-independent snapshot'] = browse_snapshot_link swh_objects = [{'type': 'release', 'id': sha1_git}] if snapshot_context: snapshot_id = snapshot_context['snapshot_id'] if snapshot_id: swh_objects.append({'type': 'snapshot', 'id': snapshot_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) note_header = 'None' if len(release_note_lines) > 0: note_header = release_note_lines[0] release['note_header'] = note_header release['note_body'] = '\n'.join(release_note_lines[1:]) heading = 'Release - %s' % release['name'] if snapshot_context: context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading += ' - %s' % context_found return render(request, 'browse/release.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Release', 'swh_object_metadata': release_data, 'release': release, 'snapshot_context': snapshot_context, 'show_actions_menu': True, 'breadcrumbs': None, 'vault_cooking': vault_cooking, 'top_right_link': None, 'swh_ids': swh_ids}) diff --git a/swh/web/browse/views/revision.py b/swh/web/browse/views/revision.py index 76be95e0..61f41d7c 100644 --- a/swh/web/browse/views/revision.py +++ b/swh/web/browse/views/revision.py @@ -1,525 +1,525 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import hashlib import json import textwrap from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.utils.html import escape from django.utils.safestring import mark_safe from swh.model.identifiers import persistent_identifier from swh.web.common import service from swh.web.common.utils import ( reverse, format_utc_iso_date, gen_path_info, swh_object_icons ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.browseurls import browse_route from swh.web.browse.utils import ( gen_link, gen_revision_link, gen_revision_url, get_snapshot_context, get_revision_log_url, get_directory_entries, gen_directory_link, request_content, prepare_content_for_display, content_display_max_size, gen_snapshot_link, get_readme_to_display, get_swh_persistent_ids, format_log_entries, gen_person_mail_link ) def _gen_content_url(revision, query_string, path, snapshot_context): if snapshot_context: url_args = snapshot_context['url_args'] url_args['path'] = path query_params = snapshot_context['query_params'] query_params['revision'] = revision['id'] content_url = reverse('browse-origin-content', url_args=url_args, query_params=query_params) else: content_path = '%s/%s' % (revision['directory'], path) content_url = reverse('browse-content', url_args={'query_string': query_string}, query_params={'path': content_path}) return content_url def _gen_diff_link(idx, diff_anchor, link_text): if idx < _max_displayed_file_diffs: return gen_link(diff_anchor, link_text) else: return link_text # TODO: put in conf _max_displayed_file_diffs = 1000 def _gen_revision_changes_list(revision, changes, snapshot_context): """ Returns a HTML string describing the file changes introduced in a revision. As this string will be displayed in the browse revision view, links to adequate file diffs are also generated. Args: revision (str): hexadecimal representation of a revision identifier changes (list): list of file changes in the revision snapshot_context (dict): optional origin context used to reverse the content urls Returns: A string to insert in a revision HTML view. """ changes_msg = [] for i, change in enumerate(changes): hasher = hashlib.sha1() from_query_string = '' to_query_string = '' diff_id = 'diff-' if change['from']: from_query_string = 'sha1_git:' + change['from']['target'] diff_id += change['from']['target'] + '-' + change['from_path'] diff_id += '-' if change['to']: to_query_string = 'sha1_git:' + change['to']['target'] diff_id += change['to']['target'] + change['to_path'] change['path'] = change['to_path'] or change['from_path'] url_args = {'from_query_string': from_query_string, 'to_query_string': to_query_string} query_params = {'path': change['path']} change['diff_url'] = reverse('diff-contents', url_args=url_args, query_params=query_params) hasher.update(diff_id.encode('utf-8')) diff_id = hasher.hexdigest() change['id'] = diff_id panel_diff_link = '#panel_' + diff_id if change['type'] == 'modify': change['content_url'] = \ _gen_content_url(revision, to_query_string, change['to_path'], snapshot_context) changes_msg.append('modified: %s' % _gen_diff_link(i, panel_diff_link, change['to_path'])) elif change['type'] == 'insert': change['content_url'] = \ _gen_content_url(revision, to_query_string, change['to_path'], snapshot_context) changes_msg.append('new file: %s' % _gen_diff_link(i, panel_diff_link, change['to_path'])) elif change['type'] == 'delete': parent = service.lookup_revision(revision['parents'][0]) change['content_url'] = \ _gen_content_url(parent, from_query_string, change['from_path'], snapshot_context) changes_msg.append('deleted: %s' % _gen_diff_link(i, panel_diff_link, change['from_path'])) elif change['type'] == 'rename': change['content_url'] = \ _gen_content_url(revision, to_query_string, change['to_path'], snapshot_context) link_text = change['from_path'] + ' → ' + change['to_path'] changes_msg.append('renamed: %s' % _gen_diff_link(i, panel_diff_link, link_text)) if not changes: changes_msg.append('No changes') return mark_safe('\n'.join(changes_msg)) @browse_route(r'revision/(?P[0-9a-f]+)/diff/', view_name='diff-revision', checksum_args=['sha1_git']) def _revision_diff(request, sha1_git): """ Browse internal endpoint to compute revision diff """ try: revision = service.lookup_revision(sha1_git) snapshot_context = None origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) if origin_url: snapshot_context = get_snapshot_context( origin_url=origin_url, timestamp=timestamp, visit_id=visit_id) except Exception as exc: return handle_view_exception(request, exc) changes = service.diff_revision(sha1_git) changes_msg = _gen_revision_changes_list(revision, changes, snapshot_context) diff_data = { 'total_nb_changes': len(changes), 'changes': changes[:_max_displayed_file_diffs], 'changes_msg': changes_msg } diff_data_json = json.dumps(diff_data, separators=(',', ': ')) return HttpResponse(diff_data_json, content_type='application/json') NB_LOG_ENTRIES = 100 @browse_route(r'revision/(?P[0-9a-f]+)/log/', view_name='browse-revision-log', checksum_args=['sha1_git']) def revision_log_browse(request, sha1_git): """ Django view that produces an HTML display of the history log for a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/log/` """ try: per_page = int(request.GET.get('per_page', NB_LOG_ENTRIES)) offset = int(request.GET.get('offset', 0)) revs_ordering = request.GET.get('revs_ordering', 'committer_date') session_key = 'rev_%s_log_ordering_%s' % (sha1_git, revs_ordering) rev_log_session = request.session.get(session_key, None) rev_log = [] revs_walker_state = None if rev_log_session: rev_log = rev_log_session['rev_log'] revs_walker_state = rev_log_session['revs_walker_state'] if len(rev_log) < offset+per_page: revs_walker = \ service.get_revisions_walker(revs_ordering, sha1_git, max_revs=offset+per_page+1, state=revs_walker_state) rev_log += [rev['id'] for rev in revs_walker] revs_walker_state = revs_walker.export_state() revs = rev_log[offset:offset+per_page] revision_log = service.lookup_revision_multiple(revs) request.session[session_key] = { 'rev_log': rev_log, 'revs_walker_state': revs_walker_state } except Exception as exc: return handle_view_exception(request, exc) revs_ordering = request.GET.get('revs_ordering', '') prev_log_url = None if len(rev_log) > offset + per_page: prev_log_url = reverse('browse-revision-log', url_args={'sha1_git': sha1_git}, query_params={'per_page': per_page, 'offset': offset + per_page, 'revs_ordering': revs_ordering}) next_log_url = None if offset != 0: next_log_url = reverse('browse-revision-log', url_args={'sha1_git': sha1_git}, query_params={'per_page': per_page, 'offset': offset - per_page, 'revs_ordering': revs_ordering}) revision_log_data = format_log_entries(revision_log, per_page) swh_rev_id = persistent_identifier('revision', sha1_git) return render(request, 'browse/revision-log.html', {'heading': 'Revision history', 'swh_object_id': swh_rev_id, 'swh_object_name': 'Revisions history', 'swh_object_metadata': None, 'revision_log': revision_log_data, 'revs_ordering': revs_ordering, 'next_log_url': next_log_url, 'prev_log_url': prev_log_url, 'breadcrumbs': None, 'top_right_link': None, 'snapshot_context': None, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': None}) @browse_route(r'revision/(?P[0-9a-f]+)/', r'revision/(?P[0-9a-f]+)/(?P.+)/', view_name='browse-revision', checksum_args=['sha1_git']) def revision_browse(request, sha1_git, extra_path=None): """ Django view that produces an HTML display of a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/`. """ try: revision = service.lookup_revision(sha1_git) origin_info = None snapshot_context = None origin_url = request.GET.get('origin_url', None) if not origin_url: origin_url = request.GET.get('origin', None) timestamp = request.GET.get('timestamp', None) visit_id = request.GET.get('visit_id', None) snapshot_id = request.GET.get('snapshot_id', None) path = request.GET.get('path', None) dir_id = None dirs, files = None, None content_data = None if origin_url: try: snapshot_context = get_snapshot_context( origin_url=origin_url, timestamp=timestamp, visit_id=visit_id) - except Exception: + except NotFoundExc: raw_rev_url = reverse('browse-revision', url_args={'sha1_git': sha1_git}) error_message = \ ('The Software Heritage archive has a revision ' 'with the hash you provided but the origin ' 'mentioned in your request appears broken: %s. ' 'Please check the URL and try again.\n\n' 'Nevertheless, you can still browse the revision ' 'without origin information: %s' % (gen_link(origin_url), gen_link(raw_rev_url))) raise NotFoundExc(error_message) origin_info = snapshot_context['origin_info'] snapshot_id = snapshot_context['snapshot_id'] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) if path: file_info = \ service.lookup_directory_with_path(revision['directory'], path) if file_info['type'] == 'dir': dir_id = file_info['target'] else: query_string = 'sha1_git:' + file_info['target'] content_data = request_content(query_string, raise_if_unavailable=False) else: dir_id = revision['directory'] if dir_id: path = '' if path is None else (path + '/') dirs, files = get_directory_entries(dir_id) except Exception as exc: return handle_view_exception(request, exc) revision_data = {} revision_data['author'] = 'None' if revision['author']: author_link = gen_person_mail_link(revision['author']) revision_data['author'] = author_link revision_data['committer'] = 'None' if revision['committer']: committer_link = gen_person_mail_link(revision['committer']) revision_data['committer'] = committer_link revision_data['committer date'] = \ format_utc_iso_date(revision['committer_date']) revision_data['date'] = format_utc_iso_date(revision['date']) revision_data['directory'] = revision['directory'] if snapshot_context: revision_data['snapshot'] = snapshot_id browse_snapshot_link = \ gen_snapshot_link(snapshot_id) revision_data['context-independent snapshot'] = browse_snapshot_link revision_data['context-independent directory'] = \ gen_directory_link(revision['directory']) revision_data['revision'] = sha1_git revision_data['merge'] = revision['merge'] revision_data['metadata'] = escape(json.dumps(revision['metadata'], sort_keys=True, indent=4, separators=(',', ': '))) if origin_info: revision_data['origin url'] = gen_link(origin_info['url'], origin_info['url']) revision_data['context-independent revision'] = \ gen_revision_link(sha1_git) parents = '' for p in revision['parents']: parent_link = gen_revision_link(p, link_text=None, link_attrs=None, snapshot_context=snapshot_context) parents += parent_link + '
' revision_data['parents'] = mark_safe(parents) revision_data['synthetic'] = revision['synthetic'] revision_data['type'] = revision['type'] message_lines = ['None'] if revision['message']: message_lines = revision['message'].split('\n') parents = [] for p in revision['parents']: parent_url = gen_revision_url(p, snapshot_context) parents.append({'id': p, 'url': parent_url}) path_info = gen_path_info(path) query_params = {'snapshot_id': snapshot_id, 'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id} breadcrumbs = [] breadcrumbs.append({'name': revision['directory'][:7], 'url': reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params)}) for pi in path_info: query_params['path'] = pi['path'] breadcrumbs.append({'name': pi['name'], 'url': reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params)}) vault_cooking = { 'directory_context': False, 'directory_id': None, 'revision_context': True, 'revision_id': sha1_git } swh_objects = [{'type': 'revision', 'id': sha1_git}] content = None content_size = None mimetype = None language = None readme_name = None readme_url = None readme_html = None readmes = {} error_code = 200 error_message = '' error_description = '' if content_data: breadcrumbs[-1]['url'] = None content_size = content_data['length'] mimetype = content_data['mimetype'] if content_data['raw_data']: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] query_params = {} if path: filename = path_info[-1]['name'] query_params['filename'] = path_info[-1]['name'] revision_data['filename'] = filename top_right_link = { 'url': reverse('browse-content-raw', url_args={'query_string': query_string}, query_params=query_params), 'icon': swh_object_icons['content'], 'text': 'Raw File' } swh_objects.append({'type': 'content', 'id': file_info['target']}) error_code = content_data['error_code'] error_message = content_data['error_message'] error_description = content_data['error_description'] else: for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: query_params['path'] = path + d['name'] d['url'] = reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) for f in files: query_params['path'] = path + f['name'] f['url'] = reverse('browse-revision', url_args={'sha1_git': sha1_git}, query_params=query_params) if f['length'] is not None: f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) top_right_link = { 'url': get_revision_log_url(sha1_git, snapshot_context), 'icon': swh_object_icons['revisions history'], 'text': 'History' } vault_cooking['directory_context'] = True vault_cooking['directory_id'] = dir_id swh_objects.append({'type': 'directory', 'id': dir_id}) diff_revision_url = reverse('diff-revision', url_args={'sha1_git': sha1_git}, query_params={'origin': origin_url, 'timestamp': timestamp, 'visit_id': visit_id}) if snapshot_id: swh_objects.append({'type': 'snapshot', 'id': snapshot_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) heading = 'Revision - %s - %s' %\ (sha1_git[:7], textwrap.shorten(message_lines[0], width=70)) if snapshot_context: context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading += ' - %s' % context_found return render(request, 'browse/revision.html', {'heading': heading, 'swh_object_id': swh_ids[0]['swh_id'], 'swh_object_name': 'Revision', 'swh_object_metadata': revision_data, 'message_header': message_lines[0], 'message_body': '\n'.join(message_lines[1:]), 'parents': parents, 'snapshot_context': snapshot_context, 'dirs': dirs, 'files': files, 'content': content, 'content_size': content_size, 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'breadcrumbs': breadcrumbs, 'top_right_link': top_right_link, 'vault_cooking': vault_cooking, 'diff_revision_url': diff_revision_url, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': error_code, 'error_message': error_message, 'error_description': error_description}, status=error_code) diff --git a/swh/web/browse/views/utils/snapshot_context.py b/swh/web/browse/views/utils/snapshot_context.py index cabba7aa..ad3f3247 100644 --- a/swh/web/browse/views/utils/snapshot_context.py +++ b/swh/web/browse/views/utils/snapshot_context.py @@ -1,933 +1,936 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information # Utility module implementing Django views for browsing the archive # in a snapshot context. # Its purpose is to factorize code for the views reachable from the # /origin/.* and /snapshot/.* endpoints. from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.utils.html import escape +import sentry_sdk from swh.model.identifiers import snapshot_identifier from swh.web.browse.utils import ( get_snapshot_context, get_directory_entries, gen_directory_link, gen_revision_link, request_content, gen_content_link, prepare_content_for_display, content_display_max_size, format_log_entries, gen_revision_log_link, gen_release_link, get_readme_to_display, get_swh_persistent_ids, gen_snapshot_link, process_snapshot_branches ) from swh.web.common import service, highlightjs from swh.web.common.exc import ( handle_view_exception, NotFoundExc ) from swh.web.common.utils import ( reverse, gen_path_info, format_utc_iso_date, swh_object_icons ) _empty_snapshot_id = snapshot_identifier({'branches': {}}) def _get_branch(branches, branch_name, snapshot_id): """ Utility function to get a specific branch from a branches list. Its purpose is to get the default HEAD branch as some software origin (e.g those with svn type) does not have it. In that latter case, check if there is a master branch instead and returns it. """ filtered_branches = [b for b in branches if b['name'] == branch_name] if filtered_branches: return filtered_branches[0] elif branch_name == 'HEAD': filtered_branches = [b for b in branches if b['name'].endswith('master')] if filtered_branches: return filtered_branches[0] elif branches: return branches[0] else: # case where a large branches list has been truncated snp = service.lookup_snapshot(snapshot_id, branches_from=branch_name, branches_count=1, target_types=['revision', 'alias']) snp_branch, _ = process_snapshot_branches(snp) if snp_branch and snp_branch[0]['name'] == branch_name: branches.append(snp_branch[0]) return snp_branch[0] def _get_release(releases, release_name, snapshot_id): """ Utility function to get a specific release from a releases list. Returns None if the release can not be found in the list. """ filtered_releases = [r for r in releases if r['name'] == release_name] if filtered_releases: return filtered_releases[0] else: # case where a large branches list has been truncated for branch_name in (release_name, f'refs/tags/{release_name}'): snp = service.lookup_snapshot(snapshot_id, branches_from=branch_name, branches_count=1, target_types=['release']) _, snp_release = process_snapshot_branches(snp) if snp_release and snp_release[0]['name'] == release_name: releases.append(snp_release[0]) return snp_release[0] def _branch_not_found(branch_type, branch, snapshot_id, snapshot_sizes, origin_info, timestamp, visit_id): """ Utility function to raise an exception when a specified branch/release can not be found. """ if branch_type == 'branch': branch_type = 'Branch' branch_type_plural = 'branches' target_type = 'revision' else: branch_type = 'Release' branch_type_plural = 'releases' target_type = 'release' if snapshot_id and snapshot_sizes[target_type] == 0: msg = ('Snapshot with id %s has an empty list' ' of %s!' % (snapshot_id, branch_type_plural)) elif snapshot_id: msg = ('%s %s for snapshot with id %s' ' not found!' % (branch_type, branch, snapshot_id)) elif visit_id and snapshot_sizes[target_type] == 0: msg = ('Origin with url %s' ' for visit with id %s has an empty list' ' of %s!' % (origin_info['url'], visit_id, branch_type_plural)) elif visit_id: msg = ('%s %s associated to visit with' ' id %s for origin with url %s' ' not found!' % (branch_type, branch, visit_id, origin_info['url'])) elif snapshot_sizes[target_type] == 0: msg = ('Origin with url %s' ' for visit with timestamp %s has an empty list' ' of %s!' % (origin_info['url'], timestamp, branch_type_plural)) else: msg = ('%s %s associated to visit with' ' timestamp %s for origin with ' 'url %s not found!' % (branch_type, branch, timestamp, origin_info['url'])) raise NotFoundExc(escape(msg)) def _process_snapshot_request(request, snapshot_id=None, origin_url=None, timestamp=None, path=None, browse_context='directory'): """ Utility function to perform common input request processing for snapshot context views. """ visit_id = request.GET.get('visit_id', None) snapshot_context = get_snapshot_context(snapshot_id, origin_url, timestamp, visit_id) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] branches = snapshot_context['branches'] releases = snapshot_context['releases'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] if snapshot_context['visit_info']: timestamp = format_utc_iso_date(snapshot_context['visit_info']['date'], '%Y-%m-%dT%H:%M:%SZ') snapshot_context['timestamp'] = format_utc_iso_date( snapshot_context['visit_info']['date']) browse_view_name = 'browse-' + swh_type + '-' + browse_context root_sha1_git = None revision_id = request.GET.get('revision', None) release_name = request.GET.get('release', None) release_id = None branch_name = None snapshot_sizes = snapshot_context['snapshot_sizes'] snapshot_total_size = sum(snapshot_sizes.values()) if snapshot_total_size and revision_id: revision = service.lookup_revision(revision_id) root_sha1_git = revision['directory'] branches.append({'name': revision_id, 'revision': revision_id, 'directory': root_sha1_git, 'url': None}) branch_name = revision_id query_params['revision'] = revision_id elif snapshot_total_size and release_name: release = _get_release(releases, release_name, snapshot_context['snapshot_id']) try: root_sha1_git = release['directory'] revision_id = release['target'] release_id = release['id'] query_params['release'] = release_name - except Exception: + except Exception as exc: + sentry_sdk.capture_exception(exc) _branch_not_found('release', release_name, snapshot_id, snapshot_sizes, origin_info, timestamp, visit_id) elif snapshot_total_size: branch_name = request.GET.get('branch', None) if branch_name: query_params['branch'] = branch_name branch = _get_branch(branches, branch_name or 'HEAD', snapshot_context['snapshot_id']) try: branch_name = branch['name'] revision_id = branch['revision'] root_sha1_git = branch['directory'] - except Exception: + except Exception as exc: + sentry_sdk.capture_exception(exc) _branch_not_found('branch', branch_name, snapshot_id, snapshot_sizes, origin_info, timestamp, visit_id) for b in branches: branch_url_args = dict(url_args) branch_query_params = dict(query_params) if 'release' in branch_query_params: del branch_query_params['release'] branch_query_params['branch'] = b['name'] if path: b['path'] = path branch_url_args['path'] = path b['url'] = reverse(browse_view_name, url_args=branch_url_args, query_params=branch_query_params) for r in releases: release_url_args = dict(url_args) release_query_params = dict(query_params) if 'branch' in release_query_params: del release_query_params['branch'] release_query_params['release'] = r['name'] if path: r['path'] = path release_url_args['path'] = path r['url'] = reverse(browse_view_name, url_args=release_url_args, query_params=release_query_params) snapshot_context['query_params'] = query_params snapshot_context['root_sha1_git'] = root_sha1_git snapshot_context['revision_id'] = revision_id snapshot_context['branch'] = branch_name snapshot_context['release'] = release_name snapshot_context['release_id'] = release_id return snapshot_context def browse_snapshot_directory(request, snapshot_id=None, origin_url=None, timestamp=None, path=None): """ Django view implementation for browsing a directory in a snapshot context. """ try: snapshot_context = _process_snapshot_request( request, snapshot_id, origin_url, timestamp, path, browse_context='directory') root_sha1_git = snapshot_context['root_sha1_git'] sha1_git = root_sha1_git if root_sha1_git and path: dir_info = service.lookup_directory_with_path(root_sha1_git, path) sha1_git = dir_info['target'] dirs = [] files = [] if sha1_git: dirs, files = get_directory_entries(sha1_git) except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] revision_id = snapshot_context['revision_id'] snapshot_id = snapshot_context['snapshot_id'] path_info = gen_path_info(path) browse_view_name = 'browse-' + swh_type + '-directory' breadcrumbs = [] if root_sha1_git: breadcrumbs.append({'name': root_sha1_git[:7], 'url': reverse(browse_view_name, url_args=url_args, query_params=query_params)}) for pi in path_info: bc_url_args = dict(url_args) bc_url_args['path'] = pi['path'] breadcrumbs.append({'name': pi['name'], 'url': reverse(browse_view_name, url_args=bc_url_args, query_params=query_params)}) path = '' if path is None else (path + '/') for d in dirs: if d['type'] == 'rev': d['url'] = reverse('browse-revision', url_args={'sha1_git': d['target']}) else: bc_url_args = dict(url_args) bc_url_args['path'] = path + d['name'] d['url'] = reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) sum_file_sizes = 0 readmes = {} browse_view_name = 'browse-' + swh_type + '-content' for f in files: bc_url_args = dict(url_args) bc_url_args['path'] = path + f['name'] f['url'] = reverse(browse_view_name, url_args=bc_url_args, query_params=query_params) if f['length'] is not None: sum_file_sizes += f['length'] f['length'] = filesizeformat(f['length']) if f['name'].lower().startswith('readme'): readmes[f['name']] = f['checksums']['sha1'] readme_name, readme_url, readme_html = get_readme_to_display(readmes) browse_view_name = 'browse-' + swh_type + '-log' history_url = None if snapshot_id != _empty_snapshot_id: history_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) nb_files = None nb_dirs = None dir_path = None if root_sha1_git: nb_files = len(files) nb_dirs = len(dirs) sum_file_sizes = filesizeformat(sum_file_sizes) dir_path = '/' + path browse_dir_link = gen_directory_link(sha1_git) browse_rev_link = gen_revision_link(revision_id) browse_snp_link = gen_snapshot_link(snapshot_id) dir_metadata = {"directory": sha1_git, "context-independent directory": browse_dir_link, "number of regular files": nb_files, "number of subdirectories": nb_dirs, "sum of regular file sizes": sum_file_sizes, "path": dir_path, "revision": revision_id, "context-independent revision": browse_rev_link, "snapshot": snapshot_id, "context-independent snapshot": browse_snp_link} if origin_info: dir_metadata['origin url'] = origin_info['url'] dir_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) dir_metadata['origin visit type'] = visit_info['type'] vault_cooking = { 'directory_context': True, 'directory_id': sha1_git, 'revision_context': True, 'revision_id': revision_id } swh_objects = [{'type': 'directory', 'id': sha1_git}, {'type': 'revision', 'id': revision_id}, {'type': 'snapshot', 'id': snapshot_id}] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) browse_rel_link = gen_release_link(release_id) dir_metadata['release'] = release_id dir_metadata['context-independent release'] = browse_rel_link swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = ('Directory - %s - %s - %s' % (dir_path, snapshot_context['branch'], context_found)) top_right_link = None if not snapshot_context['is_empty']: top_right_link = { 'url': history_url, 'icon': swh_object_icons['revisions history'], 'text': 'History' } return render(request, 'browse/directory.html', {'heading': heading, 'swh_object_name': 'Directory', 'swh_object_metadata': dir_metadata, 'dirs': dirs, 'files': files, 'breadcrumbs': breadcrumbs if root_sha1_git else [], 'top_right_link': top_right_link, 'readme_name': readme_name, 'readme_url': readme_url, 'readme_html': readme_html, 'snapshot_context': snapshot_context, 'vault_cooking': vault_cooking, 'show_actions_menu': True, 'swh_ids': swh_ids}) def browse_snapshot_content(request, snapshot_id=None, origin_url=None, timestamp=None, path=None, selected_language=None): """ Django view implementation for browsing a content in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, origin_url, timestamp, path, browse_context='content') root_sha1_git = snapshot_context['root_sha1_git'] sha1_git = None query_string = None content_data = None directory_id = None split_path = path.split('/') filename = split_path[-1] filepath = path[:-len(filename)] if root_sha1_git: content_info = service.lookup_directory_with_path(root_sha1_git, path) sha1_git = content_info['target'] query_string = 'sha1_git:' + sha1_git content_data = request_content(query_string, raise_if_unavailable=False) if filepath: dir_info = service.lookup_directory_with_path(root_sha1_git, filepath) directory_id = dir_info['target'] else: directory_id = root_sha1_git except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] revision_id = snapshot_context['revision_id'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] snapshot_id = snapshot_context['snapshot_id'] content = None language = None mimetype = None if content_data and content_data['raw_data'] is not None: content_display_data = prepare_content_for_display( content_data['raw_data'], content_data['mimetype'], path) content = content_display_data['content_data'] language = content_display_data['language'] mimetype = content_display_data['mimetype'] # Override language with user-selected language if selected_language is not None: language = selected_language available_languages = None if mimetype and 'text/' in mimetype: available_languages = highlightjs.get_supported_languages() browse_view_name = 'browse-' + swh_type + '-directory' breadcrumbs = [] path_info = gen_path_info(filepath) if root_sha1_git: breadcrumbs.append({'name': root_sha1_git[:7], 'url': reverse(browse_view_name, url_args=url_args, query_params=query_params)}) for pi in path_info: bc_url_args = dict(url_args) bc_url_args['path'] = pi['path'] breadcrumbs.append({'name': pi['name'], 'url': reverse(browse_view_name, url_args=bc_url_args, query_params=query_params)}) breadcrumbs.append({'name': filename, 'url': None}) browse_content_link = gen_content_link(sha1_git) content_raw_url = None if query_string: content_raw_url = reverse('browse-content-raw', url_args={'query_string': query_string}, query_params={'filename': filename}) browse_rev_link = gen_revision_link(revision_id) browse_dir_link = gen_directory_link(directory_id) content_metadata = { 'context-independent content': browse_content_link, 'path': None, 'filename': None, 'directory': directory_id, 'context-independent directory': browse_dir_link, 'revision': revision_id, 'context-independent revision': browse_rev_link, 'snapshot': snapshot_id } cnt_sha1_git = None content_size = None error_code = 200 error_description = '' error_message = '' if content_data: for checksum in content_data['checksums'].keys(): content_metadata[checksum] = content_data['checksums'][checksum] content_metadata['mimetype'] = content_data['mimetype'] content_metadata['encoding'] = content_data['encoding'] content_metadata['size'] = filesizeformat(content_data['length']) content_metadata['language'] = content_data['language'] content_metadata['licenses'] = content_data['licenses'] content_metadata['path'] = '/' + filepath content_metadata['filename'] = filename cnt_sha1_git = content_data['checksums']['sha1_git'] content_size = content_data['length'] error_code = content_data['error_code'] error_message = content_data['error_message'] error_description = content_data['error_description'] if origin_info: content_metadata['origin url'] = origin_info['url'] content_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) content_metadata['origin visit type'] = visit_info['type'] browse_snapshot_link = gen_snapshot_link(snapshot_id) content_metadata['context-independent snapshot'] = browse_snapshot_link swh_objects = [{'type': 'content', 'id': cnt_sha1_git}, {'type': 'directory', 'id': directory_id}, {'type': 'revision', 'id': revision_id}, {'type': 'snapshot', 'id': snapshot_id}] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) browse_rel_link = gen_release_link(release_id) content_metadata['release'] = release_id content_metadata['context-independent release'] = browse_rel_link swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) content_path = '/'.join([bc['name'] for bc in breadcrumbs]) context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = ('Content - %s - %s - %s' % (content_path, snapshot_context['branch'], context_found)) top_right_link = None if not snapshot_context['is_empty']: top_right_link = { 'url': content_raw_url, 'icon': swh_object_icons['content'], 'text': 'Raw File' } return render(request, 'browse/content.html', {'heading': heading, 'swh_object_name': 'Content', 'swh_object_metadata': content_metadata, 'content': content, 'content_size': content_size, 'max_content_size': content_display_max_size, 'mimetype': mimetype, 'language': language, 'available_languages': available_languages, 'breadcrumbs': breadcrumbs if root_sha1_git else [], 'top_right_link': top_right_link, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids, 'error_code': error_code, 'error_message': error_message, 'error_description': error_description}, status=error_code) PER_PAGE = 100 def browse_snapshot_log(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a revision history in a snapshot context. """ try: snapshot_context = _process_snapshot_request( request, snapshot_id, origin_url, timestamp, browse_context='log') revision_id = snapshot_context['revision_id'] per_page = int(request.GET.get('per_page', PER_PAGE)) offset = int(request.GET.get('offset', 0)) revs_ordering = request.GET.get('revs_ordering', 'committer_date') session_key = 'rev_%s_log_ordering_%s' % (revision_id, revs_ordering) rev_log_session = request.session.get(session_key, None) rev_log = [] revs_walker_state = None if rev_log_session: rev_log = rev_log_session['rev_log'] revs_walker_state = rev_log_session['revs_walker_state'] if len(rev_log) < offset+per_page: revs_walker = service.get_revisions_walker( revs_ordering, revision_id, max_revs=offset+per_page+1, state=revs_walker_state) rev_log += [rev['id'] for rev in revs_walker] revs_walker_state = revs_walker.export_state() revs = rev_log[offset:offset+per_page] revision_log = service.lookup_revision_multiple(revs) request.session[session_key] = { 'rev_log': rev_log, 'revs_walker_state': revs_walker_state } except Exception as exc: return handle_view_exception(request, exc) swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] visit_info = snapshot_context['visit_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] snapshot_id = snapshot_context['snapshot_id'] query_params['per_page'] = per_page revs_ordering = request.GET.get('revs_ordering', '') query_params['revs_ordering'] = revs_ordering browse_view_name = 'browse-' + swh_type + '-log' prev_log_url = None if len(rev_log) > offset + per_page: query_params['offset'] = offset + per_page prev_log_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) next_log_url = None if offset != 0: query_params['offset'] = offset - per_page next_log_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) revision_log_data = format_log_entries(revision_log, per_page, snapshot_context) browse_rev_link = gen_revision_link(revision_id) browse_log_link = gen_revision_log_link(revision_id) browse_snp_link = gen_snapshot_link(snapshot_id) revision_metadata = { 'context-independent revision': browse_rev_link, 'context-independent revision history': browse_log_link, 'context-independent snapshot': browse_snp_link, 'snapshot': snapshot_id } if origin_info: revision_metadata['origin url'] = origin_info['url'] revision_metadata['origin visit date'] = format_utc_iso_date( visit_info['date']) revision_metadata['origin visit type'] = visit_info['type'] swh_objects = [{'type': 'revision', 'id': revision_id}, {'type': 'snapshot', 'id': snapshot_id}] release_id = snapshot_context['release_id'] if release_id: swh_objects.append({'type': 'release', 'id': release_id}) browse_rel_link = gen_release_link(release_id) revision_metadata['release'] = release_id revision_metadata['context-independent release'] = browse_rel_link swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] if origin_info: context_found = 'origin: %s' % origin_info['url'] heading = ('Revision history - %s - %s' % (snapshot_context['branch'], context_found)) return render(request, 'browse/revision-log.html', {'heading': heading, 'swh_object_name': 'Revisions history', 'swh_object_metadata': revision_metadata, 'revision_log': revision_log_data, 'revs_ordering': revs_ordering, 'next_log_url': next_log_url, 'prev_log_url': prev_log_url, 'breadcrumbs': None, 'top_right_link': None, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': True, 'swh_ids': swh_ids}) def browse_snapshot_branches(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a list of branches in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, origin_url, timestamp) branches_bc = request.GET.get('branches_breadcrumbs', '') branches_bc = branches_bc.split(',') if branches_bc else [] branches_from = branches_bc[-1] if branches_bc else '' swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] browse_view_name = 'browse-' + swh_type + '-directory' snapshot = service.lookup_snapshot(snapshot_context['snapshot_id'], branches_from, PER_PAGE+1, target_types=['revision', 'alias']) displayed_branches, _ = process_snapshot_branches(snapshot) except Exception as exc: return handle_view_exception(request, exc) for branch in displayed_branches: if snapshot_id: revision_url = reverse('browse-revision', url_args={'sha1_git': branch['revision']}, query_params={'snapshot_id': snapshot_id}) else: revision_url = reverse('browse-revision', url_args={'sha1_git': branch['revision']}, query_params={'origin': origin_info['url']}) query_params['branch'] = branch['name'] directory_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) del query_params['branch'] branch['revision_url'] = revision_url branch['directory_url'] = directory_url browse_view_name = 'browse-' + swh_type + '-branches' prev_branches_url = None next_branches_url = None if branches_bc: query_params_prev = dict(query_params) query_params_prev['branches_breadcrumbs'] = ','.join(branches_bc[:-1]) prev_branches_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_prev) elif branches_from: prev_branches_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) if len(displayed_branches) > PER_PAGE: query_params_next = dict(query_params) next_branch = displayed_branches[-1]['name'] del displayed_branches[-1] branches_bc.append(next_branch) query_params_next['branches_breadcrumbs'] = ','.join(branches_bc) next_branches_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_next) heading = 'Branches - ' if origin_info: heading += 'origin: %s' % origin_info['url'] else: heading += 'snapshot: %s' % snapshot_id return render(request, 'browse/branches.html', {'heading': heading, 'swh_object_name': 'Branches', 'swh_object_metadata': {}, 'top_right_link': None, 'displayed_branches': displayed_branches, 'prev_branches_url': prev_branches_url, 'next_branches_url': next_branches_url, 'snapshot_context': snapshot_context}) def browse_snapshot_releases(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a list of releases in a snapshot context. """ try: snapshot_context = _process_snapshot_request(request, snapshot_id, origin_url, timestamp) rel_bc = request.GET.get('releases_breadcrumbs', '') rel_bc = rel_bc.split(',') if rel_bc else [] rel_from = rel_bc[-1] if rel_bc else '' swh_type = snapshot_context['swh_type'] origin_info = snapshot_context['origin_info'] url_args = snapshot_context['url_args'] query_params = snapshot_context['query_params'] snapshot = service.lookup_snapshot(snapshot_context['snapshot_id'], rel_from, PER_PAGE+1, target_types=['release', 'alias']) _, displayed_releases = process_snapshot_branches(snapshot) except Exception as exc: return handle_view_exception(request, exc) for release in displayed_releases: if snapshot_id: query_params_tgt = {'snapshot_id': snapshot_id} else: query_params_tgt = {'origin': origin_info['url']} release_url = reverse('browse-release', url_args={'sha1_git': release['id']}, query_params=query_params_tgt) target_url = '' if release['target_type'] == 'revision': target_url = reverse('browse-revision', url_args={'sha1_git': release['target']}, query_params=query_params_tgt) elif release['target_type'] == 'directory': target_url = reverse('browse-directory', url_args={'sha1_git': release['target']}, query_params=query_params_tgt) elif release['target_type'] == 'content': target_url = reverse('browse-content', url_args={'query_string': release['target']}, query_params=query_params_tgt) elif release['target_type'] == 'release': target_url = reverse('browse-release', url_args={'sha1_git': release['target']}, query_params=query_params_tgt) release['release_url'] = release_url release['target_url'] = target_url browse_view_name = 'browse-' + swh_type + '-releases' prev_releases_url = None next_releases_url = None if rel_bc: query_params_prev = dict(query_params) query_params_prev['releases_breadcrumbs'] = ','.join(rel_bc[:-1]) prev_releases_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_prev) elif rel_from: prev_releases_url = reverse(browse_view_name, url_args=url_args, query_params=query_params) if len(displayed_releases) > PER_PAGE: query_params_next = dict(query_params) next_rel = displayed_releases[-1]['branch_name'] del displayed_releases[-1] rel_bc.append(next_rel) query_params_next['releases_breadcrumbs'] = ','.join(rel_bc) next_releases_url = reverse(browse_view_name, url_args=url_args, query_params=query_params_next) heading = 'Releases - ' if origin_info: heading += 'origin: %s' % origin_info['url'] else: heading += 'snapshot: %s' % snapshot_id return render(request, 'browse/releases.html', {'heading': heading, 'top_panel_visible': False, 'top_panel_collapsible': False, 'swh_object_name': 'Releases', 'swh_object_metadata': {}, 'top_right_link': None, 'displayed_releases': displayed_releases, 'prev_releases_url': prev_releases_url, 'next_releases_url': next_releases_url, 'snapshot_context': snapshot_context, 'vault_cooking': None, 'show_actions_menu': False}) diff --git a/swh/web/common/exc.py b/swh/web/common/exc.py index 6be3b09d..63f2725c 100644 --- a/swh/web/common/exc.py +++ b/swh/web/common/exc.py @@ -1,123 +1,125 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import traceback from django.http import HttpResponse from django.shortcuts import render from django.utils.safestring import mark_safe from django.utils.html import escape +import sentry_sdk from swh.web.config import get_config class BadInputExc(ValueError): """Wrong request to the api. Example: Asking a content with the wrong identifier format. """ pass class NotFoundExc(Exception): """Good request to the api but no result were found. Example: Asking a content with the right identifier format but that content does not exist. """ pass class ForbiddenExc(Exception): """Good request to the api, forbidden result to return due to enforce policy. Example: Asking for a raw content which exists but whose mimetype is not text. """ pass http_status_code_message = { 400: 'Bad Request', 401: 'Unauthorized', 403: 'Access Denied', 404: 'Resource not found', 500: 'Internal Server Error', 501: 'Not Implemented', 502: 'Bad Gateway', 503: 'Service unavailable' } def _generate_error_page(request, error_code, error_description): return render(request, 'error.html', {'error_code': error_code, 'error_message': http_status_code_message[error_code], 'error_description': mark_safe(error_description)}, status=error_code) def swh_handle400(request): """ Custom Django HTTP error 400 handler for swh-web. """ error_description = 'The server cannot process the request to %s due to '\ 'something that is perceived to be a client error.' %\ escape(request.META['PATH_INFO']) return _generate_error_page(request, 400, error_description) def swh_handle403(request): """ Custom Django HTTP error 403 handler for swh-web. """ error_description = 'The resource %s requires an authentication.' %\ escape(request.META['PATH_INFO']) return _generate_error_page(request, 403, error_description) def swh_handle404(request): """ Custom Django HTTP error 404 handler for swh-web. """ error_description = 'The resource %s could not be found on the server.' %\ escape(request.META['PATH_INFO']) return _generate_error_page(request, 404, error_description) def swh_handle500(request): """ Custom Django HTTP error 500 handler for swh-web. """ error_description = 'An unexpected condition was encountered when '\ 'requesting resource %s.' %\ escape(request.META['PATH_INFO']) return _generate_error_page(request, 500, error_description) def handle_view_exception(request, exc, html_response=True): """ Function used to generate an error page when an exception was raised inside a swh-web browse view. """ + sentry_sdk.capture_exception(exc) error_code = 500 error_description = '%s: %s' % (type(exc).__name__, str(exc)) if get_config()['debug']: error_description = traceback.format_exc() if isinstance(exc, BadInputExc): error_code = 400 if isinstance(exc, ForbiddenExc): error_code = 403 if isinstance(exc, NotFoundExc): error_code = 404 if html_response: return _generate_error_page(request, error_code, error_description) else: return HttpResponse(error_description, content_type='text/plain', status=error_code) diff --git a/swh/web/common/highlightjs.py b/swh/web/common/highlightjs.py index ad458149..e6b8e124 100644 --- a/swh/web/common/highlightjs.py +++ b/swh/web/common/highlightjs.py @@ -1,361 +1,361 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools - from typing import Dict from pygments.lexers import ( get_all_lexers, get_lexer_for_filename ) +import sentry_sdk # set of languages ids that can be highlighted # by highlight.js library _hljs_languages = set([ '1c', 'abnf', 'accesslog', 'actionscript', 'ada', 'angelscript', 'apache', 'applescript', 'arcade', 'arduino', 'armasm', 'asciidoc', 'aspectj', 'autohotkey', 'autoit', 'avrasm', 'awk', 'axapta', 'bash', 'basic', 'bnf', 'brainfuck', 'cal', 'capnproto', 'ceylon', 'clean', 'clojure', 'clojure-repl', 'cmake', 'coffeescript', 'coq', 'cos', 'cpp', 'crmsh', 'crystal', 'cs', 'csp', 'css', 'd', 'dart', 'delphi', 'diff', 'django', 'dns', 'dockerfile', 'dos', 'dsconfig', 'dts', 'dust', 'ebnf', 'elixir', 'elm', 'erb', 'erlang', 'erlang-repl', 'excel', 'fix', 'flix', 'fortran', 'fsharp', 'gams', 'gauss', 'gcode', 'gherkin', 'glsl', 'gml', 'go', 'golo', 'gradle', 'groovy', 'haml', 'handlebars', 'haskell', 'haxe', 'hsp', 'htmlbars', 'http', 'hy', 'inform7', 'ini', 'irpf90', 'isbl', 'java', 'javascript', 'jboss-cli', 'json', 'julia', 'julia-repl', 'kotlin', 'lasso', 'ldif', 'leaf', 'less', 'lisp', 'livecodeserver', 'livescript', 'llvm', 'lsl', 'lua', 'makefile', 'markdown', 'mathematica', 'matlab', 'maxima', 'mel', 'mercury', 'mipsasm', 'mizar', 'mojolicious', 'monkey', 'moonscript', 'n1ql', 'nginx', 'nimrod', 'nix', 'nsis', 'objectivec', 'ocaml', 'openscad', 'oxygene', 'parser3', 'perl', 'pf', 'pgsql', 'php', 'plaintext', 'pony', 'powershell', 'processing', 'profile', 'prolog', 'properties', 'protobuf', 'puppet', 'purebasic', 'python', 'q', 'qml', 'r', 'reasonml', 'rib', 'roboconf', 'routeros', 'rsl', 'ruby', 'ruleslanguage', 'rust', 'sas', 'scala', 'scheme', 'scilab', 'scss', 'shell', 'smali', 'smalltalk', 'sml', 'sqf', 'sql', 'stan', 'stata', 'step21', 'stylus', 'subunit', 'swift', 'taggerscript', 'tap', 'tcl', 'tex', 'thrift', 'tp', 'twig', 'typescript', 'vala', 'vbnet', 'vbscript', 'vbscript-html', 'verilog', 'vhdl', 'vim', 'x86asm', 'xl', 'xml', 'xquery', 'yaml', 'zephir' ]) # languages aliases defined in highlight.js _hljs_languages_aliases = { 'ado': 'stata', 'adoc': 'asciidoc', 'ahk': 'autohotkey', 'aj': 'aspectj', 'apacheconf': 'apache', 'arm': 'armasm', 'as': 'actionscript', 'asc': 'asciidoc', 'atom': 'xml', 'bas': 'basic', 'bat': 'dos', 'bf': 'brainfuck', 'bind': 'dns', 'bsl': '1c', 'c-al': 'cal', 'c': 'cpp', 'c++': 'cpp', 'capnp': 'capnproto', 'cc': 'cpp', 'clj': 'clojure', 'cls': 'cos', 'cmake.in': 'cmake', 'cmd': 'dos', 'coffee': 'coffeescript', 'console': 'shell', 'cr': 'crystal', 'craftcms': 'twig', 'crm': 'crmsh', 'csharp': 'cs', 'cson': 'coffeescript', 'dcl': 'clean', 'dfm': 'delphi', 'do': 'stata', 'docker': 'dockerfile', 'dpr': 'delphi', 'dst': 'dust', 'dtsi': 'dts', 'ep': 'mojolicious', 'erl': 'erlang', 'ex': 'elixir', 'exs': 'elixir', 'f90': 'fortran', 'f95': 'fortran', 'feature': 'gherkin', 'freepascal': 'delphi', 'fs': 'fsharp', 'fsx': 'fsharp', 'gemspec': 'ruby', 'GML': 'gml', 'gms': 'gams', 'golang': 'go', 'graph': 'roboconf', 'gss': 'gauss', 'gyp': 'python', 'h': 'cpp', 'h++': 'cpp', 'hbs': 'handlebars', 'hpp': 'cpp', 'hs': 'haskell', 'html': 'xml', 'html.handlebars': 'handlebars', 'html.hbs': 'handlebars', 'https': 'http', 'hx': 'haxe', 'hylang': 'hy', 'i7': 'inform7', 'i7x': 'inform7', 'iced': 'coffeescript', 'icl': 'clean', 'ino': 'arduino', 'instances': 'roboconf', 'ipynb': 'json', 'irb': 'ruby', 'jinja': 'django', 'js': 'javascript', 'jsp': 'java', 'jsx': 'javascript', 'k': 'q', 'kdb': 'q', 'kt': 'kotlin', 'lassoscript': 'lasso', 'lazarus': 'delphi', 'lc': 'livecode', 'lfm': 'delphi', 'll': 'llvm', 'lpr': 'delphi', 'ls': 'livescript', 'm': 'matlab', 'mak': 'makefile', 'md': 'markdown', 'mikrotik': 'routeros', 'mips': 'mipsasm', 'mk': 'monkey', 'mkd': 'markdown', 'mkdown': 'markdown', 'ml': 'ocaml', 'mli': 'ocaml', 'mm': 'objectivec', 'mma': 'mathematica', 'moo': 'mercury', 'moon': 'moonscript', 'nav': 'cal', 'nb': 'mathematica', 'nc': 'gcode', 'nginxconf': 'nginx', 'ni': 'inform7', 'nim': 'nimrod', 'nixos': 'nix', 'nsi': 'nsis', 'obj-c': 'objectivec', 'objc': 'objectivec', 'osascript': 'applescript', 'osl': 'rsl', 'p': 'parser3', 'p21': 'step21', 'pas': 'delphi', 'pascal': 'delphi', 'patch': 'diff', 'pb': 'purebasic', 'pbi': 'purebasic', 'pcmk': 'crmsh', 'pde': 'processing', 'pf.conf': 'pf', 'php3': 'php', 'php4': 'php', 'php5': 'php', 'php6': 'php', 'php7': 'php', 'pl': 'perl', 'plist': 'xml', 'pm': 'perl', 'podspec': 'ruby', 'postgres': 'pgsql', 'postgresql': 'pgsql', 'pp': 'puppet', 'proto': 'protobuf', 'ps': 'powershell', 'ps1': 'powershell', 'psd1': 'powershell', 'psm1': 'powershell', 'py': 'python', 'qt': 'qml', 'rb': 'ruby', 're': 'reasonml', 'rei': 'reasonml', 'rs': 'rust', 'rsc': 'routeros', 'rss': 'xml', 'rst': 'nohighlight', 's': 'armasm', 'SAS': 'sas', 'scad': 'openscad', 'sci': 'scilab', 'scm': 'scheme', 'sh': 'bash', 'sig': 'sml', 'sl': 'rsl', 'st': 'smalltalk', 'step': 'step21', 'stp': 'step21', 'styl': 'stylus', 'sv': 'verilog', 'svh': 'verilog', 'tao': 'xl', 'thor': 'ruby', 'tk': 'tcl', 'toml': 'ini', 'ts': 'typescript', 'txt': 'nohighlight', 'v': 'coq', 'vb': 'vbnet', 'vbs': 'vbscript', 'vhd': 'vhdl', 'wildfly-cli': 'jboss-cli', 'wl': 'mathematica', 'wls': 'mathematica', 'xhtml': 'xml', 'xjb': 'xml', 'xls': 'excel', 'xlsx': 'excel', 'xpath': 'xquery', 'xpo': 'axapta', 'xpp': 'axapta', 'xq': 'xquery', 'xqy': 'xquery', 'xsd': 'xml', 'xsl': 'xml', 'YAML': 'yaml', 'yml': 'yaml', 'zep': 'zephir', 'zone': 'dns', 'zsh': 'bash' } # dictionary mapping pygment lexers to hljs languages _pygments_lexer_to_hljs_language = {} # type: Dict[str, str] # dictionary mapping mime types to hljs languages _mime_type_to_hljs_language = { 'text/x-c': 'cpp', 'text/x-c++': 'cpp', 'text/x-msdos-batch': 'dos', 'text/x-lisp': 'lisp', 'text/x-shellscript': 'bash', } # dictionary mapping filenames to hljs languages _filename_to_hljs_language = { 'cmakelists.txt': 'cmake', '.htaccess': 'apache', 'httpd.conf': 'apache', 'access.log': 'accesslog', 'nginx.log': 'accesslog', 'resolv.conf': 'dns', 'dockerfile': 'docker', 'nginx.conf': 'nginx', 'pf.conf': 'pf' } # function to fill the above dictionaries def _init_pygments_to_hljs_map(): if len(_pygments_lexer_to_hljs_language) == 0: for lexer in get_all_lexers(): lexer_name = lexer[0] lang_aliases = lexer[1] lang_mime_types = lexer[3] lang = None for lang_alias in lang_aliases: if lang_alias in _hljs_languages: lang = lang_alias _pygments_lexer_to_hljs_language[lexer_name] = lang_alias break if lang: for lang_mime_type in lang_mime_types: _mime_type_to_hljs_language[lang_mime_type] = lang def get_hljs_language_from_filename(filename): """Function that tries to associate a language supported by highlight.js from a filename. Args: filename: input filename Returns: highlight.js language id or None if no correspondence has been found """ _init_pygments_to_hljs_map() if filename: filename_lower = filename.lower() if filename_lower in _filename_to_hljs_language: return _filename_to_hljs_language[filename_lower] if filename_lower in _hljs_languages: return filename_lower exts = filename_lower.split('.') # check if file extension matches an hljs language # also handle .ext.in cases for ext in reversed(exts[-2:]): if ext in _hljs_languages: return ext if ext in _hljs_languages_aliases: return _hljs_languages_aliases[ext] # otherwise use Pygments language database lexer = None # try to find a Pygment lexer try: lexer = get_lexer_for_filename(filename) - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) # if there is a correspondence between the lexer and an hljs # language, return it if lexer and lexer.name in _pygments_lexer_to_hljs_language: return _pygments_lexer_to_hljs_language[lexer.name] # otherwise, try to find a match between the file extensions # associated to the lexer and the hljs language aliases if lexer: exts = [ext.replace('*.', '') for ext in lexer.filenames] for ext in exts: if ext in _hljs_languages_aliases: return _hljs_languages_aliases[ext] return None def get_hljs_language_from_mime_type(mime_type): """Function that tries to associate a language supported by highlight.js from a mime type. Args: mime_type: input mime type Returns: highlight.js language id or None if no correspondence has been found """ _init_pygments_to_hljs_map() if mime_type and mime_type in _mime_type_to_hljs_language: return _mime_type_to_hljs_language[mime_type] return None @functools.lru_cache() def get_supported_languages(): """ Return the list of programming languages that can be highlighted using the highlight.js library. Returns: List[str]: the list of supported languages """ return sorted(list(_hljs_languages)) diff --git a/swh/web/common/middlewares.py b/swh/web/common/middlewares.py index 3e09e2dc..cb6ca5ac 100644 --- a/swh/web/common/middlewares.py +++ b/swh/web/common/middlewares.py @@ -1,70 +1,71 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from bs4 import BeautifulSoup from htmlmin import minify +import sentry_sdk class HtmlPrettifyMiddleware(object): """ Django middleware for prettifying generated HTML in development mode. """ def __init__(self, get_response): self.get_response = get_response def __call__(self, request): response = self.get_response(request) if 'text/html' in response.get('Content-Type', ''): if hasattr(response, 'content'): content = response.content response.content = BeautifulSoup(content, 'lxml').prettify() elif hasattr(response, 'streaming_content'): content = b''.join(response.streaming_content) response.streaming_content = \ BeautifulSoup(content, 'lxml').prettify() return response class HtmlMinifyMiddleware(object): """ Django middleware for minifying generated HTML in production mode. """ def __init__(self, get_response=None): self.get_response = get_response def __call__(self, request): response = self.get_response(request) if 'text/html' in response.get('Content-Type', ''): try: minified_html = minify(response.content.decode('utf-8')) response.content = minified_html.encode('utf-8') - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) return response class ThrottlingHeadersMiddleware(object): """ Django middleware for inserting rate limiting related headers in HTTP response. """ def __init__(self, get_response=None): self.get_response = get_response def __call__(self, request): resp = self.get_response(request) if 'RateLimit-Limit' in request.META: resp['X-RateLimit-Limit'] = request.META['RateLimit-Limit'] if 'RateLimit-Remaining' in request.META: resp['X-RateLimit-Remaining'] = request.META['RateLimit-Remaining'] if 'RateLimit-Reset' in request.META: resp['X-RateLimit-Reset'] = request.META['RateLimit-Reset'] return resp diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py index e028a448..56c8d70c 100644 --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -1,529 +1,529 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -import json -import logging - from bisect import bisect_right from datetime import datetime, timezone, timedelta - -import requests +import json +import logging from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError from django.core.validators import URLValidator from django.utils.html import escape +import requests +import sentry_sdk + from swh.web import config from swh.web.common import service from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc from swh.web.common.models import ( SaveUnauthorizedOrigin, SaveAuthorizedOrigin, SaveOriginRequest, SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, SAVE_REQUEST_PENDING, SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_SCHEDULED, SAVE_TASK_SUCCEED, SAVE_TASK_FAILED, SAVE_TASK_RUNNING ) from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import parse_timestamp from swh.scheduler.utils import create_oneshot_task_dict scheduler = config.scheduler() logger = logging.getLogger(__name__) def get_origin_save_authorized_urls(): """ Get the list of origin url prefixes authorized to be immediately loaded into the archive (whitelist). Returns: list: The list of authorized origin url prefix """ return [origin.url for origin in SaveAuthorizedOrigin.objects.all()] def get_origin_save_unauthorized_urls(): """ Get the list of origin url prefixes forbidden to be loaded into the archive (blacklist). Returns: list: the list of unauthorized origin url prefix """ return [origin.url for origin in SaveUnauthorizedOrigin.objects.all()] def can_save_origin(origin_url): """ Check if a software origin can be saved into the archive. Based on the origin url, the save request will be either: * immediately accepted if the url is whitelisted * rejected if the url is blacklisted * put in pending state for manual review otherwise Args: origin_url (str): the software origin url to check Returns: str: the origin save request status, either **accepted**, **rejected** or **pending** """ # origin url may be blacklisted for url_prefix in get_origin_save_unauthorized_urls(): if origin_url.startswith(url_prefix): return SAVE_REQUEST_REJECTED # if the origin url is in the white list, it can be immediately saved for url_prefix in get_origin_save_authorized_urls(): if origin_url.startswith(url_prefix): return SAVE_REQUEST_ACCEPTED # otherwise, the origin url needs to be manually verified return SAVE_REQUEST_PENDING # map visit type to scheduler task # TODO: do not hardcode the task name here (T1157) _visit_type_task = { 'git': 'load-git', 'hg': 'load-hg', 'svn': 'load-svn' } # map scheduler task status to origin save status _save_task_status = { 'next_run_not_scheduled': SAVE_TASK_NOT_YET_SCHEDULED, 'next_run_scheduled': SAVE_TASK_SCHEDULED, 'completed': SAVE_TASK_SUCCEED, 'disabled': SAVE_TASK_FAILED } def get_savable_visit_types(): return sorted(list(_visit_type_task.keys())) def _check_visit_type_savable(visit_type): """ Get the list of visit types that can be performed through a save request. Returns: list: the list of saveable visit types """ allowed_visit_types = ', '.join(get_savable_visit_types()) if visit_type not in _visit_type_task: raise BadInputExc('Visit of type %s can not be saved! ' 'Allowed types are the following: %s' % (visit_type, allowed_visit_types)) _validate_url = URLValidator(schemes=['http', 'https', 'svn', 'git']) def _check_origin_url_valid(origin_url): try: _validate_url(origin_url) except ValidationError: raise BadInputExc('The provided origin url (%s) is not valid!' % escape(origin_url)) def _get_visit_info_for_save_request(save_request): visit_date = None visit_status = None try: origin = {'url': save_request.origin_url} origin_info = service.lookup_origin(origin) origin_visits = get_origin_visits(origin_info) visit_dates = [parse_timestamp(v['date']) for v in origin_visits] i = bisect_right(visit_dates, save_request.request_date) if i != len(visit_dates): visit_date = visit_dates[i] visit_status = origin_visits[i]['status'] if origin_visits[i]['status'] == 'ongoing': visit_date = None - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) return visit_date, visit_status def _check_visit_update_status(save_request, save_task_status): visit_date, visit_status = _get_visit_info_for_save_request(save_request) save_request.visit_date = visit_date # visit has been performed, mark the saving task as succeed if visit_date and visit_status is not None: save_task_status = SAVE_TASK_SUCCEED elif visit_status == 'ongoing': save_task_status = SAVE_TASK_RUNNING else: time_now = datetime.now(tz=timezone.utc) time_delta = time_now - save_request.request_date # consider the task as failed if it is still in scheduled state # 30 days after its submission if time_delta.days > 30: save_task_status = SAVE_TASK_FAILED return visit_date, save_task_status def _save_request_dict(save_request, task=None): must_save = False visit_date = save_request.visit_date # save task still in scheduler db if task: save_task_status = _save_task_status[task['status']] # Consider request from which a visit date has already been found # as succeeded to avoid retrieving it again if save_task_status == SAVE_TASK_SCHEDULED and visit_date: save_task_status = SAVE_TASK_SUCCEED if save_task_status in (SAVE_TASK_FAILED, SAVE_TASK_SUCCEED) \ and not visit_date: visit_date, _ = _get_visit_info_for_save_request(save_request) save_request.visit_date = visit_date must_save = True # Check tasks still marked as scheduled / not yet scheduled if save_task_status in (SAVE_TASK_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED): visit_date, save_task_status = _check_visit_update_status( save_request, save_task_status) # save task may have been archived else: save_task_status = save_request.loading_task_status if save_task_status in (SAVE_TASK_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED): visit_date, save_task_status = _check_visit_update_status( save_request, save_task_status) else: save_task_status = save_request.loading_task_status if save_request.loading_task_status != save_task_status: save_request.loading_task_status = save_task_status must_save = True if must_save: save_request.save() return {'id': save_request.id, 'visit_type': save_request.visit_type, 'origin_url': save_request.origin_url, 'save_request_date': save_request.request_date.isoformat(), 'save_request_status': save_request.status, 'save_task_status': save_task_status, 'visit_date': visit_date.isoformat() if visit_date else None} def create_save_origin_request(visit_type, origin_url): """ Create a loading task to save a software origin into the archive. This function aims to create a software origin loading task trough the use of the swh-scheduler component. First, some checks are performed to see if the visit type and origin url are valid but also if the the save request can be accepted. If those checks passed, the loading task is then created. Otherwise, the save request is put in pending or rejected state. All the submitted save requests are logged into the swh-web database to keep track of them. Args: visit_type (str): the type of visit to perform (currently only ``git`` but ``svn`` and ``hg`` will soon be available) origin_url (str): the url of the origin to save Raises: BadInputExc: the visit type or origin url is invalid ForbiddenExc: the provided origin url is blacklisted Returns: dict: A dict describing the save request with the following keys: * **visit_type**: the type of visit to perform * **origin_url**: the url of the origin * **save_request_date**: the date the request was submitted * **save_request_status**: the request status, either **accepted**, **rejected** or **pending** * **save_task_status**: the origin loading task status, either **not created**, **not yet scheduled**, **scheduled**, **succeed** or **failed** """ _check_visit_type_savable(visit_type) _check_origin_url_valid(origin_url) save_request_status = can_save_origin(origin_url) task = None # if the origin save request is accepted, create a scheduler # task to load it into the archive if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority kwargs = { 'priority': 'high', 'url': origin_url, } sor = None # get list of previously sumitted save requests current_sors = \ list(SaveOriginRequest.objects.filter(visit_type=visit_type, origin_url=origin_url)) can_create_task = False # if no save requests previously submitted, create the scheduler task if not current_sors: can_create_task = True else: # get the latest submitted save request sor = current_sors[0] # if it was in pending state, we need to create the scheduler task # and update the save request info in the database if sor.status == SAVE_REQUEST_PENDING: can_create_task = True # a task has already been created to load the origin elif sor.loading_task_id != -1: # get the scheduler task and its status tasks = scheduler.get_tasks([sor.loading_task_id]) task = tasks[0] if tasks else None task_status = _save_request_dict(sor, task)['save_task_status'] # create a new scheduler task only if the previous one has been # already executed if task_status == SAVE_TASK_FAILED or \ task_status == SAVE_TASK_SUCCEED: can_create_task = True sor = None else: can_create_task = False if can_create_task: # effectively create the scheduler task task_dict = create_oneshot_task_dict( _visit_type_task[visit_type], **kwargs) task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted if sor: sor.status = SAVE_REQUEST_ACCEPTED sor.loading_task_id = task['id'] sor.save() else: sor = SaveOriginRequest.objects.create(visit_type=visit_type, origin_url=origin_url, status=save_request_status, # noqa loading_task_id=task['id']) # noqa # save request must be manually reviewed for acceptation elif save_request_status == SAVE_REQUEST_PENDING: # check if there is already such a save request already submitted, # no need to add it to the database in that case try: sor = SaveOriginRequest.objects.get(visit_type=visit_type, origin_url=origin_url, status=save_request_status) # if not add it to the database except ObjectDoesNotExist: sor = SaveOriginRequest.objects.create(visit_type=visit_type, origin_url=origin_url, status=save_request_status) # origin can not be saved as its url is blacklisted, # log the request to the database anyway else: sor = SaveOriginRequest.objects.create(visit_type=visit_type, origin_url=origin_url, status=save_request_status) if save_request_status == SAVE_REQUEST_REJECTED: raise ForbiddenExc(('The "save code now" request has been rejected ' 'because the provided origin url is blacklisted.')) return _save_request_dict(sor, task) def get_save_origin_requests_from_queryset(requests_queryset): """ Get all save requests from a SaveOriginRequest queryset. Args: requests_queryset (django.db.models.QuerySet): input SaveOriginRequest queryset Returns: list: A list of save origin requests dict as described in :func:`swh.web.common.origin_save.create_save_origin_request` """ task_ids = [] for sor in requests_queryset: task_ids.append(sor.loading_task_id) save_requests = [] if task_ids: tasks = scheduler.get_tasks(task_ids) tasks = {task['id']: task for task in tasks} for sor in requests_queryset: sr_dict = _save_request_dict(sor, tasks.get(sor.loading_task_id)) save_requests.append(sr_dict) return save_requests def get_save_origin_requests(visit_type, origin_url): """ Get all save requests for a given software origin. Args: visit_type (str): the type of visit origin_url (str): the url of the origin Raises: BadInputExc: the visit type or origin url is invalid NotFoundExc: no save requests can be found for the given origin Returns: list: A list of save origin requests dict as described in :func:`swh.web.common.origin_save.create_save_origin_request` """ _check_visit_type_savable(visit_type) _check_origin_url_valid(origin_url) sors = SaveOriginRequest.objects.filter(visit_type=visit_type, origin_url=origin_url) if sors.count() == 0: raise NotFoundExc(('No save requests found for visit of type ' '%s on origin with url %s.') % (visit_type, origin_url)) return get_save_origin_requests_from_queryset(sors) def get_save_origin_task_info(save_request_id): """ Get detailed information about an accepted save origin request and its associated loading task. If the associated loading task info is archived and removed from the scheduler database, returns an empty dictionary. Args: save_request_id (int): identifier of a save origin request Returns: dict: A dictionary with the following keys: - **type**: loading task type - **arguments**: loading task arguments - **id**: loading task database identifier - **backend_id**: loading task celery identifier - **scheduled**: loading task scheduling date - **ended**: loading task termination date - **status**: loading task execution status Depending on the availability of the task logs in the elasticsearch cluster of Software Heritage, the returned dictionary may also contain the following keys: - **name**: associated celery task name - **message**: relevant log message from task execution - **duration**: task execution time (only if it succeeded) - **worker**: name of the worker that executed the task """ try: save_request = SaveOriginRequest.objects.get(id=save_request_id) except ObjectDoesNotExist: return {} task = scheduler.get_tasks([save_request.loading_task_id]) task = task[0] if task else None if task is None: return {} task_run = scheduler.get_task_runs([task['id']]) task_run = task_run[0] if task_run else None if task_run is None: return {} task_run['type'] = task['type'] task_run['arguments'] = task['arguments'] task_run['id'] = task_run['task'] del task_run['task'] del task_run['metadata'] del task_run['started'] es_workers_index_url = config.get_config()['es_workers_index_url'] if not es_workers_index_url: return task_run es_workers_index_url += '/_search' if save_request.visit_date: min_ts = save_request.visit_date max_ts = min_ts + timedelta(days=7) else: min_ts = save_request.request_date max_ts = min_ts + timedelta(days=30) min_ts = int(min_ts.timestamp()) * 1000 max_ts = int(max_ts.timestamp()) * 1000 save_task_status = _save_task_status[task['status']] priority = '3' if save_task_status == SAVE_TASK_FAILED else '6' query = { 'bool': { 'must': [ { 'match_phrase': { 'priority': { 'query': priority } } }, { 'match_phrase': { 'swh_task_id': { 'query': task_run['backend_id'] } } }, { 'range': { '@timestamp': { 'gte': min_ts, 'lte': max_ts, 'format': 'epoch_millis' } } } ] } } try: response = requests.post(es_workers_index_url, json={'query': query, 'sort': ['@timestamp']}, timeout=30) results = json.loads(response.text) if results['hits']['total'] >= 1: task_run_info = results['hits']['hits'][-1]['_source'] if 'swh_logging_args_runtime' in task_run_info: duration = task_run_info['swh_logging_args_runtime'] task_run['duration'] = duration if 'message' in task_run_info: task_run['message'] = task_run_info['message'] if 'swh_logging_args_name' in task_run_info: task_run['name'] = task_run_info['swh_logging_args_name'] elif 'swh_task_name' in task_run_info: task_run['name'] = task_run_info['swh_task_name'] if 'hostname' in task_run_info: task_run['worker'] = task_run_info['hostname'] elif 'host' in task_run_info: task_run['worker'] = task_run_info['host'] - except Exception as e: - logger.warning('Request to Elasticsearch failed\n%s' % str(e)) - pass + except Exception as exc: + logger.warning('Request to Elasticsearch failed\n%s', exc) + sentry_sdk.capture_exception(exc) return task_run diff --git a/swh/web/common/swh_templatetags.py b/swh/web/common/swh_templatetags.py index 4a3d5949..8f7261d0 100644 --- a/swh/web/common/swh_templatetags.py +++ b/swh/web/common/swh_templatetags.py @@ -1,185 +1,187 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from inspect import cleandoc import json import re from django import template from django.core.serializers.json import DjangoJSONEncoder from django.utils.safestring import mark_safe from docutils.core import publish_parts from docutils.writers.html4css1 import Writer, HTMLTranslator -from inspect import cleandoc + +import sentry_sdk from swh.web.common.origin_save import get_savable_visit_types register = template.Library() class NoHeaderHTMLTranslator(HTMLTranslator): """ Docutils translator subclass to customize the generation of HTML from reST-formatted docstrings """ def __init__(self, document): super().__init__(document) self.body_prefix = [] self.body_suffix = [] def visit_bullet_list(self, node): self.context.append((self.compact_simple, self.compact_p)) self.compact_p = None self.compact_simple = self.is_compactable(node) self.body.append(self.starttag(node, 'ul', CLASS='docstring')) DOCSTRING_WRITER = Writer() DOCSTRING_WRITER.translator_class = NoHeaderHTMLTranslator @register.filter def safe_docstring_display(docstring): """ Utility function to htmlize reST-formatted documentation in browsable api. """ docstring = cleandoc(docstring) return publish_parts(docstring, writer=DOCSTRING_WRITER)['html_body'] @register.filter def urlize_links_and_mails(text): """Utility function for decorating api links in browsable api. Args: text: whose content matching links should be transformed into contextual API or Browse html links. Returns The text transformed if any link is found. The text as is otherwise. """ try: if 'href="' not in text: text = re.sub(r'(/api/[^"<]*|/browse/[^"<]*|http.*$)', r'\1', text) return re.sub(r'([^ <>"]+@[^ <>"]+)', r'\1', text) - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) return text @register.filter def urlize_header_links(text): """Utility function for decorating headers links in browsable api. Args text: Text whose content contains Link header value Returns: The text transformed with html link if any link is found. The text as is otherwise. """ links = text.split(',') ret = '' for i, link in enumerate(links): ret += re.sub(r'<(/api/.*|/browse/.*)>', r'<\1>', link) # add one link per line and align them if i != len(links) - 1: ret += '\n ' return ret @register.filter def jsonify(obj): """Utility function for converting a django template variable to JSON in order to use it in script tags. Args obj: Any django template context variable Returns: JSON representation of the variable. """ return mark_safe(json.dumps(obj, cls=DjangoJSONEncoder)) @register.filter def sub(value, arg): """Django template filter for subtracting two numbers Args: value (int/float): the value to subtract from arg (int/float): the value to subtract to Returns: int/float: The subtraction result """ return value - arg @register.filter def mul(value, arg): """Django template filter for multiplying two numbers Args: value (int/float): the value to multiply from arg (int/float): the value to multiply with Returns: int/float: The multiplication result """ return value * arg @register.filter def key_value(dict, key): """Django template filter to get a value in a dictionary. Args: dict (dict): a dictionary key (str): the key to lookup value Returns: The requested value in the dictionary """ return dict[key] @register.filter def visit_type_savable(visit_type): """Django template filter to check if a save request can be created for a given visit type. Args: visit_type (str): the type of visit Returns: If the visit type is saveable or not """ return visit_type in get_savable_visit_types() @register.filter def split(value, arg): """Django template filter to split a string. Args: value (str): the string to split arg (str): the split separator Returns: list: the split string parts """ return value.split(arg) diff --git a/swh/web/common/throttling.py b/swh/web/common/throttling.py index 93d443c5..f56b0917 100644 --- a/swh/web/common/throttling.py +++ b/swh/web/common/throttling.py @@ -1,141 +1,147 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import ipaddress +from django.core.exceptions import ImproperlyConfigured from rest_framework.throttling import ScopedRateThrottle +import sentry_sdk from swh.web.config import get_config class SwhWebRateThrottle(ScopedRateThrottle): """Custom request rate limiter for DRF enabling to exempt specific networks specified in swh-web configuration. Requests are grouped into scopes. It enables to apply different requests rate limiting based on the scope name but also the input HTTP request types. To associate a scope to requests, one must add a 'throttle_scope' attribute when using a class based view, or call the 'throttle_scope' decorator when using a function based view. By default, requests do not have an associated scope and are not rate limited. Rate limiting can also be configured according to the type of the input HTTP requests for fine grained tuning. For instance, the following YAML configuration section sets a rate of: - 1 per minute for POST requests - 60 per minute for other request types for the 'swh_api' scope while exempting those coming from the 127.0.0.0/8 ip network. .. code-block:: yaml throttling: scopes: swh_api: limiter_rate: default: 60/m POST: 1/m exempted_networks: - 127.0.0.0/8 """ scope = None def __init__(self): super().__init__() self.exempted_networks = None def get_exempted_networks(self, scope_name): if not self.exempted_networks: scopes = get_config()['throttling']['scopes'] scope = scopes.get(scope_name) if scope: networks = scope.get('exempted_networks') if networks: self.exempted_networks = [ipaddress.ip_network(network) for network in networks] return self.exempted_networks def allow_request(self, request, view): # class based view case if not self.scope: + default_scope = getattr(view, self.scope_attr, None) - # check if there is a specific rate limiting associated - # to the request type - try: + request_allowed = None + if default_scope is not None: + # check if there is a specific rate limiting associated + # to the request type request_scope = default_scope + '_' + request.method.lower() setattr(view, self.scope_attr, request_scope) - request_allowed = \ - super(SwhWebRateThrottle, self).allow_request(request, view) # noqa - setattr(view, self.scope_attr, default_scope) - # use default rate limiting otherwise - except Exception: - setattr(view, self.scope_attr, default_scope) - request_allowed = \ - super(SwhWebRateThrottle, self).allow_request(request, view) # noqa + try: + request_allowed = super().allow_request(request, view) + # use default rate limiting otherwise + except ImproperlyConfigured as exc: + sentry_sdk.capture_exception(exc) + + setattr(view, self.scope_attr, default_scope) + if request_allowed is None: + request_allowed = super().allow_request(request, view) # function based view case else: default_scope = self.scope # check if there is a specific rate limiting associated # to the request type + self.scope = default_scope + '_' + request.method.lower() try: - self.scope = default_scope + '_' + request.method.lower() self.rate = self.get_rate() # use default rate limiting otherwise - except Exception: + except ImproperlyConfigured as exc: + sentry_sdk.capture_exception(exc) self.scope = default_scope self.rate = self.get_rate() self.num_requests, self.duration = self.parse_rate(self.rate) request_allowed = \ super(ScopedRateThrottle, self).allow_request(request, view) self.scope = default_scope exempted_networks = self.get_exempted_networks(default_scope) exempted_ip = False if exempted_networks: remote_address = ipaddress.ip_address(self.get_ident(request)) exempted_ip = any(remote_address in network for network in exempted_networks) request_allowed = exempted_ip or request_allowed # set throttling related data in the request metadata # in order for the ThrottlingHeadersMiddleware to # add X-RateLimit-* headers in the HTTP response if not exempted_ip and hasattr(self, 'history'): hit_count = len(self.history) request.META['RateLimit-Limit'] = self.num_requests request.META['RateLimit-Remaining'] = self.num_requests - hit_count request.META['RateLimit-Reset'] = int(self.now + self.wait()) return request_allowed def throttle_scope(scope): """Decorator that allows the throttle scope of a DRF function based view to be set:: @api_view(['GET', ]) @throttle_scope('scope') def view(request): ... """ def decorator(func): SwhScopeRateThrottle = type( 'CustomScopeRateThrottle', (SwhWebRateThrottle,), {'scope': scope} ) func.throttle_classes = (SwhScopeRateThrottle, ) return func return decorator diff --git a/swh/web/doc_config.py b/swh/web/doc_config.py index c8d252c5..65f5b170 100644 --- a/swh/web/doc_config.py +++ b/swh/web/doc_config.py @@ -1,74 +1,73 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +import importlib.util import os from sphinxcontrib import httpdomain from sphinx.ext import autodoc # guard to avoid ImportError when running tests through sbuild # as there is no Debian package built for swh-docs -try: +if importlib.util.find_spec('swh.docs'): from swh.docs.sphinx.conf import setup as orig_setup -except Exception: - pass class SimpleDocumenter(autodoc.FunctionDocumenter): """ Custom autodoc directive to display a docstring unindented and without function signature header. """ objtype = "simple" # ensure the priority is lesser than the base FunctionDocumenter # to avoid side effects with autodoc processing priority = -1 # do not indent the content content_indent = "" # do not add a header to the docstring def add_directive_header(self, sig): pass _swh_web_base_url = 'https://archive.softwareheritage.org' _swh_web_api_endpoint = 'api' _swh_web_api_version = 1 _swh_web_api_url = '%s/%s/%s/' % (_swh_web_base_url, _swh_web_api_endpoint, _swh_web_api_version) _swh_web_browse_endpoint = 'browse' _swh_web_browse_url = '%s/%s/' % (_swh_web_base_url, _swh_web_browse_endpoint) def setup(app): orig_setup(app) app.add_autodocumenter(SimpleDocumenter) # set an environment variable indicating we are currently # building the swh-web documentation os.environ['SWH_WEB_DOC_BUILD'] = '1' def customize_sphinx_conf(sphinx_conf): """ Utility function used to customize the sphinx doc build for swh-web globally (when building doc from swh-docs) or locally (when building doc from swh-web). Args: sphinx_conf (module): a reference to the sphinx conf.py module used to build the doc. """ # fix for sphinxcontrib.httpdomain 1.3 if 'Link' not in httpdomain.HEADER_REFS: httpdomain.HEADER_REFS['Link'] = httpdomain.IETFRef(5988, '5') sphinx_conf.extlinks['swh_web'] = (_swh_web_base_url + '/%s', None) sphinx_conf.extlinks['swh_web_api'] = (_swh_web_api_url + '%s', None) sphinx_conf.extlinks['swh_web_browse'] = (_swh_web_browse_url + '%s', None) sphinx_conf.setup = setup diff --git a/swh/web/gunicorn_config.py b/swh/web/gunicorn_config.py new file mode 100644 index 00000000..9c9a5b5c --- /dev/null +++ b/swh/web/gunicorn_config.py @@ -0,0 +1,14 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from sentry_sdk.integrations.django import DjangoIntegration + +from swh.core.api.gunicorn_config import * # noqa +from swh.core.api.gunicorn_config import post_fork as _post_fork + + +def post_fork(server, worker): # type: ignore + _post_fork(server, worker, + flask=False, sentry_integrations=[DjangoIntegration()]) diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py index 602a74aa..d2e6112a 100644 --- a/swh/web/misc/urls.py +++ b/swh/web/misc/urls.py @@ -1,79 +1,80 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import requests +import sentry_sdk from django.conf.urls import url, include from django.contrib.staticfiles import finders from django.http import HttpResponse from django.shortcuts import render from swh.web.common import service from swh.web.config import get_config def _jslicenses(request): jslicenses_file = finders.find('jssources/jslicenses.json') jslicenses_data = json.load(open(jslicenses_file)) jslicenses_data = sorted(jslicenses_data.items(), key=lambda item: item[0].split('/')[-1]) return render(request, "misc/jslicenses.html", {'jslicenses_data': jslicenses_data}) def _stat_counters(request): stat = service.stat_counters() url = get_config()['history_counters_url'] stat_counters_history = 'null' if url: try: response = requests.get(url, timeout=5) stat_counters_history = response.text - except Exception: - pass + except Exception as exc: + sentry_sdk.capture_exception(exc) json_data = '{"stat_counters": %s, "stat_counters_history": %s}' % ( json.dumps(stat), stat_counters_history) return HttpResponse(json_data, content_type='application/json') urlpatterns = [ url(r'^', include('swh.web.misc.coverage')), url(r'^jslicenses/$', _jslicenses, name='jslicenses'), url(r'^', include('swh.web.misc.origin_save')), url(r'^stat_counters', _stat_counters, name='stat-counters'), url(r'^', include('swh.web.misc.badges')), ] # when running end to end tests trough cypress, declare some extra # endpoints to provide input data for some of those tests if get_config()['e2e_tests_mode']: from swh.web.tests.data import ( get_content_code_data_by_ext, get_content_other_data_by_ext, get_content_code_data_all_exts, get_content_code_data_by_filename, get_content_code_data_all_filenames, ) # noqa urlpatterns.append( url(r'^tests/data/content/code/extension/(?P.+)/$', get_content_code_data_by_ext, name='tests-content-code-extension')) urlpatterns.append( url(r'^tests/data/content/other/extension/(?P.+)/$', get_content_other_data_by_ext, name='tests-content-other-extension')) urlpatterns.append(url(r'^tests/data/content/code/extensions/$', get_content_code_data_all_exts, name='tests-content-code-extensions')) urlpatterns.append( url(r'^tests/data/content/code/filename/(?P.+)/$', get_content_code_data_by_filename, name='tests-content-code-filename')) urlpatterns.append(url(r'^tests/data/content/code/filenames/$', get_content_code_data_all_filenames, name='tests-content-code-filenames')) diff --git a/swh/web/tests/test_gunicorn_config.py b/swh/web/tests/test_gunicorn_config.py new file mode 100644 index 00000000..89b86f66 --- /dev/null +++ b/swh/web/tests/test_gunicorn_config.py @@ -0,0 +1,51 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import os +from unittest.mock import patch + +import swh.web.gunicorn_config as gunicorn_config + + +def test_post_fork_default(): + with patch('sentry_sdk.init') as sentry_sdk_init: + gunicorn_config.post_fork(None, None) + + sentry_sdk_init.assert_not_called() + + +def test_post_fork_with_dsn_env(): + django_integration = object() # unique object to check for equality + with patch('swh.web.gunicorn_config.DjangoIntegration', + new=lambda: django_integration): + with patch('sentry_sdk.init') as sentry_sdk_init: + with patch.dict(os.environ, {'SWH_SENTRY_DSN': 'test_dsn'}): + gunicorn_config.post_fork(None, None) + + sentry_sdk_init.assert_called_once_with( + dsn='test_dsn', + integrations=[django_integration], + debug=False, + ) + + +def test_post_fork_debug(): + django_integration = object() # unique object to check for equality + with patch('swh.web.gunicorn_config.DjangoIntegration', + new=lambda: django_integration): + with patch('sentry_sdk.init') as sentry_sdk_init: + with patch.dict(os.environ, {'SWH_SENTRY_DSN': 'test_dsn', + 'SWH_SENTRY_DEBUG': '1'}): + gunicorn_config.post_fork(None, None) + + sentry_sdk_init.assert_called_once_with( + dsn='test_dsn', + integrations=[django_integration], + debug=True, + )