diff --git a/cypress/integration/layout.spec.js b/cypress/integration/layout.spec.js index 5bdf6d74..a1e171e9 100644 --- a/cypress/integration/layout.spec.js +++ b/cypress/integration/layout.spec.js @@ -1,144 +1,227 @@ /** - * Copyright (C) 2019 The Software Heritage developers + * Copyright (C) 2019-2020 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ const url = '/browse/help/'; +const statusUrl = 'https://status.softwareheritage.org'; describe('Test top-bar', function() { beforeEach(function() { cy.clearLocalStorage(); cy.visit(url); }); it('should should contain all navigation links', function() { cy.get('.swh-top-bar a') .should('have.length.of.at.least', 4) .and('be.visible') .and('have.attr', 'href'); }); it('should show donate button on lg screen', function() { cy.get('.swh-donate-link') .should('be.visible'); }); it('should hide donate button on sm screen', function() { cy.viewport(600, 800); cy.get('.swh-donate-link') .should('not.be.visible'); }); it('should hide full width switch on small screens', function() { cy.viewport(360, 740); cy.get('#swh-full-width-switch-container') .should('not.be.visible'); cy.viewport(600, 800); cy.get('#swh-full-width-switch-container') .should('not.be.visible'); cy.viewport(800, 600); cy.get('#swh-full-width-switch-container') .should('not.be.visible'); }); it('should show full width switch on large screens', function() { cy.viewport(1024, 768); cy.get('#swh-full-width-switch-container') .should('be.visible'); cy.viewport(1920, 1080); cy.get('#swh-full-width-switch-container') .should('be.visible'); }); it('should change container width when toggling Full width switch', function() { cy.get('#swh-web-content') .should('have.class', 'container') .should('not.have.class', 'container-fluid'); cy.should(() => { expect(JSON.parse(localStorage.getItem('swh-web-full-width'))).to.be.null; }); cy.get('#swh-full-width-switch') .click({force: true}); cy.get('#swh-web-content') .should('not.have.class', 'container') .should('have.class', 'container-fluid'); cy.should(() => { expect(JSON.parse(localStorage.getItem('swh-web-full-width'))).to.be.true; }); cy.get('#swh-full-width-switch') .click({force: true}); cy.get('#swh-web-content') .should('have.class', 'container') .should('not.have.class', 'container-fluid'); cy.should(() => { expect(JSON.parse(localStorage.getItem('swh-web-full-width'))).to.be.false; }); }); it('should restore container width when loading page again', function() { cy.visit(url) .get('#swh-web-content') .should('have.class', 'container') .should('not.have.class', 'container-fluid'); cy.get('#swh-full-width-switch') .click({force: true}); cy.visit(url) .get('#swh-web-content') .should('not.have.class', 'container') .should('have.class', 'container-fluid'); cy.get('#swh-full-width-switch') .click({force: true}); cy.visit(url) .get('#swh-web-content') .should('have.class', 'container') .should('not.have.class', 'container-fluid'); }); + + function genStatusResponse(status, statusCode) { + return { + 'result': { + 'status': [ + { + 'id': '5f7c4c567f50b304c1e7bd5f', + 'name': 'Save Code Now', + 'updated': '2020-11-30T13:51:21.151Z', + 'status': 'Operational', + 'status_code': 100 + }, + { + 'id': '5f7c4c6f8338bc04b7f476fe', + 'name': 'Source Code Crawlers', + 'updated': '2020-11-30T13:51:21.151Z', + 'status': status, + 'status_code': statusCode + } + ] + } + }; + } + + it('should display swh status widget when data are available', function() { + const statusTestData = [ + { + status: 'Operational', + statusCode: 100, + color: 'green' + }, + { + status: 'Scheduled Maintenance', + statusCode: 200, + color: 'blue' + }, + { + status: 'Degraded Performance', + statusCode: 300, + color: 'yellow' + }, + { + status: 'Partial Service Disruption', + statusCode: 400, + color: 'yellow' + }, + { + status: 'Service Disruption', + statusCode: 500, + color: 'red' + }, + { + status: 'Security Event', + statusCode: 600, + color: 'red' + } + ]; + + for (let std of statusTestData) { + cy.server(); + cy.route({ + url: `${statusUrl}/**`, + response: genStatusResponse(std.status, std.statusCode) + }).as('getSwhStatusData'); + cy.visit(url); + cy.wait('@getSwhStatusData'); + cy.get('.swh-current-status-indicator').should('have.class', std.color); + cy.get('#swh-current-status-description').should('have.text', std.status); + } + }); + + it('should not display swh status widget when data are not available', function() { + cy.server(); + cy.route({ + url: `${statusUrl}/**`, + response: {} + }).as('getSwhStatusData'); + cy.visit(url); + cy.wait('@getSwhStatusData'); + cy.get('.swh-current-status').should('not.be.visible'); + }); + }); describe('Test navbar', function() { it('should redirect to search page when submitting search form in navbar', function() { const keyword = 'python'; cy.get('#swh-origins-search-top-input') .type(keyword); cy.get('.swh-search-navbar') .submit(); cy.url() .should('include', `${this.Urls.browse_search()}?q=${keyword}`); }); }); describe('Test footer', function() { beforeEach(function() { cy.visit(url); }); it('should be visible', function() { cy.get('footer') .should('be.visible'); }); it('should have correct copyright years', function() { const currentYear = new Date().getFullYear(); const copyrightText = '(C) 2015–' + currentYear.toString(); cy.get('footer') .should('contain', copyrightText); }); it('should contain link to Web API', function() { cy.get('footer') .get(`a[href="${this.Urls.api_1_homepage()}"]`) .should('contain', 'Web API'); }); }); diff --git a/swh/web/assets/src/bundles/webapp/index.js b/swh/web/assets/src/bundles/webapp/index.js index 9871b775..90663d72 100644 --- a/swh/web/assets/src/bundles/webapp/index.js +++ b/swh/web/assets/src/bundles/webapp/index.js @@ -1,26 +1,27 @@ /** * Copyright (C) 2018-2020 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // webapp entrypoint bundle centralizing global custom stylesheets // and utility js modules used in all swh-web applications // global swh-web custom stylesheets import './webapp.css'; import './breadcrumbs.css'; export * from './webapp-utils'; // utility js modules export * from './code-highlighting'; export * from './readme-rendering'; export * from './pdf-rendering'; export * from './notebook-rendering'; export * from './xss-filtering'; export * from './history-counters'; export * from './badges'; export * from './sentry'; export * from './math-typesetting'; +export * from './status-widget'; diff --git a/swh/web/assets/src/bundles/webapp/status-widget.css b/swh/web/assets/src/bundles/webapp/status-widget.css new file mode 100644 index 00000000..f6623b73 --- /dev/null +++ b/swh/web/assets/src/bundles/webapp/status-widget.css @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2020 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +.swh-current-status-indicator { + width: 12px; + height: 12px; + margin: 0 0 0 5px; + display: inline-block; + border-radius: 6px; +} + +.swh-current-status-indicator.small { + width: 8px; + height: 8px; + margin: 0 0 0 5px; + display: inline-block; + border-radius: 4px; +} + +.swh-current-status-indicator.green { + background: #27ae60; +} + +.swh-current-status-indicator.yellow { + background: #ffa837; +} + +.swh-current-status-indicator.red { + background: #c44031; +} + +.swh-current-status-indicator.blue { + background: #00aaf0; +} diff --git a/swh/web/assets/src/bundles/webapp/status-widget.js b/swh/web/assets/src/bundles/webapp/status-widget.js new file mode 100644 index 00000000..5f28d9de --- /dev/null +++ b/swh/web/assets/src/bundles/webapp/status-widget.js @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2020 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +import './status-widget.css'; + +const statusCodeColor = { + '100': 'green', // Operational + '200': 'blue', // Scheduled Maintenance + '300': 'yellow', // Degraded Performance + '400': 'yellow', // Partial Service Disruption + '500': 'red', // Service Disruption + '600': 'red' // Security Event +}; + +export function initStatusWidget(statusDataURL) { + $('.swh-current-status-indicator').ready(() => { + let maxStatusCode = ''; + let maxStatusDescription = ''; + let sc = ''; + let sd = ''; + fetch(statusDataURL) + .then(resp => resp.json()) + .then(data => { + for (let s of data.result.status) { + sc = s.status_code; + sd = s.status; + if (maxStatusCode < sc) { + maxStatusCode = sc; + maxStatusDescription = sd; + } + } + if (maxStatusCode === '') { + $('.swh-current-status').remove(); + return; + } + $('.swh-current-status-indicator').removeClass('green'); + $('.swh-current-status-indicator').addClass(statusCodeColor[maxStatusCode]); + $('#swh-current-status-description').text(maxStatusDescription); + }) + .catch(e => { + console.log(e); + $('.swh-current-status').remove(); + }); + + }); +} diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py index 25f5260e..464d553f 100644 --- a/swh/web/common/utils.py +++ b/swh/web/common/utils.py @@ -1,355 +1,356 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime, timezone import os import re from typing import Any, Dict, Optional from bs4 import BeautifulSoup from docutils.core import publish_parts import docutils.parsers.rst import docutils.utils from docutils.writers.html5_polyglot import HTMLTranslator, Writer from iso8601 import ParseError, parse_date from prometheus_client.registry import CollectorRegistry from django.http import HttpRequest, QueryDict from django.urls import reverse as django_reverse from rest_framework.authentication import SessionAuthentication from swh.web.common.exc import BadInputExc from swh.web.common.typing import QueryParameters from swh.web.config import get_config SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) swh_object_icons = { "alias": "mdi mdi-star", "branch": "mdi mdi-source-branch", "branches": "mdi mdi-source-branch", "content": "mdi mdi-file-document", "directory": "mdi mdi-folder", "origin": "mdi mdi-source-repository", "person": "mdi mdi-account", "revisions history": "mdi mdi-history", "release": "mdi mdi-tag", "releases": "mdi mdi-tag", "revision": "mdi mdi-rotate-90 mdi-source-commit", "snapshot": "mdi mdi-camera", "visits": "mdi mdi-calendar-month", } def reverse( viewname: str, url_args: Optional[Dict[str, Any]] = None, query_params: Optional[QueryParameters] = None, current_app: Optional[str] = None, urlconf: Optional[str] = None, request: Optional[HttpRequest] = None, ) -> str: """An override of django reverse function supporting query parameters. Args: viewname: the name of the django view from which to compute a url url_args: dictionary of url arguments indexed by their names query_params: dictionary of query parameters to append to the reversed url current_app: the name of the django app tighten to the view urlconf: url configuration module request: build an absolute URI if provided Returns: str: the url of the requested view with processed arguments and query parameters """ if url_args: url_args = {k: v for k, v in url_args.items() if v is not None} url = django_reverse( viewname, urlconf=urlconf, kwargs=url_args, current_app=current_app ) if query_params: query_params = {k: v for k, v in query_params.items() if v is not None} if query_params and len(query_params) > 0: query_dict = QueryDict("", mutable=True) for k in sorted(query_params.keys()): query_dict[k] = query_params[k] url += "?" + query_dict.urlencode(safe="/;:") if request is not None: url = request.build_absolute_uri(url) return url def datetime_to_utc(date): """Returns datetime in UTC without timezone info Args: date (datetime.datetime): input datetime with timezone info Returns: datetime.datetime: datetime in UTC without timezone info """ if date.tzinfo and date.tzinfo != timezone.utc: return date.astimezone(tz=timezone.utc) else: return date def parse_iso8601_date_to_utc(iso_date: str) -> datetime: """Given an ISO 8601 datetime string, parse the result as UTC datetime. Returns: a timezone-aware datetime representing the parsed date Raises: swh.web.common.exc.BadInputExc: provided date does not respect ISO 8601 format Samples: - 2016-01-12 - 2016-01-12T09:19:12+0100 - 2007-01-14T20:34:22Z """ try: date = parse_date(iso_date) return datetime_to_utc(date) except ParseError as e: raise BadInputExc(e) def shorten_path(path): """Shorten the given path: for each hash present, only return the first 8 characters followed by an ellipsis""" sha256_re = r"([0-9a-f]{8})[0-9a-z]{56}" sha1_re = r"([0-9a-f]{8})[0-9a-f]{32}" ret = re.sub(sha256_re, r"\1...", path) return re.sub(sha1_re, r"\1...", ret) def format_utc_iso_date(iso_date, fmt="%d %B %Y, %H:%M UTC"): """Turns a string representation of an ISO 8601 datetime string to UTC and format it into a more human readable one. For instance, from the following input string: '2017-05-04T13:27:13+02:00' the following one is returned: '04 May 2017, 11:27 UTC'. Custom format string may also be provided as parameter Args: iso_date (str): a string representation of an ISO 8601 date fmt (str): optional date formatting string Returns: str: a formatted string representation of the input iso date """ if not iso_date: return iso_date date = parse_iso8601_date_to_utc(iso_date) return date.strftime(fmt) def gen_path_info(path): """Function to generate path data navigation for use with a breadcrumb in the swh web ui. For instance, from a path /folder1/folder2/folder3, it returns the following list:: [{'name': 'folder1', 'path': 'folder1'}, {'name': 'folder2', 'path': 'folder1/folder2'}, {'name': 'folder3', 'path': 'folder1/folder2/folder3'}] Args: path: a filesystem path Returns: list: a list of path data for navigation as illustrated above. """ path_info = [] if path: sub_paths = path.strip("/").split("/") path_from_root = "" for p in sub_paths: path_from_root += "/" + p path_info.append({"name": p, "path": path_from_root.strip("/")}) return path_info def parse_rst(text, report_level=2): """ Parse a reStructuredText string with docutils. Args: text (str): string with reStructuredText markups in it report_level (int): level of docutils report messages to print (1 info 2 warning 3 error 4 severe 5 none) Returns: docutils.nodes.document: a parsed docutils document """ parser = docutils.parsers.rst.Parser() components = (docutils.parsers.rst.Parser,) settings = docutils.frontend.OptionParser( components=components ).get_default_values() settings.report_level = report_level document = docutils.utils.new_document("rst-doc", settings=settings) parser.parse(text, document) return document def get_client_ip(request): """ Return the client IP address from an incoming HTTP request. Args: request (django.http.HttpRequest): the incoming HTTP request Returns: str: The client IP address """ x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") if x_forwarded_for: ip = x_forwarded_for.split(",")[0] else: ip = request.META.get("REMOTE_ADDR") return ip browsers_supported_image_mimes = set( [ "image/gif", "image/png", "image/jpeg", "image/bmp", "image/webp", "image/svg", "image/svg+xml", ] ) def context_processor(request): """ Django context processor used to inject variables in all swh-web templates. """ config = get_config() if ( hasattr(request, "user") and request.user.is_authenticated and not hasattr(request.user, "backend") ): # To avoid django.template.base.VariableDoesNotExist errors # when rendering templates when standard Django user is logged in. request.user.backend = "django.contrib.auth.backends.ModelBackend" return { "swh_object_icons": swh_object_icons, "available_languages": None, "swh_client_config": config["client_config"], "oidc_enabled": bool(config["keycloak"]["server_url"]), "browsers_supported_image_mimes": browsers_supported_image_mimes, "keycloak": config["keycloak"], "site_base_url": request.build_absolute_uri("/"), "DJANGO_SETTINGS_MODULE": os.environ["DJANGO_SETTINGS_MODULE"], + "status": config["status"], } class EnforceCSRFAuthentication(SessionAuthentication): """ Helper class to enforce CSRF validation on a DRF view when a user is not authenticated. """ def authenticate(self, request): user = getattr(request._request, "user", None) self.enforce_csrf(request) return (user, None) def resolve_branch_alias( snapshot: Dict[str, Any], branch: Optional[Dict[str, Any]] ) -> Optional[Dict[str, Any]]: """ Resolve branch alias in snapshot content. Args: snapshot: a full snapshot content branch: a branch alias contained in the snapshot Returns: The real snapshot branch that got aliased. """ while branch and branch["target_type"] == "alias": if branch["target"] in snapshot["branches"]: branch = snapshot["branches"][branch["target"]] else: from swh.web.common import archive snp = archive.lookup_snapshot( snapshot["id"], branches_from=branch["target"], branches_count=1 ) if snp and branch["target"] in snp["branches"]: branch = snp["branches"][branch["target"]] else: branch = None return branch class _NoHeaderHTMLTranslator(HTMLTranslator): """ Docutils translator subclass to customize the generation of HTML from reST-formatted docstrings """ def __init__(self, document): super().__init__(document) self.body_prefix = [] self.body_suffix = [] _HTML_WRITER = Writer() _HTML_WRITER.translator_class = _NoHeaderHTMLTranslator def rst_to_html(rst: str) -> str: """ Convert reStructuredText document into HTML. Args: rst: A string containing a reStructuredText document Returns: Body content of the produced HTML conversion. """ settings = { "initial_header_level": 2, } pp = publish_parts(rst, writer=_HTML_WRITER, settings_overrides=settings) return f'
{pp["html_body"]}
' def prettify_html(html: str) -> str: """ Prettify an HTML document. Args: html: Input HTML document Returns: The prettified HTML document """ return BeautifulSoup(html, "lxml").prettify() diff --git a/swh/web/config.py b/swh/web/config.py index a044d08f..f03fa55b 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,170 +1,177 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from typing import Any, Dict from swh.core import config from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.search import get_search from swh.storage import get_storage from swh.vault import get_vault from swh.web import settings SWH_WEB_INTERNAL_SERVER_NAME = "archive.internal.softwareheritage.org" SETTINGS_DIR = os.path.dirname(settings.__file__) DEFAULT_CONFIG = { "allowed_hosts": ("list", []), "search": ( "dict", {"cls": "remote", "url": "http://127.0.0.1:5010/", "timeout": 10,}, ), "storage": ( "dict", {"cls": "remote", "url": "http://127.0.0.1:5002/", "timeout": 10,}, ), "indexer_storage": ( "dict", {"cls": "remote", "url": "http://127.0.0.1:5007/", "timeout": 1,}, ), "log_dir": ("string", "/tmp/swh/log"), "debug": ("bool", False), "serve_assets": ("bool", False), "host": ("string", "127.0.0.1"), "port": ("int", 5004), "secret_key": ("string", "development key"), # do not display code highlighting for content > 1MB "content_display_max_size": ("int", 5 * 1024 * 1024), "snapshot_content_max_size": ("int", 1000), "throttling": ( "dict", { "cache_uri": None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None "scopes": { "swh_api": { "limiter_rate": {"default": "120/h"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_search": { "limiter_rate": {"default": "10/m"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_vault_cooking": { "limiter_rate": {"default": "120/h", "GET": "60/m"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_save_origin": { "limiter_rate": {"default": "120/h", "POST": "10/h"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_visit_latest": { "limiter_rate": {"default": "700/m"}, "exempted_networks": ["127.0.0.0/8"], }, }, }, ), "vault": ("dict", {"cls": "remote", "args": {"url": "http://127.0.0.1:5005/",}}), "scheduler": ("dict", {"cls": "remote", "url": "http://127.0.0.1:5008/"}), "development_db": ("string", os.path.join(SETTINGS_DIR, "db.sqlite3")), "test_db": ("string", os.path.join(SETTINGS_DIR, "testdb.sqlite3")), "production_db": ("string", "/var/lib/swh/web.sqlite3"), "deposit": ( "dict", { "private_api_url": "https://deposit.softwareheritage.org/1/private/", "private_api_user": "swhworker", "private_api_password": "", }, ), "coverage_count_origins": ("bool", False), "e2e_tests_mode": ("bool", False), "es_workers_index_url": ("string", ""), "history_counters_url": ( "string", "https://stats.export.softwareheritage.org/history_counters.json", ), "client_config": ("dict", {}), "keycloak": ("dict", {"server_url": "", "realm_name": ""}), "graph": ( "dict", {"server_url": "http://graph.internal.softwareheritage.org:5009/graph/"}, ), + "status": ( + "dict", + { + "server_url": "https://status.softwareheritage.org/", + "json_path": "1.0/status/578e5eddcdc0cc7951000520", + }, + ), } swhweb_config = {} # type: Dict[str, Any] def get_config(config_file="web/web"): """Read the configuration file `config_file`. If an environment variable SWH_CONFIG_FILENAME is defined, this takes precedence over the config_file parameter. In any case, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration. """ if not swhweb_config: config_filename = os.environ.get("SWH_CONFIG_FILENAME") if config_filename: config_file = config_filename cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) config.prepare_folders(swhweb_config, "log_dir") if swhweb_config.get("search"): swhweb_config["search"] = get_search(**swhweb_config["search"]) else: swhweb_config["search"] = None swhweb_config["storage"] = get_storage(**swhweb_config["storage"]) swhweb_config["vault"] = get_vault(**swhweb_config["vault"]) swhweb_config["indexer_storage"] = get_indexer_storage( **swhweb_config["indexer_storage"] ) swhweb_config["scheduler"] = get_scheduler(**swhweb_config["scheduler"]) return swhweb_config def search(): """Return the current application's search. """ return get_config()["search"] def storage(): """Return the current application's storage. """ return get_config()["storage"] def vault(): """Return the current application's vault. """ return get_config()["vault"] def indexer_storage(): """Return the current application's indexer storage. """ return get_config()["indexer_storage"] def scheduler(): """Return the current application's scheduler. """ return get_config()["scheduler"] diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html index 51bc6ecc..4dd0b1f3 100644 --- a/swh/web/templates/layout.html +++ b/swh/web/templates/layout.html @@ -1,258 +1,266 @@ {% comment %} Copyright (C) 2015-2020 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load js_reverse %} {% load static %} {% load render_bundle from webpack_loader %} {% load swh_templatetags %} {% block title %}{% endblock %} {% render_bundle 'vendors' %} {% render_bundle 'webapp' %} {% block header %}{% endblock %} {% if "production" in DJANGO_SETTINGS_MODULE %} {% endif %}
{% block content %}{% endblock %}
{% include "includes/global-modals.html" %}
back to top