diff --git a/cypress/integration/home.spec.js b/cypress/integration/home.spec.js index d34244a0..5ec2c506 100644 --- a/cypress/integration/home.spec.js +++ b/cypress/integration/home.spec.js @@ -1,25 +1,33 @@ /** * Copyright (C) 2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ const $ = Cypress.$; const url = '/'; describe('Home Page Tests', function() { it('should display positive stats for each category', function() { + + cy.server(); + + cy.route({ + method: 'GET', + url: this.Urls.stat_counters() + }).as('getStatCounters'); + cy.visit(url) - .wait(3000) // wait counters request result + .wait('@getStatCounters') .get('.swh-counter') .then((counters) => { for (let counter of counters) { let innerText = $(counter).text(); const value = parseInt(innerText.replace(/,/g, '')); assert.isAbove(value, 0); } }); }); }); diff --git a/swh/web/assets/config/.eslintrc b/swh/web/assets/config/.eslintrc index d92e6663..4f66e6fb 100644 --- a/swh/web/assets/config/.eslintrc +++ b/swh/web/assets/config/.eslintrc @@ -1,308 +1,308 @@ { "parser": "babel-eslint", "parserOptions": { "ecmaVersion": 2017, "ecmaFeatures": { "experimentalObjectRestSpread": true, "jsx": true }, "sourceType": "module", "allowImportExportEverywhere": true }, "env": { "es6": true, "node": true, "cypress/globals": true }, "plugins": [ "import", "node", "promise", "standard", "cypress", "chai-friendly" ], "globals": { "document": false, "navigator": false, "window": false, "$": false, "jQuery": false, "history": false, "localStorage": false, "sessionStorage": false, "Urls": false, "hljs": false, "Waypoint": false, "swh": false, "fetch": false, "__STATIC__": false, "Image": false, "Cookies": false, "nb": false, "MathJax": false }, "rules": { "accessor-pairs": "error", "arrow-spacing": ["error", { "before": true, "after": true }], "block-spacing": ["error", "always"], "brace-style": ["error", "1tbs", { "allowSingleLine": true }], "camelcase": ["error", { "properties": "never" }], "comma-dangle": ["error", { "arrays": "never", "objects": "never", "imports": "never", "exports": "never", "functions": "never" }], "comma-spacing": ["error", { "before": false, "after": true }], "comma-style": ["error", "last"], "constructor-super": "error", "curly": ["error", "multi-line"], "dot-location": ["error", "property"], "eol-last": "error", "eqeqeq": ["error", "always", { "null": "ignore" }], "func-call-spacing": ["error", "never"], "generator-star-spacing": ["error", { "before": true, "after": true }], "handle-callback-err": ["error", "^(err|error)$"], "indent": ["error", 2, { "SwitchCase": 1, "VariableDeclarator": 1, "outerIIFEBody": 1, - "MemberExpression": 1, + "MemberExpression": "off", "FunctionDeclaration": { "parameters": "first", "body": 1 }, "FunctionExpression": { "parameters": "first", "body": 1 }, "CallExpression": { "arguments": "first" }, "ArrayExpression": "first", "ObjectExpression": "first", "ImportDeclaration": "first", "flatTernaryExpressions": false, "ignoreComments": false }], "key-spacing": ["error", { "beforeColon": false, "afterColon": true }], "keyword-spacing": ["error", { "before": true, "after": true }], "new-cap": ["error", { "newIsCap": true, "capIsNew": false }], "new-parens": "error", "no-array-constructor": "error", "no-caller": "error", "no-class-assign": "error", "no-compare-neg-zero": "error", "no-cond-assign": "error", "no-const-assign": "error", "no-constant-condition": ["error", { "checkLoops": false }], "no-control-regex": "error", "no-debugger": "error", "no-delete-var": "error", "no-dupe-args": "error", "no-dupe-class-members": "error", "no-dupe-keys": "error", "no-duplicate-case": "error", "no-empty-character-class": "error", "no-empty-pattern": "error", "no-eval": "error", "no-ex-assign": "error", "no-extend-native": "error", "no-extra-bind": "error", "no-extra-boolean-cast": "error", "no-extra-parens": ["error", "functions"], "no-fallthrough": "error", "no-floating-decimal": "error", "no-func-assign": "error", "no-global-assign": "error", "no-implied-eval": "error", "no-inner-declarations": ["error", "functions"], "no-invalid-regexp": "error", "no-irregular-whitespace": "error", "no-iterator": "error", "no-label-var": "error", "no-labels": ["error", { "allowLoop": false, "allowSwitch": false }], "no-lone-blocks": "error", "no-mixed-operators": ["error", { "groups": [ ["==", "!=", "===", "!==", ">", ">=", "<", "<="], ["&&", "||"], ["in", "instanceof"] ], "allowSamePrecedence": true }], "no-mixed-spaces-and-tabs": "error", "no-multi-spaces": "error", "no-multi-str": "error", "no-multiple-empty-lines": ["error", { "max": 1, "maxEOF": 0 }], "no-negated-in-lhs": "error", "no-new": 0, "no-new-func": "error", "no-new-object": "error", "no-new-require": "error", "no-new-symbol": "error", "no-new-wrappers": "error", "no-obj-calls": "error", "no-octal": "error", "no-octal-escape": "error", "no-path-concat": "error", "no-proto": "error", "no-redeclare": "error", "no-regex-spaces": "error", "no-return-assign": ["error", "except-parens"], "no-return-await": "error", "no-self-assign": "error", "no-self-compare": "error", "no-sequences": "error", "no-shadow-restricted-names": "error", "no-sparse-arrays": "error", "no-tabs": "error", "no-template-curly-in-string": "error", "no-this-before-super": "error", "no-throw-literal": "error", "no-trailing-spaces": "error", "no-undef": "error", "no-undef-init": "error", "no-unexpected-multiline": "error", "no-unmodified-loop-condition": "error", "no-unneeded-ternary": ["error", { "defaultAssignment": false }], "no-unreachable": "error", "no-unsafe-finally": "error", "no-unsafe-negation": "error", "no-unused-expressions": 0, "no-unused-vars": ["error", { "vars": "all", "args": "none", "ignoreRestSiblings": true }], "no-use-before-define": ["error", { "functions": false, "classes": false, "variables": false }], "no-useless-call": "error", "no-useless-computed-key": "error", "no-useless-constructor": "error", "no-useless-escape": "error", "no-useless-rename": "error", "no-useless-return": "error", "no-whitespace-before-property": "error", "no-with": "error", "object-property-newline": ["error", { "allowMultiplePropertiesPerLine": true }], "one-var": ["error", { "initialized": "never" }], "operator-linebreak": ["error", "after", { "overrides": { "?": "before", ":": "before" } }], "padded-blocks": ["off", { "blocks": "never", "switches": "never", "classes": "never" }], "prefer-promise-reject-errors": "error", "quotes": ["error", "single", { "avoidEscape": true, "allowTemplateLiterals": true }], "rest-spread-spacing": ["error", "never"], "semi": ["error", "always"], "semi-spacing": ["error", { "before": false, "after": true }], "space-before-blocks": ["error", "always"], "space-before-function-paren": ["error", "never"], "space-in-parens": ["error", "never"], "space-infix-ops": "error", "space-unary-ops": ["error", { "words": true, "nonwords": false }], "spaced-comment": ["error", "always", { "line": { "markers": ["*package", "!", "/", ",", "="] }, "block": { "balanced": true, "markers": ["*package", "!", ",", ":", "::", "flow-include"], "exceptions": ["*"] } }], "symbol-description": "error", "template-curly-spacing": ["error", "never"], "template-tag-spacing": ["error", "never"], "unicode-bom": ["error", "never"], "use-isnan": "error", "valid-typeof": ["error", { "requireStringLiterals": true }], "wrap-iife": ["error", "any", { "functionPrototypeMethods": true }], "yield-star-spacing": ["error", "both"], "yoda": ["error", "never"], "import/export": "off", "import/first": "error", "import/no-duplicates": "error", "import/no-webpack-loader-syntax": "off", "node/no-deprecated-api": "error", "node/process-exit-as-throw": "error", "promise/param-names": "error", "standard/array-bracket-even-spacing": ["error", "either"], "standard/computed-property-even-spacing": ["error", "even"], "standard/no-callback-literal": "error", "standard/object-curly-even-spacing": ["error", "either"], "chai-friendly/no-unused-expressions": 2 } } \ No newline at end of file diff --git a/swh/web/assets/src/bundles/origin/visits-reporting.css b/swh/web/assets/src/bundles/origin/visits-reporting.css index 51cbe801..c4a85539 100644 --- a/swh/web/assets/src/bundles/origin/visits-reporting.css +++ b/swh/web/assets/src/bundles/origin/visits-reporting.css @@ -1,125 +1,94 @@ /** * Copyright (C) 2018 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ .swh-visit-full { color: green; position: relative; } .swh-visit-full::before { content: '\f00c'; font-family: 'FontAwesome'; left: -20px; position: absolute; top: -2px; } .swh-visit-partial { color: #edc344; position: relative; } .swh-visit-partial::before { content: '\f071'; font-family: 'FontAwesome'; left: -20px; position: absolute; top: -2px; } .swh-visit-failed { color: #f00; position: relative; } .swh-visit-failed::before { content: '\f06a'; font-family: 'FontAwesome'; left: -20px; position: absolute; top: -2px; } .swh-visit-ongoing { color: #00f; position: relative; } .swh-visit-ongoing::before { content: '\f021'; font-family: 'FontAwesome'; left: -20px; position: absolute; top: -2px; } #swh-visits-calendar.calendar { min-height: 700px; overflow: visible; } #swh-visits-calendar.calendar table td { width: 28px; height: 28px; padding: 0; } -.d3-wrapper { - position: relative; - height: 0; - width: 100%; - padding: 0; - - /* padding-bottom will be overwritten by JavaScript later */ - padding-bottom: 100%; -} - -.d3-wrapper > svg { - position: absolute; - height: 100%; - width: 100%; - left: 0; - top: 0; -} - svg .grid line { stroke: lightgrey; stroke-opacity: 0.7; shape-rendering: crispEdges; } svg .grid path { stroke-width: 0; } -div.d3-tooltip { - position: absolute; - text-align: center; - width: auto; - height: auto; - padding: 2px; - font: 12px sans-serif; - background: white; - border: 1px solid black; - border-radius: 4px; - pointer-events: none; -} - .swh-visits-list-column { float: left; padding: 10px; } .swh-visits-list-row { padding-left: 50px; } .swh-visits-list-row::after { content: ""; display: table; clear: both; } diff --git a/swh/web/assets/src/bundles/webapp/history-counters.css b/swh/web/assets/src/bundles/webapp/history-counters.css new file mode 100644 index 00000000..2486d15b --- /dev/null +++ b/swh/web/assets/src/bundles/webapp/history-counters.css @@ -0,0 +1,32 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +.swh-history-counter-line { + fill: none; + stroke: #ffab00; + stroke-width: 3; +} + +.swh-history-counter-overlay { + fill: none; + pointer-events: all; +} + +.swh-history-counter-focus circle { + fill: none; + stroke: #ffab00; + stroke-width: 3; +} + +.swh-history-counter-focus text { + font-size: 14px; +} + +.swh-history-counter-tooltip { + fill: white; + stroke: #000; +} diff --git a/swh/web/assets/src/bundles/webapp/history-counters.js b/swh/web/assets/src/bundles/webapp/history-counters.js new file mode 100644 index 00000000..9b15bd67 --- /dev/null +++ b/swh/web/assets/src/bundles/webapp/history-counters.js @@ -0,0 +1,143 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +import './history-counters.css'; + +export async function drawHistoryCounterGraph(container, historyData) { + + const d3 = await import(/* webpackChunkName: "d3" */ 'utils/d3'); + + // remove previously created histogram and tooltip if any + d3.select(container).select('svg').remove(); + d3.select(`${container}-tooltip`).remove(); + + // histogram size and margins + let width = 400; + let height = 300; + const margin = {top: 20, right: 50, bottom: 70, left: 30}; + + // create responsive svg + const svg = d3.select(container) + .attr('style', + `padding-bottom: ${Math.ceil(height * 100 / width)}%`) + .append('svg') + .attr('viewBox', `0 0 ${width} ${height}`); + + // create tooltip div + const tooltip = d3.select('body') + .append('div') + .attr('class', 'd3-tooltip') + .attr('id', `${container}-tooltip`) + .style('opacity', 0); + + // update width and height without margins + width = width - margin.left - margin.right; + height = height - margin.top - margin.bottom; + + const firstPoint = historyData[0]; + const lastPoint = historyData[historyData.length - 1]; + + // create main svg group element + const g = svg.append('g') + .attr('transform', `translate(${margin.left}, ${margin.top})`); + + // create x scale + const xScale = d3.scaleTime() + .rangeRound([0, width]) + .domain([firstPoint[0], lastPoint[0]]) + .nice(); + + // create y scale + const yScale = d3.scaleLinear() + .range([height, 0]) + .domain([firstPoint[1], lastPoint[1]]) + .nice(); + + // create line generator + const line = d3.line() + .x(d => xScale(d[0])) + .y(d => yScale(d[1])); + + // utility functions + const dateFormatter = d3.timeFormat('%d %b %Y'); + const valueFormatter = d3.format('.3s'); + const bisectDate = d3.bisector(d => d[0]).left; + + // add x axis + g.append('g') + .attr('class', 'axis') + .attr('transform', `translate(0, ${height})`) + .call( + d3.axisBottom(xScale) + .ticks(10) + .tickFormat(dateFormatter) + ) + .selectAll('text') + .style('text-anchor', 'end') + .attr('dx', '-.8em') + .attr('dy', '.15em') + .attr('transform', 'rotate(-65)'); + + // add y axis + g.append('g') + .attr('class', 'axis') + .attr('transform', `translate(${width}, 0)`) + .call( + d3.axisRight(yScale) + .ticks(10) + .tickFormat(valueFormatter) + ); + + // add data plot + g.append('path') + .datum(historyData) + .attr('class', 'swh-history-counter-line') + .attr('d', line); + + // add tooltip + const focus = g.append('g') + .attr('class', 'swh-history-counter-focus') + .style('display', 'none'); + + focus.append('circle') + .attr('r', 8); + + g.append('rect') + .attr('class', 'swh-history-counter-overlay') + .attr('width', width) + .attr('height', height) + .on('mouseover', function() { + focus.style('display', null); + updateTooltip(this); + tooltip.transition() + .duration(200) + .style('opacity', 1); + }) + .on('mouseout', () => { + focus.style('display', 'none'); + tooltip.transition() + .duration(200) + .style('opacity', 0); + }) + .on('mousemove', function() { + updateTooltip(this); + }); + + function updateTooltip(elt) { + const x0 = xScale.invert(d3.mouse(elt)[0]); + const i = bisectDate(historyData, x0, 1); + if (i >= historyData.length) return; + const d0 = historyData[i - 1]; + const d1 = historyData[i]; + const d = x0 - d0[0] > d1[0] - x0 ? d1 : d0; + focus.attr('transform', `translate(${xScale(d[0])}, ${yScale(d[1])})`); + const tooltipText = `${dateFormatter(d[0])} ${valueFormatter(d[1])}`; + tooltip.html(tooltipText) + .style('left', d3.event.pageX + 15 + 'px') + .style('top', d3.event.pageY + 'px'); + } +} diff --git a/swh/web/assets/src/bundles/webapp/index.js b/swh/web/assets/src/bundles/webapp/index.js index 686a4a29..7e91f70a 100644 --- a/swh/web/assets/src/bundles/webapp/index.js +++ b/swh/web/assets/src/bundles/webapp/index.js @@ -1,25 +1,26 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // webapp entrypoint bundle centralizing global custom stylesheets // and utility js modules used in all swh-web applications // explicitly import the vendors bundle import '../vendors'; // global swh-web custom stylesheets import './webapp.css'; import './breadcrumbs.css'; export * from './webapp-utils'; // utility js modules export * from './code-highlighting'; export * from './readme-rendering'; export * from './pdf-rendering'; export * from './notebook-rendering'; export * from './xss-filtering'; +export * from './history-counters'; diff --git a/swh/web/assets/src/bundles/webapp/webapp-utils.js b/swh/web/assets/src/bundles/webapp/webapp-utils.js index 79bda034..3eba16c3 100644 --- a/swh/web/assets/src/bundles/webapp/webapp-utils.js +++ b/swh/web/assets/src/bundles/webapp/webapp-utils.js @@ -1,245 +1,266 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import objectFitImages from 'object-fit-images'; import {Layout} from 'admin-lte'; import {selectText} from 'utils/functions'; import {BREAKPOINT_MD} from 'utils/constants'; let collapseSidebar = false; let previousSidebarState = localStorage.getItem('swh-sidebar-collapsed'); if (previousSidebarState !== undefined) { collapseSidebar = JSON.parse(previousSidebarState); } // adapt implementation of fixLayoutHeight from admin-lte Layout.prototype.fixLayoutHeight = () => { let heights = { window: $(window).height(), header: $('.main-header').outerHeight(), footer: $('.footer').outerHeight(), sidebar: $('.main-sidebar').height(), topbar: $('.swh-top-bar').height() }; let offset = 10; $('.content-wrapper').css('min-height', heights.window - heights.topbar - heights.header - heights.footer - offset); $('.main-sidebar').css('min-height', heights.window - heights.topbar - heights.header - heights.footer - offset); }; $(document).on('DOMContentLoaded', () => { // set state to collapsed on smaller devices if ($(window).width() < BREAKPOINT_MD) { collapseSidebar = true; } // restore previous sidebar state (collapsed/expanded) if (collapseSidebar) { // hack to avoid animated transition for collapsing sidebar // when loading a page let sidebarTransition = $('.main-sidebar, .main-sidebar:before').css('transition'); let sidebarEltsTransition = $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition'); $('.main-sidebar, .main-sidebar:before').css('transition', 'none'); $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition', 'none'); $('body').addClass('sidebar-collapse'); $('.swh-words-logo-swh').css('visibility', 'visible'); // restore transitions for user navigation setTimeout(() => { $('.main-sidebar, .main-sidebar:before').css('transition', sidebarTransition); $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition', sidebarEltsTransition); }); } }); $(document).on('collapsed.lte.pushmenu', event => { if ($('body').width() >= BREAKPOINT_MD) { $('.swh-words-logo-swh').css('visibility', 'visible'); } }); $(document).on('shown.lte.pushmenu', event => { $('.swh-words-logo-swh').css('visibility', 'hidden'); }); function ensureNoFooterOverflow() { $('body').css('padding-bottom', $('footer').outerHeight() + 'px'); } $(document).ready(() => { // redirect to last browse page if any when clicking on the 'Browse' entry // in the sidebar $(`.swh-browse-link`).click(event => { let lastBrowsePage = sessionStorage.getItem('last-browse-page'); if (lastBrowsePage) { event.preventDefault(); window.location = lastBrowsePage; } }); // ensure footer do not overflow main content for mobile devices // or after resizing the browser window ensureNoFooterOverflow(); $(window).resize(function() { ensureNoFooterOverflow(); if ($('body').hasClass('sidebar-collapse') && $('body').width() >= BREAKPOINT_MD) { $('.swh-words-logo-swh').css('visibility', 'visible'); } }); // activate css polyfill 'object-fit: contain' in old browsers objectFitImages(); // reparent the modals to the top navigation div in order to be able // to display them $('.swh-browse-top-navigation').append($('.modal')); let selectedCode = null; function getCodeOrPreEltUnderPointer(e) { let elts = document.elementsFromPoint(e.clientX, e.clientY); for (let elt of elts) { if (elt.nodeName === 'CODE' || elt.nodeName === 'PRE') { return elt; } } return null; } // click handler to set focus on code block for copy $(document).click(e => { selectedCode = getCodeOrPreEltUnderPointer(e); }); function selectCode(event, selectedCode) { if (selectedCode) { let hljsLnCodeElts = $(selectedCode).find('.hljs-ln-code'); if (hljsLnCodeElts.length) { selectText(hljsLnCodeElts[0], hljsLnCodeElts[hljsLnCodeElts.length - 1]); } else { selectText(selectedCode.firstChild, selectedCode.lastChild); } event.preventDefault(); } } // select the whole text of focused code block when user // double clicks or hits Ctrl+A $(document).dblclick(e => { if ((e.ctrlKey || e.metaKey)) { selectCode(e, getCodeOrPreEltUnderPointer(e)); } }); $(document).keydown(e => { if ((e.ctrlKey || e.metaKey) && e.key === 'a') { selectCode(e, selectedCode); } }); // show/hide back-to-top button let scrollThreshold = 0; scrollThreshold += $('.swh-top-bar').height() || 0; scrollThreshold += $('.navbar').height() || 0; $(window).scroll(() => { if ($(window).scrollTop() > scrollThreshold) { $('#back-to-top').css('display', 'block'); } else { $('#back-to-top').css('display', 'none'); } }); }); export function initPage(page) { $(document).ready(() => { // set relevant sidebar link to page active $(`.swh-${page}-item`).addClass('active'); $(`.swh-${page}-link`).addClass('active'); // triggered when unloading the current page $(window).on('unload', () => { // backup sidebar state (collapsed/expanded) let sidebarCollapsed = $('body').hasClass('sidebar-collapse'); localStorage.setItem('swh-sidebar-collapsed', JSON.stringify(sidebarCollapsed)); // backup current browse page if (page === 'browse') { sessionStorage.setItem('last-browse-page', window.location); } }); }); } +export function initHomePage() { + $(document).ready(() => { + $('.swh-coverage-list').iFrameResize({heightCalculationMethod: 'taggedElement'}); + fetch(Urls.stat_counters()) + .then(response => response.json()) + .then(data => { + if (data.stat_counters) { + $('#nb-files').html(data.stat_counters.content.toLocaleString()); + $('#nb-commits').html(data.stat_counters.revision.toLocaleString()); + $('#nb-projects').html(data.stat_counters.origin.toLocaleString()); + } + if (data.stat_counters_history) { + swh.webapp.drawHistoryCounterGraph('#nb-files-history', data.stat_counters_history.content); + swh.webapp.drawHistoryCounterGraph('#nb-commits-history', data.stat_counters_history.revision); + swh.webapp.drawHistoryCounterGraph('#nb-projects-history', data.stat_counters_history.origin); + } + }); + }); + initPage('home'); +} + export function showModalMessage(title, message) { $('#swh-web-modal-message .modal-title').text(title); $('#swh-web-modal-message .modal-content p').text(message); $('#swh-web-modal-message').modal('show'); } export function showModalConfirm(title, message, callback) { $('#swh-web-modal-confirm .modal-title').text(title); $('#swh-web-modal-confirm .modal-content p').text(message); $('#swh-web-modal-confirm #swh-web-modal-confirm-ok-btn').bind('click', () => { callback(); $('#swh-web-modal-confirm').modal('hide'); $('#swh-web-modal-confirm #swh-web-modal-confirm-ok-btn').unbind('click'); }); $('#swh-web-modal-confirm').modal('show'); } export function addJumpToPagePopoverToDataTable(dataTableElt) { dataTableElt.on('draw.dt', function() { $('.paginate_button.disabled').css('cursor', 'pointer'); $('.paginate_button.disabled').on('click', event => { const pageInfo = dataTableElt.page.info(); let content = ' / ${pageInfo.pages}`; $(event.target).popover({ 'title': 'Jump to page', 'content': content, 'html': true, 'placement': 'top', 'sanitizeFn': swh.webapp.filterXSS }); $(event.target).popover('show'); $('.jump-to-page').on('change', function() { $('.paginate_button.disabled').popover('hide'); const pageNumber = parseInt($(this).val()) - 1; dataTableElt.page(pageNumber).draw('page'); }); }); }); dataTableElt.on('preXhr.dt', () => { $('.paginate_button.disabled').popover('hide'); }); } let swhObjectIcons; export function setSwhObjectIcons(icons) { swhObjectIcons = icons; } export function getSwhObjectIcon(swhObjectType) { return swhObjectIcons[swhObjectType]; } let browsedSwhObjectMetadata = {}; export function setBrowsedSwhObjectMetadata(metadata) { browsedSwhObjectMetadata = metadata; } export function getBrowsedSwhObjectMetadata() { return browsedSwhObjectMetadata; } diff --git a/swh/web/assets/src/bundles/webapp/webapp.css b/swh/web/assets/src/bundles/webapp/webapp.css index 945d6a3b..cf7bb82e 100644 --- a/swh/web/assets/src/bundles/webapp/webapp.css +++ b/swh/web/assets/src/bundles/webapp/webapp.css @@ -1,536 +1,573 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ html { height: 100%; overflow-x: hidden; } body { min-height: 100%; margin: 0; position: relative; padding-bottom: 120px; } a:active, a.active { outline: none; } code { background-color: #f9f2f4; } pre code { background-color: transparent; } footer { background-color: #262626; color: #fff; font-size: 0.8rem; position: absolute; bottom: 0; width: 100%; padding-top: 20px; padding-bottom: 20px; } footer a, footer a:visited, footer a:hover { color: #fecd1b; } footer a:hover { text-decoration: underline; } .link-color { color: #fecd1b; } pre { background-color: #f5f5f5; border: 1px solid #ccc; border-radius: 4px; padding: 9.5px; font-size: 0.8rem; } .btn.active { background-color: #e7e7e7; } .card { margin-bottom: 5px !important; overflow-x: auto; } .navbar-brand { padding: 5px; margin-right: 0; } .table { margin-bottom: 0; } .swh-table thead { background-color: #f2f4f5; border-top: 1px solid rgba(0, 0, 0, 0.2); font-weight: normal; } .swh-table-striped th { border-top: none; } .swh-table-striped tbody tr:nth-child(even) { background-color: #f2f4f5; } .swh-table-striped tbody tr:nth-child(odd) { background-color: #fff; } .swh-web-app-link a { text-decoration: none; border: none; } .swh-web-app-link:hover { background-color: #efeff2; } .table > thead > tr > th { border-top: none; border-bottom: 1px solid #e20026; } .table > tbody > tr > td { border-style: none; } .sitename .first-word, .sitename .second-word { color: rgba(0, 0, 0, 0.75); font-weight: normal; font-size: 1.2rem; } .sitename .first-word { font-family: 'Alegreya Sans', sans-serif; } .sitename .second-word { font-family: 'Alegreya', serif; } .swh-counter { font-size: 150%; } +@media (max-width: 600px) { + .swh-counter-container { + margin-top: 1rem; + } +} + .swh-http-error { margin: 0 auto; text-align: center; } .swh-http-error-head { color: #2d353c; font-size: 30px; } .swh-http-error-code { bottom: 60%; color: #2d353c; font-size: 96px; line-height: 80px; margin-bottom: 10px !important; } .swh-http-error-desc { font-size: 12px; color: #647788; text-align: center; } .swh-http-error-desc pre { display: inline-block; text-align: left; max-width: 800px; white-space: pre-wrap; } .popover { max-width: 97%; z-index: 40000; } .modal { text-align: center; padding: 0 !important; } .modal::before { content: ''; display: inline-block; height: 100%; vertical-align: middle; margin-right: -4px; } .modal-dialog { display: inline-block; text-align: left; vertical-align: middle; } .dropdown-submenu { position: relative; } .dropdown-submenu .dropdown-menu { top: 0; left: -100%; margin-top: -5px; margin-left: -2px; } .dropdown-item:hover, .dropdown-item:focus { background-color: rgba(0, 0, 0, 0.1); } a.dropdown-left::before { content: "\f0d9"; font-family: 'FontAwesome'; display: block; width: 20px; height: 20px; float: left; margin-left: 0; } #swh-navbar { border-top-style: none; border-left-style: none; border-right-style: none; border-bottom-style: solid; border-bottom-width: 5px; border-image: linear-gradient(to right, rgb(226, 0, 38) 0%, rgb(254, 205, 27) 100%) 1 1 1 1; width: 100%; padding: 5px; margin-bottom: 10px; margin-top: 30px; justify-content: normal; flex-wrap: nowrap; height: 72px; overflow: hidden; } #back-to-top { display: none; position: fixed; bottom: 30px; right: 30px; z-index: 10; } #back-to-top a img { display: block; width: 32px; height: 32px; background-size: 32px 32px; text-indent: -999px; overflow: hidden; } .swh-top-bar { direction: ltr; height: 30px; position: fixed; top: 0; left: 0; width: 100%; z-index: 99999; background-color: #262626; color: #fff; text-align: center; font-size: 14px; } .swh-top-bar ul { margin-top: 4px; padding-left: 0; white-space: nowrap; } .swh-top-bar li { display: inline-block; margin-left: 10px; margin-right: 10px; } .swh-top-bar a, .swh-top-bar a:visited { color: white; } .swh-top-bar a.swh-current-site, .swh-top-bar a.swh-current-site:visited { color: #fecd1b; } .swh-position-right { position: absolute; right: 0; } .swh-donate-link { border: 1px solid #fecd1b; background-color: #e20026; color: white !important; padding: 3px; border-radius: 3px; } .swh-navbar-content h4 { padding-top: 7px; } .swh-navbar-content .bread-crumbs { display: block; margin-left: -40px; } .swh-navbar-content .bread-crumbs li.bc-no-root { padding-top: 7px; } .main-sidebar { margin-top: 30px; } .content-wrapper { background: none; } .brand-image { max-height: 40px; } .brand-link { padding-top: 18.5px; padding-bottom: 18px; padding-left: 4px; border-bottom: 5px solid #e20026 !important; } .navbar-header a, ul.dropdown-menu a, ul.navbar-nav a, ul.nav-sidebar a { border-bottom-style: none; color: #323232; } .swh-sidebar .nav-link.active { color: #323232 !important; background-color: #e7e7e7 !important; } .swh-image-error { width: 80px; height: auto; } @media (max-width: 600px) { .card { min-width: 80%; } .swh-image-error { width: 40px; height: auto; } .swh-navbar-content h4 { font-size: 1rem; } .swh-donate-link { display: none; } } .form-check-label { padding-top: 4px; } .swh-id-option { display: inline-block; margin-right: 5px; line-height: 1rem; } .nav-pills .nav-link:not(.active):hover { color: rgba(0, 0, 0, 0.55); } .swh-heading-color { color: #e20026 !important; } .sidebar-mini.sidebar-collapse .main-sidebar:hover { width: 4.6rem; } .sidebar-mini.sidebar-collapse .main-sidebar:hover .user-panel > .info, .sidebar-mini.sidebar-collapse .main-sidebar:hover .nav-sidebar .nav-link p, .sidebar-mini.sidebar-collapse .main-sidebar:hover .brand-text { visibility: hidden !important; } .sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info { transition: none; } .sidebar-mini.sidebar-mini.sidebar-collapse .sidebar { padding-right: 0; } .swh-words-logo { position: absolute; top: 0; left: 0; width: 73px; height: 73px; text-align: center; font-size: 10pt; color: rgba(0, 0, 0, 0.75); } .swh-words-logo:hover { text-decoration: none; } .swh-words-logo-swh { line-height: 1; padding-top: 13px; visibility: hidden; } hr.swh-faded-line { border: 0; height: 1px; background-image: linear-gradient(to left, #f0f0f0, #8c8b8b, #f0f0f0); } .swh-readme-txt pre { background: none; border: none; } .swh-coverage-col { padding-left: 10px; padding-right: 10px; } .swh-coverage { height: calc(65px + 1em); padding-top: 0.3rem; border: none; } .swh-coverage a { text-decoration: none; } .swh-coverage-logo { display: block; width: 100%; height: 50px; margin-left: auto; margin-right: auto; object-fit: contain; /* polyfill for old browsers, see https://github.com/bfred-it/object-fit-images */ font-family: 'object-fit: contain;'; } .swh-coverage-list { width: 100%; height: 320px; border: none; } tr.swh-tr-hover-highlight:hover td { background: #ededed; } tr.swh-api-doc-route a { text-decoration: none; } .swh-apidoc .col { margin: 10px; } a.toggle-col { text-decoration: none; } a.toggle-col.col-hidden { text-decoration: line-through; } .admonition.warning { background: #fcf8e3; border: 1px solid #faebcc; padding: 15px; border-radius: 4px; } .admonition.warning p { margin-bottom: 0; } .admonition.warning .first { font-size: 1.5rem; } .swh-popover { max-height: 50vh; overflow-y: auto; overflow-x: auto; padding: 0; padding-right: 1.4em; } @media screen and (min-width: 768px) { .swh-popover { max-width: 50vw; } } .swh-metadata-table-row { border-top: 1px solid #ddd !important; } .swh-metadata-table-key { min-width: 200px; max-width: 200px; width: 200px; } .swh-metadata-table-value pre { white-space: pre-wrap; } + +.d3-wrapper { + position: relative; + height: 0; + width: 100%; + padding: 0; + + /* padding-bottom will be overwritten by JavaScript later */ + padding-bottom: 100%; +} + +.d3-wrapper > svg { + position: absolute; + height: 100%; + width: 100%; + left: 0; + top: 0; +} + +div.d3-tooltip { + position: absolute; + text-align: center; + width: auto; + height: auto; + padding: 2px; + font: 12px sans-serif; + background: white; + border: 1px solid black; + border-radius: 4px; + pointer-events: none; +} diff --git a/swh/web/assets/src/utils/d3-custom.js b/swh/web/assets/src/utils/d3-custom.js new file mode 100644 index 00000000..210967ee --- /dev/null +++ b/swh/web/assets/src/utils/d3-custom.js @@ -0,0 +1,22 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +import * as d3 from 'd3-format'; + +const d3Format = d3.format; + +// Override d3.format in order to use B for denoting billions instead of G +function customD3Format() { + const ret = d3Format.apply(d3, arguments); + return (function(args) { + return function() { + return args.apply(d3, arguments).replace(/G/, 'B'); + }; + })(ret); +} + +export {customD3Format as format}; diff --git a/swh/web/assets/src/utils/d3.js b/swh/web/assets/src/utils/d3.js index 939c487e..9059ca63 100644 --- a/swh/web/assets/src/utils/d3.js +++ b/swh/web/assets/src/utils/d3.js @@ -1,38 +1,42 @@ /** * Copyright (C) 2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ +// export overridden d3 functions first +export * from './d3-custom'; + +// export required d3 modules export * from 'd3-array'; export * from 'd3-axis'; // export * from 'd3-brush'; // export * from 'd3-chord'; export * from 'd3-collection'; // export * from 'd3-color'; // export * from 'd3-contour'; // export * from 'd3-dispatch'; // export * from 'd3-drag'; // export * from 'd3-dsv'; // export * from 'd3-ease'; // export * from 'd3-fetch'; // export * from 'd3-force'; -// export * from 'd3-format'; +export * from 'd3-format'; // export * from 'd3-geo'; // export * from 'd3-hierarchy'; // export * from 'd3-interpolate'; // export * from 'd3-path'; // export * from 'd3-polygon'; // export * from 'd3-quadtree'; // export * from 'd3-random'; export * from 'd3-scale'; // export * from 'd3-scale-chromatic'; export * from 'd3-selection'; export * from 'd3-shape'; export * from 'd3-time'; export * from 'd3-time-format'; // export * from 'd3-timer'; export * from 'd3-transition'; // export * from 'd3-voronoi'; // export * from 'd3-zoom'; diff --git a/swh/web/config.py b/swh/web/config.py index d90350a2..19b30826 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,157 +1,158 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from swh.core import config from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.storage import get_storage from swh.vault import get_vault from swh.web import settings SETTINGS_DIR = os.path.dirname(settings.__file__) DEFAULT_CONFIG = { 'allowed_hosts': ('list', []), 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', 'timeout': 10, }, }), 'indexer_storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5007/', 'timeout': 1, } }), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', False), 'serve_assets': ('bool', False), 'host': ('string', '127.0.0.1'), 'port': ('int', 5004), 'secret_key': ('string', 'development key'), # do not display code highlighting for content > 1MB 'content_display_max_size': ('int', 5 * 1024 * 1024), 'snapshot_content_max_size': ('int', 1000), 'throttling': ('dict', { 'cache_uri': None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None 'scopes': { 'swh_api': { 'limiter_rate': { 'default': '120/h' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_vault_cooking': { 'limiter_rate': { 'default': '120/h', 'GET': '60/m' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_save_origin': { 'limiter_rate': { 'default': '120/h', 'POST': '10/h' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_api_origin_visit_latest': { 'limiter_rate': { 'default': '700/m' }, 'exempted_networks': ['127.0.0.0/8'], }, } }), 'vault': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5005/', } }), 'scheduler': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5008/' } }), 'development_db': ('string', os.path.join(SETTINGS_DIR, 'db.sqlite3')), 'test_db': ('string', os.path.join(SETTINGS_DIR, 'testdb.sqlite3')), 'production_db': ('string', '/var/lib/swh/web.sqlite3'), 'deposit': ('dict', { 'private_api_url': 'https://deposit.softwareheritage.org/1/private/', 'private_api_user': 'swhworker', 'private_api_password': '' }), 'coverage_count_origins': ('bool', False), 'e2e_tests_mode': ('bool', False), 'es_workers_index_url': ('string', ''), + 'history_counters_url': ('string', 'https://stats.export.softwareheritage.org/history_counters.json'), # noqa } swhweb_config = {} def get_config(config_file='web/web'): """Read the configuration file `config_file`. If an environment variable SWH_CONFIG_FILENAME is defined, this takes precedence over the config_file parameter. In any case, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration. """ if not swhweb_config: config_filename = os.environ.get('SWH_CONFIG_FILENAME') if config_filename: config_file = config_filename cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) config.prepare_folders(swhweb_config, 'log_dir') swhweb_config['storage'] = get_storage(**swhweb_config['storage']) swhweb_config['vault'] = get_vault(**swhweb_config['vault']) swhweb_config['indexer_storage'] = \ get_indexer_storage(**swhweb_config['indexer_storage']) swhweb_config['scheduler'] = get_scheduler( **swhweb_config['scheduler']) return swhweb_config def storage(): """Return the current application's storage. """ return get_config()['storage'] def vault(): """Return the current application's vault. """ return get_config()['vault'] def indexer_storage(): """Return the current application's indexer storage. """ return get_config()['indexer_storage'] def scheduler(): """Return the current application's scheduler. """ return get_config()['scheduler'] diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py index 83f3f407..a37f675c 100644 --- a/swh/web/misc/urls.py +++ b/swh/web/misc/urls.py @@ -1,58 +1,75 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json +import requests + from django.conf.urls import url, include from django.contrib.staticfiles import finders +from django.http import HttpResponse from django.shortcuts import render +from swh.web.common import service from swh.web.config import get_config def _jslicenses(request): jslicenses_file = finders.find('jssources/jslicenses.json') jslicenses_data = json.load(open(jslicenses_file)) jslicenses_data = sorted(jslicenses_data.items(), key=lambda item: item[0].split('/')[-1]) return render(request, "misc/jslicenses.html", {'jslicenses_data': jslicenses_data}) +def _stat_counters(request): + stat = service.stat_counters() + url = get_config()['history_counters_url'] + stat_counters_history = 'null' + if url: + response = requests.get(url) + stat_counters_history = response.text + json_data = '{"stat_counters": %s, "stat_counters_history": %s}' % ( + json.dumps(stat), stat_counters_history) + return HttpResponse(json_data, content_type='application/json') + + urlpatterns = [ url(r'^', include('swh.web.misc.coverage')), url(r'^jslicenses/$', _jslicenses, name='jslicenses'), url(r'^', include('swh.web.misc.origin_save')), + url(r'^stat_counters', _stat_counters, name='stat-counters'), ] # when running end to end tests trough cypress, declare some extra # endpoints to provide input data for some of those tests if get_config()['e2e_tests_mode']: from swh.web.tests.data import ( get_content_code_data_by_ext, get_content_other_data_by_ext, get_content_code_data_all_exts, get_content_code_data_by_filename, get_content_code_data_all_filenames, ) # noqa urlpatterns.append( url(r'^tests/data/content/code/extension/(?P.+)/$', get_content_code_data_by_ext, name='tests-content-code-extension')) urlpatterns.append( url(r'^tests/data/content/other/extension/(?P.+)/$', get_content_other_data_by_ext, name='tests-content-other-extension')) urlpatterns.append(url(r'^tests/data/content/code/extensions/$', get_content_code_data_all_exts, name='tests-content-code-extensions')) urlpatterns.append( url(r'^tests/data/content/code/filename/(?P.+)/$', get_content_code_data_by_filename, name='tests-content-code-filename')) urlpatterns.append(url(r'^tests/data/content/code/filenames/$', get_content_code_data_all_filenames, name='tests-content-code-filenames')) diff --git a/swh/web/settings/tests.py b/swh/web/settings/tests.py index ced666b3..a91d22a9 100644 --- a/swh/web/settings/tests.py +++ b/swh/web/settings/tests.py @@ -1,102 +1,103 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django tests settings for swh-web. """ import sys from swh.web.config import get_config scope1_limiter_rate = 3 scope1_limiter_rate_post = 1 scope2_limiter_rate = 5 scope2_limiter_rate_post = 2 scope3_limiter_rate = 1 scope3_limiter_rate_post = 1 save_origin_rate_post = 10 swh_web_config = get_config() swh_web_config.update({ 'debug': False, 'secret_key': 'test', + 'history_counters_url': '', 'throttling': { 'cache_uri': None, 'scopes': { 'swh_api': { 'limiter_rate': { 'default': '60/min' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_api_origin_visit_latest': { 'limiter_rate': { 'default': '6000/min' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_vault_cooking': { 'limiter_rate': { 'default': '120/h', 'GET': '60/m' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_save_origin': { 'limiter_rate': { 'default': '120/h', 'POST': '%s/h' % save_origin_rate_post, } }, 'scope1': { 'limiter_rate': { 'default': '%s/min' % scope1_limiter_rate, 'POST': '%s/min' % scope1_limiter_rate_post, } }, 'scope2': { 'limiter_rate': { 'default': '%s/min' % scope2_limiter_rate, 'POST': '%s/min' % scope2_limiter_rate_post } }, 'scope3': { 'limiter_rate': { 'default': '%s/min' % scope3_limiter_rate, 'POST': '%s/min' % scope3_limiter_rate_post }, 'exempted_networks': ['127.0.0.0/8'] } } } }) from .common import * # noqa from .common import ALLOWED_HOSTS, LOGGING # noqa DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': swh_web_config['test_db'], } } # when not running unit tests, make the webapp fetch data from memory storages if 'pytest' not in sys.argv[0]: swh_web_config.update({ 'debug': True, 'e2e_tests_mode': True }) from swh.web.tests.data import get_tests_data, override_storages # noqa test_data = get_tests_data() override_storages(test_data['storage'], test_data['idx_storage']) else: ALLOWED_HOSTS += ['testserver'] # Silent DEBUG output when running unit tests LOGGING['handlers']['console']['level'] = 'INFO' diff --git a/swh/web/templates/homepage.html b/swh/web/templates/homepage.html index 4bd35348..c3ca9e07 100644 --- a/swh/web/templates/homepage.html +++ b/swh/web/templates/homepage.html @@ -1,121 +1,102 @@ {% extends "layout.html" %} {% comment %} Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load static %} {% block title %}Welcome to the Software Heritage archive{% endblock %} {% block navbar-content %}

Welcome to the Software Heritage archive

{% endblock %} {% block content %}

Overview

The long term goal of the Software Heritage initiative is to collect all publicly available software in source code form together with its development history, replicate it massively to ensure its preservation, and share it with everyone who needs it. The Software Heritage archive is growing over time as we crawl new source code from software projects and development forges. We will incrementally release archive search and browse functionalities — as of now you can check whether source code you care about is already present in the archive or not.

Content

A significant amount of source code has already been ingested in the Software Heritage archive. It currently includes:

Size

As of today the archive already contains and keeps safe for you the following amount of objects:

-
+
Source files
+
-
-
Directories
- -
-
+
Commits
+
-
-
Authors
- -
-
+
Projects
-
-
-
Releases
- +
+ +

+ Note: the counters and graphs above are based on heuristics that might not reflect the exact + size of the archive. While the long-term trends shown and ballpark figures are reliable, + individual point-in-time values might not be. +

+

Access

{% endblock %}