diff --git a/package.json b/package.json index a4643bb5..f2546756 100644 --- a/package.json +++ b/package.json @@ -1,104 +1,107 @@ { "name": "swh-web", "version": "0.0.190", "description": "Static assets management for swh-web", "scripts": { "build-dev": "NODE_ENV=development webpack --config ./swh/web/assets/config/webpack.config.development.js --display-modules --progress --colors", "start-dev": "NODE_ENV=development nodemon --watch swh/web/api --watch swh/web/browse --watch swh/web/templates --watch swh/web/common --watch swh/web/settings --watch swh/web/assets/config --ext py,html,js --exec \"webpack-dev-server --config ./swh/web/assets/config/webpack.config.development.js --progress --colors\"", "build": "NODE_ENV=production webpack --config ./swh/web/assets/config/webpack.config.production.js --display-modules --progress --colors" }, "repository": { "type": "git", "url": "https://forge.softwareheritage.org/source/swh-web" }, "author": "The Software Heritage developers", "license": "AGPL-3.0-or-later", "dependencies": { "admin-lte": "^3.0.0-alpha", + "ansi_up": "^4.0.3", "bootstrap": "^4.3.1", "bootstrap-year-calendar-bs4": "^1.0.0", "clipboard": "^2.0.4", "d3": "^5.9.2", "datatables.net-bs4": "^1.10.19", "dompurify": "^1.0.10", "elementsfrompoint-polyfill": "^1.0.0", "font-awesome": "^4.7.0", "highlight.js": "^9.15.6", "highlightjs-line-numbers.js": "^2.7.0", "iframe-resizer": "^4.1.1", "jquery": "^3.4.0", "js-cookie": "^2.2.0", + "notebookjs": "^0.4.2", "object-fit-images": "^3.2.4", "octicons": "^8.5.0", "open-iconic": "^1.1.1", "org": "^0.2.0", "pdfjs-dist": "^2.0.943", "popper.js": "^1.15.0", "showdown": "^1.9.0", "typeface-alegreya": "0.0.69", "typeface-alegreya-sans": "^0.0.72", "url-search-params-polyfill": "^5.1.0", "validate.js": "^0.12.0", "waypoints": "^4.0.1", "whatwg-fetch": "^3.0.0" }, "devDependencies": { "@babel/core": "^7.4.3", "@babel/plugin-syntax-dynamic-import": "^7.2.0", "@babel/plugin-transform-runtime": "^7.4.3", "@babel/polyfill": "^7.4.3", "@babel/preset-env": "^7.4.3", "@babel/runtime-corejs2": "^7.4.3", "autoprefixer": "^9.5.1", "babel-eslint": "^10.0.1", "babel-loader": "^8.0.5", "bootstrap-loader": "^3.0.2", "cache-loader": "^2.0.1", "clean-webpack-plugin": "^2.0.1", "copy-webpack-plugin": "^5.0.2", "css-loader": "^2.1.1", "ejs": "^2.6.1", "eslint": "^5.15.3", "eslint-loader": "^2.1.2", "eslint-plugin-import": "^2.17.2", "eslint-plugin-node": "^8.0.1", "eslint-plugin-promise": "^4.1.1", "eslint-plugin-standard": "^4.0.0", "exports-loader": "^0.7.0", "expose-loader": "^0.7.5", "file-loader": "^3.0.1", "imports-loader": "^0.8.0", "less": "^3.9.0", "less-loader": "^4.1.0", "mini-css-extract-plugin": "^0.6.0", "node-sass": "^4.11.0", "nodemon": "^1.18.11", "optimize-css-assets-webpack-plugin": "^5.0.1", "postcss-loader": "^3.0.0", "postcss-normalize": "^7.0.1", "resolve-url-loader": "^3.1.0", "robotstxt-webpack-plugin": "^5.0.0", "sass-loader": "^7.1.0", "schema-utils": "^1.0.0", + "script-loader": "^0.7.2", "spdx-expression-parse": "^3.0.0", "style-loader": "^0.23.1", "stylelint": "^10.0.1", "stylelint-config-standard": "^18.3.0", "terser-webpack-plugin": "^1.2.3", "url-loader": "^1.1.2", "webpack": "^4.30.0", "webpack-bundle-tracker": "^0.4.2-beta", "webpack-cli": "^3.3.0", "webpack-dev-server": "^3.3.1" }, "browserslist": [ "cover 99.5%", "not dead" ], "postcss": { "plugins": { "autoprefixer": {}, "postcss-normalize": {} } } } diff --git a/swh/web/assets/config/.eslintrc b/swh/web/assets/config/.eslintrc index 0dcf18df..dde1f8bd 100644 --- a/swh/web/assets/config/.eslintrc +++ b/swh/web/assets/config/.eslintrc @@ -1,307 +1,309 @@ { "parser": "babel-eslint", "parserOptions": { "ecmaVersion": 2017, "ecmaFeatures": { "experimentalObjectRestSpread": true, "jsx": true }, "sourceType": "module", "allowImportExportEverywhere": true }, "env": { "es6": true, "node": true }, "plugins": [ "import", "node", "promise", "standard" ], "globals": { "document": false, "navigator": false, "window": false, "$": false, "jQuery": false, "history": false, "localStorage": false, "sessionStorage": false, "Urls": false, "hljs": false, "Waypoint": false, "swh": false, "fetch": false, "__STATIC__": false, "Image": false, "Cookies": false, - "grecaptcha": false + "grecaptcha": false, + "nb": false, + "MathJax": false }, "rules": { "accessor-pairs": "error", "arrow-spacing": ["error", { "before": true, "after": true }], "block-spacing": ["error", "always"], "brace-style": ["error", "1tbs", { "allowSingleLine": true }], "camelcase": ["error", { "properties": "never" }], "comma-dangle": ["error", { "arrays": "never", "objects": "never", "imports": "never", "exports": "never", "functions": "never" }], "comma-spacing": ["error", { "before": false, "after": true }], "comma-style": ["error", "last"], "constructor-super": "error", "curly": ["error", "multi-line"], "dot-location": ["error", "property"], "eol-last": "error", "eqeqeq": ["error", "always", { "null": "ignore" }], "func-call-spacing": ["error", "never"], "generator-star-spacing": ["error", { "before": true, "after": true }], "handle-callback-err": ["error", "^(err|error)$"], "indent": ["error", 2, { "SwitchCase": 1, "VariableDeclarator": 1, "outerIIFEBody": 1, "MemberExpression": 1, "FunctionDeclaration": { "parameters": "first", "body": 1 }, "FunctionExpression": { "parameters": "first", "body": 1 }, "CallExpression": { "arguments": "first" }, "ArrayExpression": "first", "ObjectExpression": "first", "ImportDeclaration": "first", "flatTernaryExpressions": false, "ignoreComments": false }], "key-spacing": ["error", { "beforeColon": false, "afterColon": true }], "keyword-spacing": ["error", { "before": true, "after": true }], "new-cap": ["error", { "newIsCap": true, "capIsNew": false }], "new-parens": "error", "no-array-constructor": "error", "no-caller": "error", "no-class-assign": "error", "no-compare-neg-zero": "error", "no-cond-assign": "error", "no-const-assign": "error", "no-constant-condition": ["error", { "checkLoops": false }], "no-control-regex": "error", "no-debugger": "error", "no-delete-var": "error", "no-dupe-args": "error", "no-dupe-class-members": "error", "no-dupe-keys": "error", "no-duplicate-case": "error", "no-empty-character-class": "error", "no-empty-pattern": "error", "no-eval": "error", "no-ex-assign": "error", "no-extend-native": "error", "no-extra-bind": "error", "no-extra-boolean-cast": "error", "no-extra-parens": ["error", "functions"], "no-fallthrough": "error", "no-floating-decimal": "error", "no-func-assign": "error", "no-global-assign": "error", "no-implied-eval": "error", "no-inner-declarations": ["error", "functions"], "no-invalid-regexp": "error", "no-irregular-whitespace": "error", "no-iterator": "error", "no-label-var": "error", "no-labels": ["error", { "allowLoop": false, "allowSwitch": false }], "no-lone-blocks": "error", "no-mixed-operators": ["error", { "groups": [ ["==", "!=", "===", "!==", ">", ">=", "<", "<="], ["&&", "||"], ["in", "instanceof"] ], "allowSamePrecedence": true }], "no-mixed-spaces-and-tabs": "error", "no-multi-spaces": "error", "no-multi-str": "error", "no-multiple-empty-lines": ["error", { "max": 1, "maxEOF": 0 }], "no-negated-in-lhs": "error", "no-new": 0, "no-new-func": "error", "no-new-object": "error", "no-new-require": "error", "no-new-symbol": "error", "no-new-wrappers": "error", "no-obj-calls": "error", "no-octal": "error", "no-octal-escape": "error", "no-path-concat": "error", "no-proto": "error", "no-redeclare": "error", "no-regex-spaces": "error", "no-return-assign": ["error", "except-parens"], "no-return-await": "error", "no-self-assign": "error", "no-self-compare": "error", "no-sequences": "error", "no-shadow-restricted-names": "error", "no-sparse-arrays": "error", "no-tabs": "error", "no-template-curly-in-string": "error", "no-this-before-super": "error", "no-throw-literal": "error", "no-trailing-spaces": "error", "no-undef": "error", "no-undef-init": "error", "no-unexpected-multiline": "error", "no-unmodified-loop-condition": "error", "no-unneeded-ternary": ["error", { "defaultAssignment": false }], "no-unreachable": "error", "no-unsafe-finally": "error", "no-unsafe-negation": "error", "no-unused-expressions": ["error", { "allowShortCircuit": true, "allowTernary": true, "allowTaggedTemplates": true }], "no-unused-vars": ["error", { "vars": "all", "args": "none", "ignoreRestSiblings": true }], "no-use-before-define": ["error", { "functions": false, "classes": false, "variables": false }], "no-useless-call": "error", "no-useless-computed-key": "error", "no-useless-constructor": "error", "no-useless-escape": "error", "no-useless-rename": "error", "no-useless-return": "error", "no-whitespace-before-property": "error", "no-with": "error", "object-property-newline": ["error", { "allowMultiplePropertiesPerLine": true }], "one-var": ["error", { "initialized": "never" }], "operator-linebreak": ["error", "after", { "overrides": { "?": "before", ":": "before" } }], "padded-blocks": ["off", { "blocks": "never", "switches": "never", "classes": "never" }], "prefer-promise-reject-errors": "error", "quotes": ["error", "single", { "avoidEscape": true, "allowTemplateLiterals": true }], "rest-spread-spacing": ["error", "never"], "semi": ["error", "always"], "semi-spacing": ["error", { "before": false, "after": true }], "space-before-blocks": ["error", "always"], "space-before-function-paren": ["error", "never"], "space-in-parens": ["error", "never"], "space-infix-ops": "error", "space-unary-ops": ["error", { "words": true, "nonwords": false }], "spaced-comment": ["error", "always", { "line": { "markers": ["*package", "!", "/", ",", "="] }, "block": { "balanced": true, "markers": ["*package", "!", ",", ":", "::", "flow-include"], "exceptions": ["*"] } }], "symbol-description": "error", "template-curly-spacing": ["error", "never"], "template-tag-spacing": ["error", "never"], "unicode-bom": ["error", "never"], "use-isnan": "error", "valid-typeof": ["error", { "requireStringLiterals": true }], "wrap-iife": ["error", "any", { "functionPrototypeMethods": true }], "yield-star-spacing": ["error", "both"], "yoda": ["error", "never"], "import/export": "error", "import/first": "error", "import/no-duplicates": "error", "import/no-webpack-loader-syntax": "off", "node/no-deprecated-api": "error", "node/process-exit-as-throw": "error", "promise/param-names": "error", "standard/array-bracket-even-spacing": ["error", "either"], "standard/computed-property-even-spacing": ["error", "even"], "standard/no-callback-literal": "error", "standard/object-curly-even-spacing": ["error", "either"] } } \ No newline at end of file diff --git a/swh/web/assets/config/mathjax-js-files.js b/swh/web/assets/config/mathjax-js-files.js new file mode 100644 index 00000000..33d2f25a --- /dev/null +++ b/swh/web/assets/config/mathjax-js-files.js @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +function getLoadedMathJaxJsFilesInfo(mathJaxJsFiles, cdnPrefix, unpackedLocationPrefix) { + let ret = {}; + for (let mathJaxJsFile of mathJaxJsFiles) { + ret[cdnPrefix + mathJaxJsFile] = [{ + 'id': mathJaxJsFile, + 'path': unpackedLocationPrefix + mathJaxJsFile, + 'spdxLicenseExpression': 'Apache-2.0', + 'licenseFilePath': '' + }]; + } + return ret; +} + +let cdnPrefix = 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/'; + +let unpackedLocationPrefix = 'https://raw.githubusercontent.com/mathjax/MathJax/2.7.5/unpacked/'; + +let mathJaxJsFiles = [ + 'MathJax.js', + 'config/TeX-AMS_HTML.js', + 'extensions/MathMenu.js', + 'jax/element/mml/optable/BasicLatin.js', + 'jax/output/HTML-CSS/jax.js', + 'jax/output/HTML-CSS/fonts/TeX/fontdata.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/Main.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/BBBold.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/GeneralPunctuation.js', + 'jax/output/HTML-CSS/autoload/mtable.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/MiscTechnical.js', + 'jax/output/HTML-CSS/fonts/TeX/Typewriter/Regular/Main.js', + 'jax/element/mml/optable/MathOperators.js', + 'jax/output/HTML-CSS/autoload/multiline.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/MathOperators.js', + 'jax/element/mml/optable/GeneralPunctuation.js' +]; + +module.exports = getLoadedMathJaxJsFilesInfo(mathJaxJsFiles, cdnPrefix, unpackedLocationPrefix); diff --git a/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js index f794db68..61b22a95 100644 --- a/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js +++ b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js @@ -1,383 +1,381 @@ /** * Copyright (C) 2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // This is a plugin for webpack >= 4 enabling to generate a Web Labels page intended // to be consume by the LibreJS Firefox plugin (https://www.gnu.org/software/librejs/). // See README.md for its complete documentation. const ejs = require('ejs'); const fs = require('fs'); const log = require('webpack-log'); const path = require('path'); const schema = require('./plugin-options-schema.json'); const spdxParse = require('spdx-expression-parse'); const spdxLicensesMapping = require('./spdx-licenses-mapping'); const validateOptions = require('schema-utils'); const pluginName = 'GenerateWebLabelsPlugin'; class GenerateWebLabelsPlugin { constructor(opts) { // check that provided options match JSON schema validateOptions(schema, opts, pluginName); this.options = opts || {}; this.weblabelsDirName = this.options['outputDir'] || 'jssources'; this.outputType = this.options['outputType'] || 'html'; // source file extension handled by webpack and compiled to js this.srcExts = ['js', 'ts', 'coffee', 'lua']; this.srcExtsRegexp = new RegExp('^.*.(' + this.srcExts.join('|') + ')$'); this.chunkNameToJsAsset = {}; this.chunkJsAssetToSrcFiles = {}; this.packageJsonCache = {}; this.packageLicenseFile = {}; this.exclude = []; this.copiedFiles = new Set(); this.logger = log({name: pluginName}); // populate module prefix patterns to exclude if (Array.isArray(this.options['exclude'])) { this.options['exclude'].forEach(toExclude => { if (!toExclude.startsWith('.')) { this.exclude.push('./' + path.join('node_modules', toExclude)); } else { this.exclude.push(toExclude); } }); } } apply(compiler) { compiler.hooks.done.tap(pluginName, statsObj => { // get the stats object in JSON format let stats = statsObj.toJson(); this.stats = stats; // set output folder this.weblabelsOutputDir = path.join(stats.outputPath, this.weblabelsDirName); this.recursiveMkdir(this.weblabelsOutputDir); // map each generated webpack chunk to its js asset Object.keys(stats.assetsByChunkName).forEach((chunkName, i) => { if (Array.isArray(stats.assetsByChunkName[chunkName])) { for (let asset of stats.assetsByChunkName[chunkName]) { if (asset.endsWith('.js')) { this.chunkNameToJsAsset[chunkName] = asset; this.chunkNameToJsAsset[i] = asset; break; } } } else if (stats.assetsByChunkName[chunkName].endsWith('.js')) { this.chunkNameToJsAsset[chunkName] = stats.assetsByChunkName[chunkName]; this.chunkNameToJsAsset[i] = stats.assetsByChunkName[chunkName]; } }); // iterate on all bundled webpack modules stats.modules.forEach(mod => { let srcFilePath = mod.name; // do not process non js related modules if (!this.srcExtsRegexp.test(srcFilePath)) { return; } // do not process modules unrelated to a source file if (!srcFilePath.startsWith('./')) { return; } // do not process modules in the exclusion list for (let toExclude of this.exclude) { if (srcFilePath.startsWith(toExclude)) { return; } } // remove webpack loader call if any let loaderEndPos = srcFilePath.indexOf('!'); if (loaderEndPos !== -1) { srcFilePath = srcFilePath.slice(loaderEndPos + 1); } // iterate on all chunks containing the module mod.chunks.forEach(chunk => { let chunkJsAsset = stats.publicPath + this.chunkNameToJsAsset[chunk]; // init the chunk to source files mapping if needed if (!this.chunkJsAssetToSrcFiles.hasOwnProperty(chunkJsAsset)) { this.chunkJsAssetToSrcFiles[chunkJsAsset] = []; } // check if the source file needs to be replaces if (this.options['srcReplace'] && this.options['srcReplace'].hasOwnProperty(srcFilePath)) { srcFilePath = this.options['srcReplace'][srcFilePath]; } // init source file metadata let srcFileData = {'id': this.cleanupPath(srcFilePath)}; // find and parse the corresponding package.json file let packageJsonPath; let nodeModule = srcFilePath.startsWith('./node_modules/'); if (nodeModule) { packageJsonPath = this.findPackageJsonPath(srcFilePath); } else { packageJsonPath = './package.json'; } let packageJson = this.parsePackageJson(packageJsonPath); // extract license information, overriding it if needed let licenseOverridden = false; let licenseFilePath; if (this.options['licenseOverride']) { for (let srcFilePrefixKey of Object.keys(this.options['licenseOverride'])) { let srcFilePrefix = srcFilePrefixKey; if (!srcFilePrefixKey.startsWith('.')) { srcFilePrefix = './' + path.join('node_modules', srcFilePrefixKey); } if (srcFilePath.startsWith(srcFilePrefix)) { let spdxLicenseExpression = this.options['licenseOverride'][srcFilePrefixKey]['spdxLicenseExpression']; licenseFilePath = this.options['licenseOverride'][srcFilePrefixKey]['licenseFilePath']; let parsedSpdxLicenses = this.parseSpdxLicenseExpression(spdxLicenseExpression, `file ${srcFilePath}`); srcFileData['licenses'] = this.spdxToWebLabelsLicenses(parsedSpdxLicenses); licenseOverridden = true; break; } } } if (!licenseOverridden) { srcFileData['licenses'] = this.extractLicenseInformation(packageJson); let licenseDir = path.join(...packageJsonPath.split('/').slice(0, -1)); licenseFilePath = this.findLicenseFile(licenseDir); } // copy original license file and get its url let licenseCopyUrl = this.copyLicenseFile(licenseFilePath); srcFileData['licenses'].forEach(license => { license['copy_url'] = licenseCopyUrl; }); // generate url for downloading non-minified source code srcFileData['src_url'] = stats.publicPath + path.join(this.weblabelsDirName, srcFileData['id']); // add source file metadata to the webpack chunk this.chunkJsAssetToSrcFiles[chunkJsAsset].push(srcFileData); // copy non-minified source to output folder this.copyFileToOutputPath(srcFilePath); }); }); // process additional scripts if needed if (this.options['additionalScripts']) { for (let script of Object.keys(this.options['additionalScripts'])) { let scriptFilesData = this.options['additionalScripts'][script]; - if (!script.startsWith('http:') && !script.startsWith('https:')) { + if (script.indexOf('://') === -1) { script = stats.publicPath + script; } this.chunkJsAssetToSrcFiles[script] = []; for (let scriptSrc of scriptFilesData) { let scriptSrcData = {'id': scriptSrc['id']}; let licenceFilePath = scriptSrc['licenseFilePath']; let parsedSpdxLicenses = this.parseSpdxLicenseExpression(scriptSrc['spdxLicenseExpression'], `file ${scriptSrc['path']}`); scriptSrcData['licenses'] = this.spdxToWebLabelsLicenses(parsedSpdxLicenses); let licenseCopyUrl = this.copyLicenseFile(licenceFilePath); scriptSrcData['licenses'].forEach(license => { license['copy_url'] = licenseCopyUrl; }); - if (!scriptSrc['path'].startsWith('http:') && !scriptSrc['path'].startsWith('https:')) { + if (scriptSrc['path'].indexOf('://') === -1) { scriptSrcData['src_url'] = stats.publicPath + path.join(this.weblabelsDirName, scriptSrc['id']); } else { scriptSrcData['src_url'] = scriptSrc['path']; } this.chunkJsAssetToSrcFiles[script].push(scriptSrcData); this.copyFileToOutputPath(scriptSrc['path']); } } } if (this.outputType === 'json') { // generate the jslicenses.json file let weblabelsData = JSON.stringify(this.chunkJsAssetToSrcFiles); let weblabelsJsonFile = path.join(this.weblabelsOutputDir, 'jslicenses.json'); fs.writeFileSync(weblabelsJsonFile, weblabelsData); } else { // generate the jslicenses.html file let weblabelsPageFile = path.join(this.weblabelsOutputDir, 'jslicenses.html'); ejs.renderFile(path.join(__dirname, 'jslicenses.ejs'), {'jslicenses_data': this.chunkJsAssetToSrcFiles}, {'rmWhitespace': true}, (e, str) => { fs.writeFileSync(weblabelsPageFile, str); }); } }); } cleanupPath(moduleFilePath) { return moduleFilePath.replace(/^[./]*node_modules\//, '').replace(/^.\//, ''); } findPackageJsonPath(srcFilePath) { let pathSplit = srcFilePath.split('/'); let packageJsonPath; for (let i = 3; i < pathSplit.length; ++i) { packageJsonPath = path.join(...pathSplit.slice(0, i), 'package.json'); if (fs.existsSync(packageJsonPath)) { break; } } return packageJsonPath; } findLicenseFile(packageJsonDir) { if (!this.packageLicenseFile.hasOwnProperty(packageJsonDir)) { let foundLicenseFile; fs.readdirSync(packageJsonDir).forEach(file => { if (foundLicenseFile) { return; } if (file.toLowerCase().startsWith('license')) { foundLicenseFile = path.join(packageJsonDir, file); } }); this.packageLicenseFile[packageJsonDir] = foundLicenseFile; } return this.packageLicenseFile[packageJsonDir]; } copyLicenseFile(licenseFilePath) { let licenseCopyPath = ''; if (licenseFilePath && fs.existsSync(licenseFilePath)) { let ext = ''; // add a .txt extension in order to serve license file with text/plain // content type to client browsers if (licenseFilePath.toLowerCase().indexOf('license.') === -1) { ext = '.txt'; } this.copyFileToOutputPath(licenseFilePath, ext); licenseFilePath = this.cleanupPath(licenseFilePath); licenseCopyPath = this.stats.publicPath + path.join(this.weblabelsDirName, licenseFilePath + ext); } return licenseCopyPath; } parsePackageJson(packageJsonPath) { if (!this.packageJsonCache.hasOwnProperty(packageJsonPath)) { let packageJsonStr = fs.readFileSync(packageJsonPath).toString('utf8'); this.packageJsonCache[packageJsonPath] = JSON.parse(packageJsonStr); } return this.packageJsonCache[packageJsonPath]; } parseSpdxLicenseExpression(spdxLicenseExpression, context) { let parsedLicense; try { parsedLicense = spdxParse(spdxLicenseExpression); if (spdxLicenseExpression.indexOf('AND') !== -1) { this.logger.warn(`The SPDX license expression '${spdxLicenseExpression}' associated to ${context} ` + 'contains an AND operator, this is currently not properly handled and erroneous ' + 'licenses information may be provided to LibreJS'); } } catch (e) { this.logger.warn(`Unable to parse the SPDX license expression '${spdxLicenseExpression}' associated to ${context}.`); this.logger.warn('Some generated JavaScript assets may be blocked by LibreJS due to missing license information.'); parsedLicense = {'license': spdxLicenseExpression}; } return parsedLicense; } spdxToWebLabelsLicense(spdxLicenceId) { for (let i = 0; i < spdxLicensesMapping.length; ++i) { if (spdxLicensesMapping[i]['spdx_ids'].indexOf(spdxLicenceId) !== -1) { let licenseData = Object.assign({}, spdxLicensesMapping[i]); delete licenseData['spdx_ids']; delete licenseData['magnet_link']; licenseData['copy_url'] = ''; return licenseData; } } this.logger.warn(`Unable to associate the SPDX license identifier '${spdxLicenceId}' to a LibreJS supported license.`); this.logger.warn('Some generated JavaScript assets may be blocked by LibreJS due to missing license information.'); return { 'name': spdxLicenceId, 'url': '', 'copy_url': '' }; } spdxToWebLabelsLicenses(spdxLicenses) { // This method simply extracts all referenced licenses in the SPDX expression // regardless of their combinations. // TODO: Handle licenses combination properly once LibreJS has a spec for it. let ret = []; if (spdxLicenses.hasOwnProperty('license')) { ret.push(this.spdxToWebLabelsLicense(spdxLicenses['license'])); } else if (spdxLicenses.hasOwnProperty('left')) { if (spdxLicenses['left'].hasOwnProperty('license')) { let licenseData = this.spdxToWebLabelsLicense(spdxLicenses['left']['license']); ret.push(licenseData); } else { ret = ret.concat(this.spdxToWebLabelsLicenses(spdxLicenses['left'])); } ret = ret.concat(this.spdxToWebLabelsLicenses(spdxLicenses['right'])); } return ret; } extractLicenseInformation(packageJson) { let spdxLicenseExpression; if (packageJson.hasOwnProperty('license')) { spdxLicenseExpression = packageJson['license']; } else if (packageJson.hasOwnProperty('licenses')) { // for node packages using deprecated licenses property let licenses = packageJson['licenses']; if (Array.isArray(licenses)) { let l = []; licenses.forEach(license => { l.push(license['type']); }); spdxLicenseExpression = l.join(' OR '); } else { spdxLicenseExpression = licenses['type']; } } let parsedSpdxLicenses = this.parseSpdxLicenseExpression(spdxLicenseExpression, `module ${packageJson['name']}`); return this.spdxToWebLabelsLicenses(parsedSpdxLicenses); } copyFileToOutputPath(srcFilePath, ext = '') { - if (this.copiedFiles.has(srcFilePath) || - srcFilePath.startsWith('http:') || - srcFilePath.startsWith('https:')) { + if (this.copiedFiles.has(srcFilePath) || srcFilePath.indexOf('://') !== -1) { return; } let destPath = this.cleanupPath(srcFilePath); let destDir = path.join(this.weblabelsOutputDir, ...destPath.split('/').slice(0, -1)); this.recursiveMkdir(destDir); destPath = path.join(this.weblabelsOutputDir, destPath + ext); fs.copyFileSync(srcFilePath, destPath); this.copiedFiles.add(srcFilePath); } recursiveMkdir(destPath) { let destPathSplit = destPath.split('/'); for (let i = 1; i < destPathSplit.length; ++i) { let currentPath = path.join('/', ...destPathSplit.slice(0, i + 1)); if (!fs.existsSync(currentPath)) { fs.mkdirSync(currentPath); } } } }; module.exports = GenerateWebLabelsPlugin; diff --git a/swh/web/assets/config/webpack.config.development.js b/swh/web/assets/config/webpack.config.development.js index ed845528..eb3cd4aa 100644 --- a/swh/web/assets/config/webpack.config.development.js +++ b/swh/web/assets/config/webpack.config.development.js @@ -1,403 +1,407 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // webpack configuration for compiling static assets in development mode // import required node modules and webpack plugins const fs = require('fs'); const path = require('path'); const webpack = require('webpack'); const BundleTracker = require('webpack-bundle-tracker'); const RobotstxtPlugin = require('robotstxt-webpack-plugin'); const CleanWebpackPlugin = require('clean-webpack-plugin'); const MiniCssExtractPlugin = require('mini-css-extract-plugin'); const FixSwhSourceMapsPlugin = require('./webpack-plugins/fix-swh-source-maps-webpack-plugin'); const CopyWebpackPlugin = require('copy-webpack-plugin'); const GenerateWebLabelsPlugin = require('./webpack-plugins/generate-weblabels-webpack-plugin'); +const loadedMathJaxJsFiles = require('./mathjax-js-files'); // are we running webpack-dev-server ? const isDevServer = process.argv.find(v => v.includes('webpack-dev-server')); // webpack-dev-server configuration const devServerPort = 3000; const devServerPublicPath = 'http://localhost:' + devServerPort + '/static/'; // set publicPath according if we are using webpack-dev-server to serve // our assets or not const publicPath = isDevServer ? devServerPublicPath : '/static/'; const nodeModules = path.resolve(__dirname, '../../../../node_modules/'); // collect all bundles we want to produce with webpack var bundles = {}; const bundlesDir = path.join(__dirname, '../src/bundles'); fs.readdirSync(bundlesDir).forEach(file => { bundles[file] = ['bundles/' + file + '/index.js']; }); // common loaders for css related assets (css, sass, less) let cssLoaders = [ MiniCssExtractPlugin.loader, { loader: 'cache-loader' }, { loader: 'css-loader', options: { sourceMap: !isDevServer } }, { loader: 'postcss-loader', options: { ident: 'postcss', sourceMap: !isDevServer, plugins: [ // lint swh-web stylesheets require('stylelint')({ 'config': { 'extends': 'stylelint-config-standard', 'rules': { 'indentation': 4, 'font-family-no-missing-generic-family-keyword': null, 'no-descending-specificity': null }, 'ignoreFiles': ['node_modules/**/*.css', 'swh/web/assets/src/thirdparty/**/*.css'] } }), // automatically add vendor prefixes to css rules require('autoprefixer')(), require('postcss-normalize')() ] } } ]; // webpack development configuration module.exports = { // use caching to speedup incremental builds cache: true, // set mode to development mode: 'development', // use eval source maps when using webpack-dev-server for quick debugging, // otherwise generate source map files (more expensive) devtool: isDevServer ? 'eval' : 'source-map', // webpack-dev-server configuration devServer: { clientLogLevel: 'warning', port: devServerPort, publicPath: devServerPublicPath, // enable to serve static assets not managed by webpack contentBase: path.resolve('./swh/web/'), // we do not use hot reloading here (as a framework like React needs to be used in order to fully benefit from that feature) // and prefere to fully reload the frontend application in the browser instead hot: false, inline: true, historyApiFallback: true, headers: { 'Access-Control-Allow-Origin': '*' }, compress: true, stats: { colors: true }, overlay: { warnings: true, errors: true } }, // set entries to the bundles we want to produce entry: bundles, // assets output configuration output: { path: path.resolve('./swh/web/static/'), filename: 'js/[name].[chunkhash].js', chunkFilename: 'js/[name].[chunkhash].js', publicPath: publicPath, // each bundle will be compiled as a umd module with its own namespace // in order to easily use them in django templates library: ['swh', '[name]'], libraryTarget: 'umd' }, // module resolving configuration resolve: { // alias pdfjs to its minified version alias: { 'pdfjs-dist': 'pdfjs-dist/build/pdf.min.js' }, // configure base paths for resolving modules with webpack modules: [ 'node_modules', path.resolve(__dirname, '../src') ] }, // module import configuration module: { rules: [ { // Preprocess all js files with eslint for consistent code style // and avoid bad js development practices. enforce: 'pre', test: /\.js$/, exclude: /node_modules/, use: [{ loader: 'eslint-loader', options: { configFile: path.join(__dirname, '.eslintrc'), ignorePath: path.join(__dirname, '.eslintignore'), emitWarning: true } }] }, { // Use babel-loader in order to use es6 syntax in js files // but also advanced js features like async/await syntax. // All code get transpiled to es5 in order to be executed // in a large majority of browsers. test: /\.js$/, exclude: /node_modules/, use: [ { loader: 'cache-loader' }, { loader: 'babel-loader', options: { presets: [ // use env babel presets to benefit from es6 syntax ['@babel/preset-env', { // Do not transform es6 module syntax to another module type // in order to benefit from dead code elimination (aka tree shaking) // when running webpack in production mode 'loose': true, 'modules': false }] ], plugins: [ // use babel transform-runtime plugin in order to use aync/await syntax ['@babel/plugin-transform-runtime', { 'corejs': 2, 'regenerator': true }], // use other babel plugins to benefit from advanced js features (es2017) '@babel/plugin-syntax-dynamic-import' ] } }] }, // expose jquery to the global context as $ and jQuery when importing it { test: require.resolve('jquery'), use: [{ loader: 'expose-loader', options: 'jQuery' }, { loader: 'expose-loader', options: '$' }] }, // expose highlightjs to the global context as hljs when importing it { test: require.resolve('highlight.js'), use: [{ loader: 'expose-loader', options: 'hljs' }] }, { test: require.resolve('js-cookie'), use: [{ loader: 'expose-loader', options: 'Cookies' }] }, // css import configuration: // - first process it with postcss // - then extract it to a dedicated file associated to each bundle { test: /\.css$/, use: cssLoaders }, // sass import configuration: // - generate css with sass-loader // - process it with postcss // - then extract it to a dedicated file associated to each bundle { test: /\.scss$/, use: cssLoaders.concat([ { loader: 'sass-loader', options: { sourceMap: !isDevServer } } ]) }, // less import configuration: // - generate css with less-loader // - process it with postcss // - then extract it to a dedicated file associated to each bundle { test: /\.less$/, use: cssLoaders.concat([ { loader: 'less-loader', options: { sourceMap: !isDevServer } } ]) }, // web fonts import configuration { test: /\.(woff|woff2)(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] }, { test: /\.ttf(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] }, { test: /\.eot(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] }, { test: /\.svg(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] }, { test: /\.otf(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] } ], // tell webpack to not parse minified pdfjs file to speedup build process noParse: [path.resolve(nodeModules, 'pdfjs-dist/build/pdf.min.js')] }, // webpack plugins plugins: [ // cleanup previously generated assets new CleanWebpackPlugin({ cleanOnceBeforeBuildPatterns: ['**/*', '!xml', '!xml/*', '!img', '!img/*', '!img/logos', '!img/logos/*', '!img/icons', '!img/icons/*'] }), // needed in order to use django_webpack_loader new BundleTracker({ filename: './swh/web/static/webpack-stats.json' }), // for generating the robots.txt file new RobotstxtPlugin({ policy: [{ userAgent: '*', disallow: '/api/' }] }), // for extracting all stylesheets in separate css files new MiniCssExtractPlugin({ filename: 'css/[name].[chunkhash].css', chunkFilename: 'css/[name].[chunkhash].css' }), // fix generated asset sourcemaps to workaround a Firefox issue new FixSwhSourceMapsPlugin(), // define some global variables accessible from js code new webpack.DefinePlugin({ __STATIC__: JSON.stringify(publicPath) }), // needed in order to use bootstrap 4.x new webpack.ProvidePlugin({ Popper: ['popper.js', 'default'], Alert: 'exports-loader?Alert!bootstrap/js/dist/alert', Button: 'exports-loader?Button!bootstrap/js/dist/button', Carousel: 'exports-loader?Carousel!bootstrap/js/dist/carousel', Collapse: 'exports-loader?Collapse!bootstrap/js/dist/collapse', Dropdown: 'exports-loader?Dropdown!bootstrap/js/dist/dropdown', Modal: 'exports-loader?Modal!bootstrap/js/dist/modal', Popover: 'exports-loader?Popover!bootstrap/js/dist/popover', Scrollspy: 'exports-loader?Scrollspy!bootstrap/js/dist/scrollspy', Tab: 'exports-loader?Tab!bootstrap/js/dist/tab', Tooltip: 'exports-loader?Tooltip!bootstrap/js/dist/tooltip', Util: 'exports-loader?Util!bootstrap/js/dist/util' }), // needed in order to use pdf.js new webpack.IgnorePlugin(/^\.\/pdf.worker.js$/), new CopyWebpackPlugin([{ from: path.resolve(nodeModules, 'pdfjs-dist/build/pdf.worker.min.js'), to: path.resolve(__dirname, '../../static/js/') }]), new GenerateWebLabelsPlugin({ outputType: 'json', exclude: ['mini-css-extract-plugin', 'bootstrap-loader'], srcReplace: { './node_modules/pdfjs-dist/build/pdf.min.js': './node_modules/pdfjs-dist/build/pdf.js', './node_modules/admin-lte/dist/js/adminlte.min.js': './node_modules/admin-lte/dist/js/adminlte.js' }, licenseOverride: { './swh/web/assets/src/thirdparty/jquery.tabSlideOut/jquery.tabSlideOut.js': { 'spdxLicenseExpression': 'GPL-3.0', 'licenseFilePath': './swh/web/assets/src/thirdparty/jquery.tabSlideOut/LICENSE' } }, - additionalScripts: { - 'js/pdf.worker.min.js': [ - { - 'id': 'pdfjs-dist/build/pdf.worker.js', - 'path': './node_modules/pdfjs-dist/build/pdf.worker.js', - 'spdxLicenseExpression': 'Apache-2.0', - 'licenseFilePath': './node_modules/pdfjs-dist/LICENSE' + additionalScripts: Object.assign( + { + 'js/pdf.worker.min.js': [ + { + 'id': 'pdfjs-dist/build/pdf.worker.js', + 'path': './node_modules/pdfjs-dist/build/pdf.worker.js', + 'spdxLicenseExpression': 'Apache-2.0', + 'licenseFilePath': './node_modules/pdfjs-dist/LICENSE' - } - ] - } + } + ] + }, + loadedMathJaxJsFiles + ) }) ], // webpack optimizations optimization: { // ensure the vendors bundle gets emitted in a single chunk splitChunks: { cacheGroups: { vendors: { test: 'vendors', chunks: 'all', name: 'vendors', enforce: true } } } }, // disable webpack warnings about bundle sizes performance: { hints: false } }; diff --git a/swh/web/assets/src/bundles/webapp/index.js b/swh/web/assets/src/bundles/webapp/index.js index 4bec3270..686a4a29 100644 --- a/swh/web/assets/src/bundles/webapp/index.js +++ b/swh/web/assets/src/bundles/webapp/index.js @@ -1,24 +1,25 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // webapp entrypoint bundle centralizing global custom stylesheets // and utility js modules used in all swh-web applications // explicitly import the vendors bundle import '../vendors'; // global swh-web custom stylesheets import './webapp.css'; import './breadcrumbs.css'; export * from './webapp-utils'; // utility js modules export * from './code-highlighting'; export * from './readme-rendering'; export * from './pdf-rendering'; +export * from './notebook-rendering'; export * from './xss-filtering'; diff --git a/swh/web/assets/src/bundles/webapp/notebook-rendering.js b/swh/web/assets/src/bundles/webapp/notebook-rendering.js new file mode 100644 index 00000000..300ce38e --- /dev/null +++ b/swh/web/assets/src/bundles/webapp/notebook-rendering.js @@ -0,0 +1,171 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +import 'script-loader!notebookjs'; +import AnsiUp from 'ansi_up'; +import './notebook.css'; + +const ansiup = new AnsiUp(); +ansiup.escape_for_html = false; + +function escapeHTML(text) { + text = text.replace(//g, '>'); + return text; +} + +function unescapeHTML(text) { + text = text.replace(/</g, '<'); + text = text.replace(/>/g, '>'); + return text; +} + +function escapeLaTeX(text) { + + let blockMath = /\$\$(.+?)\$\$|\\\\\[(.+?)\\\\\]/msg; + let inlineMath = /\$(.+?)\$|\\\\\((.+?)\\\\\)/g; + let latexEnvironment = /\\begin\{([a-z]*\*?)\}(.+?)\\end\{\1\}/msg; + + let mathTextFound = []; + let bm; + while ((bm = blockMath.exec(text)) !== null) { + mathTextFound.push(bm[1]); + } + + let im; + while ((im = inlineMath.exec(text)) !== null) { + mathTextFound.push(im[1]); + } + + let le; + while ((le = latexEnvironment.exec(text)) !== null) { + mathTextFound.push(le[1]); + } + + for (let mathText of mathTextFound) { + // showdown will remove line breaks in LaTex array and + // some escaping sequences when converting md to html. + // So we use the following escaping hacks to keep them in the html + // output and avoid MathJax typesetting errors. + let escapedText = mathText.replace('\\\\', '\\\\\\\\'); + for (let specialLaTexChar of ['{', '}', '#', '%', '&', '_']) { + escapedText = escapedText.replace(new RegExp(`\\\\${specialLaTexChar}`, 'g'), + `\\\\${specialLaTexChar}`); + } + + // some html escaping is also needed + escapedText = escapeHTML(escapedText); + + // hack to prevent showdown to replace _ characters + // by html em tags as it will break some math typesetting + // (setting the literalMidWordUnderscores option is not + // enough as iy only works for _ characters contained in words) + escapedText = escapedText.replace(/_/g, '{@}underscore{@}'); + + if (mathText !== escapedText) { + text = text.replace(mathText, escapedText); + } + } + + return text; +} + +export async function renderNotebook(nbJsonUrl, domElt) { + + let showdown = await import(/* webpackChunkName: "showdown" */ 'utils/showdown'); + + await import(/* webpackChunkName: "highlightjs" */ 'utils/highlightjs'); + + function renderMarkdown(text) { + let converter = new showdown.Converter({ + tables: true, + simplifiedAutoLink: true, + rawHeaderId: true, + literalMidWordUnderscores: true + }); + + // some LaTeX escaping is required to get correct math typesetting + text = escapeLaTeX(text); + + // render markdown + let rendered = converter.makeHtml(text); + + // restore underscores in rendered HTML (see escapeLaTeX function) + rendered = rendered.replace(/{@}underscore{@}/g, '_'); + + return rendered; + } + + function highlightCode(text, preElt, codeElt, lang) { + // no need to unescape text processed by ansiup + if (text.indexOf(' 1MB - 'content_display_max_size': ('int', 1024 * 1024), + 'content_display_max_size': ('int', 5 * 1024 * 1024), 'snapshot_content_max_size': ('int', 1000), 'throttling': ('dict', { 'cache_uri': None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None 'scopes': { 'swh_api': { 'limiter_rate': { 'default': '120/h' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_vault_cooking': { 'limiter_rate': { 'default': '120/h', 'GET': '60/m' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_save_origin': { 'limiter_rate': { 'default': '120/h', 'POST': '10/h' }, 'exempted_networks': ['127.0.0.0/8'] } } }), 'vault': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5005/', } }), 'scheduler': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5008/' } }), 'grecaptcha': ('dict', { 'activated': True, 'validation_url': 'https://www.google.com/recaptcha/api/siteverify', 'site_key': '', 'private_key': '' }), 'development_db': ('string', os.path.join(SETTINGS_DIR, 'db.sqlite3')), 'production_db': ('string', '/var/lib/swh/web.sqlite3'), 'deposit': ('dict', { 'private_api_url': 'https://deposit.softwareheritage.org/1/private/', 'private_api_user': 'swhworker', 'private_api_password': '' }), 'coverage_count_origins': ('bool', False) } swhweb_config = {} def get_config(config_file='web/web'): """Read the configuration file `config_file`. If an environment variable SWH_CONFIG_FILENAME is defined, this takes precedence over the config_file parameter. In any case, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration. """ if not swhweb_config: config_filename = os.environ.get('SWH_CONFIG_FILENAME') if config_filename: config_file = config_filename cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) config.prepare_folders(swhweb_config, 'log_dir') swhweb_config['storage'] = get_storage(**swhweb_config['storage']) swhweb_config['vault'] = get_vault(**swhweb_config['vault']) swhweb_config['indexer_storage'] = \ get_indexer_storage(**swhweb_config['indexer_storage']) swhweb_config['scheduler'] = get_scheduler( **swhweb_config['scheduler']) return swhweb_config def storage(): """Return the current application's storage. """ return get_config()['storage'] def vault(): """Return the current application's vault. """ return get_config()['vault'] def indexer_storage(): """Return the current application's indexer storage. """ return get_config()['indexer_storage'] def scheduler(): """Return the current application's scheduler. """ return get_config()['scheduler'] diff --git a/swh/web/templates/includes/content-display.html b/swh/web/templates/includes/content-display.html index e1334fcf..5c57711a 100644 --- a/swh/web/templates/includes/content-display.html +++ b/swh/web/templates/includes/content-display.html @@ -1,54 +1,59 @@ {% comment %} Copyright (C) 2017-2018 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% if snapshot_context and snapshot_context.is_empty %} {% include "includes/empty-snapshot.html" %} {% else %}
{% if swh_object_metadata.filename %}
{{ swh_object_metadata.filename }}
{% endif %}
{% if content_size > max_content_size %} Content is too large to be displayed (size is greater than {{ max_content_size|filesizeformat }}). {% elif "inode/x-empty" == mimetype %} File is empty + {% elif swh_object_metadata.filename and swh_object_metadata.filename|default:""|slice:"-5:" == "ipynb" %} +
+
{% elif "text/" in mimetype %} 
{{ content }}
{% elif "image/" in mimetype and content %} {% elif "application/pdf" == mimetype %}
Page: /
{% elif content %} Content with mime type {{ mimetype }} can not be displayed. {% else %} {% include "includes/http-error.html" %} {% endif %}
{% endif %} diff --git a/swh/web/tests/browse/views/test_content.py b/swh/web/tests/browse/views/test_content.py index 3c3766d6..435c64f9 100644 --- a/swh/web/tests/browse/views/test_content.py +++ b/swh/web/tests/browse/views/test_content.py @@ -1,362 +1,362 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch from django.utils.html import escape from hypothesis import given from swh.web.browse.utils import ( get_mimetype_and_encoding_for_content, prepare_content_for_display, _reencode_content ) from swh.web.common.exc import NotFoundExc from swh.web.common.utils import reverse, get_swh_persistent_id from swh.web.common.utils import gen_path_info from swh.web.tests.strategies import ( content, content_text_non_utf8, content_text_no_highlight, content_image_type, content_text, invalid_sha1, unknown_content ) from swh.web.tests.testcase import WebTestCase class SwhBrowseContentTest(WebTestCase): @given(content()) def test_content_view_text(self, content): sha1_git = content['sha1_git'] url = reverse('browse-content', url_args={'query_string': content['sha1']}, query_params={'path': content['path']}) url_raw = reverse('browse-content-raw', url_args={'query_string': content['sha1']}) resp = self.client.get(url) content_display = self._process_content_for_display(content) mimetype = content_display['mimetype'] self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/content.html') if mimetype.startswith('text/'): self.assertContains(resp, '' % content_display['language']) self.assertContains(resp, escape(content_display['content_data'])) self.assertContains(resp, url_raw) swh_cnt_id = get_swh_persistent_id('content', sha1_git) swh_cnt_id_url = reverse('browse-swh-id', url_args={'swh_id': swh_cnt_id}) self.assertContains(resp, swh_cnt_id) self.assertContains(resp, swh_cnt_id_url) @given(content_text_no_highlight()) def test_content_view_text_no_highlight(self, content): sha1_git = content['sha1_git'] url = reverse('browse-content', url_args={'query_string': content['sha1']}) url_raw = reverse('browse-content-raw', url_args={'query_string': content['sha1']}) resp = self.client.get(url) content_display = self._process_content_for_display(content) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/content.html') self.assertContains(resp, '') self.assertContains(resp, escape(content_display['content_data'])) # noqa self.assertContains(resp, url_raw) swh_cnt_id = get_swh_persistent_id('content', sha1_git) swh_cnt_id_url = reverse('browse-swh-id', url_args={'swh_id': swh_cnt_id}) self.assertContains(resp, swh_cnt_id) self.assertContains(resp, swh_cnt_id_url) @given(content_text_non_utf8()) def test_content_view_no_utf8_text(self, content): sha1_git = content['sha1_git'] url = reverse('browse-content', url_args={'query_string': content['sha1']}) resp = self.client.get(url) content_display = self._process_content_for_display(content) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/content.html') swh_cnt_id = get_swh_persistent_id('content', sha1_git) swh_cnt_id_url = reverse('browse-swh-id', url_args={'swh_id': swh_cnt_id}) self.assertContains(resp, swh_cnt_id_url) self.assertContains(resp, escape(content_display['content_data'])) @given(content_image_type()) def test_content_view_image(self, content): url = reverse('browse-content', url_args={'query_string': content['sha1']}) url_raw = reverse('browse-content-raw', url_args={'query_string': content['sha1']}) resp = self.client.get(url) content_display = self._process_content_for_display(content) mimetype = content_display['mimetype'] content_data = content_display['content_data'] self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/content.html') self.assertContains(resp, '' % (mimetype, content_data.decode('utf-8'))) self.assertContains(resp, url_raw) @given(content()) def test_content_view_with_path(self, content): path = content['path'] url = reverse('browse-content', url_args={'query_string': content['sha1']}, query_params={'path': path}) resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertTemplateUsed('browse/content.html') self.assertContains(resp, '