diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ swh/web/static/js/ swh/web/static/css/ swh/web/static/fonts/ +swh/web/static/weblabels/ .cache-loader/ build/ dist/ diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -65,9 +65,11 @@ "postcss-loader": "^3.0.0", "postcss-normalize": "^7.0.1", "resolve-url-loader": "^3.0.0", + "rimraf": "^2.6.3", "robotstxt-webpack-plugin": "^5.0.0", "sass-loader": "^7.1.0", "showdown": "^1.9.0", + "spdx-expression-parse": "^3.0.0", "style-loader": "^0.23.1", "stylelint": "^9.10.1", "stylelint-config-standard": "^18.2.0", diff --git a/swh/web/assets/config/.eslintrc b/swh/web/assets/config/.eslintrc --- a/swh/web/assets/config/.eslintrc +++ b/swh/web/assets/config/.eslintrc @@ -97,8 +97,8 @@ "CallExpression": { "arguments": "first" }, - "ArrayExpression": 1, - "ObjectExpression": 1, + "ArrayExpression": "first", + "ObjectExpression": "first", "ImportDeclaration": 1, "flatTernaryExpressions": false, "ignoreComments": false diff --git a/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js new file mode 100644 --- /dev/null +++ b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js @@ -0,0 +1,288 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +// This is a first draft of a plugin for Webpack >= 4 enabling to generate a WebLabels +// page to be used with the LibreJS Firefox plugin (https://www.gnu.org/software/librejs/). +// The purpose of the WebLabels page is to prevent the loading of the generated +// assets being blocked by the LibreJS Firefox plugin in client browsers. +// To do so, the page must contain the following information. For each JavaScript asset +// generated by webpack, all bundled source files in it need to be referenced along +// with their licenses but also a link to their non-minified source code. + +// The plugin works by processing the compilation statistics available after the whole +// webpack processing and currently does the following: +// +// - it copies all non-minified source files bundles into the generated JavaScript +// assets into a weblabels folder located into the webpack output folder. +// +// - it generates a weblabels.json file into a weblabels folder located into the +// webpack output folder. This JSON file should be used with an HTML template +// engine to generate the WebLabels page and it has the following structure: +// +// { +// "": [{ +// "id": "", +// "path": "" +// "license": { +// "name": "", +// "url": "" +// }, +// "static_url": "" +// }, { +// ... +// }], +// "": [ +// ... +// ], +// ... +// } +// +// So for each JavaScript asset generated by Webpack, the JSON file references +// all the source code files bundled in it along with the detected licenses +// and a link to download the non-minified source codes. +// For each referenced source file, its path corresponds to the relative +// path from the webpack project root folder, while its identifier corresponds +// to the path without the './' or './node_modules/' prefix. + +// The following options can be provided to the plugin: +// +// - exclude: An array of module name prefixes to exclude from the generated output. +// If the prefix does not start with './', it is considered as a node module. +// +// - srcReplace: An object mapping source files to replace in the generated output. +// This should be used when a node module points to its minified version +// by default. +// +// - licenseOverride: An object mapping source files to their corresponding SPDX licenses. +// This should be used when you have source files inside your webpack +// project source tree with licenses different from the one of your +// project. +// +// - additionalScripts: An object declaring additional js scripts loaded by your web +// application but not processed by webpack. It must have the +// following structure: +// +// { +// "": [{ +// "id": "", +// "path": "" +// "license": "" +// }, { +// ... +// }], +// ... +// } + +const fs = require('fs'); +const spdxParse = require('spdx-expression-parse'); +const rimraf = require('rimraf'); +const spdxLicensesMapping = require('./spdx-licenses-mapping'); + +class GenerateWebLabelsPlugin { + + constructor(opts) { + this.options = opts || {}; + // source file extension handled by webpack and compiled to js + this.srcExts = ['js', 'ts', 'coffee', 'lua']; + this.srcExtsRegexp = new RegExp('^.*.(' + this.srcExts.join('|') + ')$'); + this.chunkNameToJsAsset = {}; + this.chunkToSrcFiles = {}; + this.packageJsonCache = {}; + this.packageLicenseFile = {}; + this.exclude = []; + // populate module prefix patterns to exclude + if (Array.isArray(this.options['exclude'])) { + this.options['exclude'].forEach(toExclude => { + if (!toExclude.startsWith('.')) { + this.exclude.push('./node_modules/' + toExclude); + } else { + this.exclude.push(toExclude); + } + }); + } + } + + apply(compiler) { + compiler.hooks.done.tap('GenerateWebLabelsPlugin', statsObj => { + + // get the stats object in JSON format + let stats = statsObj.toJson(); + + // set output folder + this.weblabelsOutputDir = stats.outputPath + '/weblabels'; + + // cleanup previously copied source files + if (fs.existsSync(this.weblabelsOutputDir)) { + rimraf.sync(this.weblabelsOutputDir); + } + + // map each generated webpack chunk to its js asset + Object.keys(stats.assetsByChunkName).forEach((chunkName, i) => { + for (let asset of stats.assetsByChunkName[chunkName]) { + if (asset.endsWith('.js')) { + this.chunkNameToJsAsset[chunkName] = asset; + this.chunkNameToJsAsset[i] = asset; + break; + } + } + }); + + // iterate on all bundled webpack modules + stats.modules.forEach((mod, i) => { + if (this.srcExtsRegexp.test(mod.name)) { + // iterate on all chunks containing the module + mod.chunks.forEach(chunk => { + chunk = this.chunkNameToJsAsset[chunk]; + // special case for webpack modules + let srcFilePath = mod.name.replace('(webpack)', './node_modules/webpack'); + // do not process modules unrelated to a source file + if (!srcFilePath.startsWith('./')) { + return; + } + // do not process modules in the exclusion list + for (let toExclude of this.exclude) { + if (srcFilePath.startsWith(toExclude)) { + return; + } + } + // init the chunk to source files mapping if needed + if (!this.chunkToSrcFiles.hasOwnProperty(chunk)) { + this.chunkToSrcFiles[chunk] = []; + } + // check if the source file needs to be replaces + if (this.options['srcReplace'] && this.options['srcReplace'].hasOwnProperty(srcFilePath)) { + srcFilePath = this.options['srcReplace'][srcFilePath]; + } + + // init source file metadata + let srcFileData = {'id': srcFilePath.replace(/^.\/node_modules\//, '').replace(/^.\//, ''), + 'path': srcFilePath}; + + // find and parse the corresponding package.json file + let packageJsonPath; + let nodeModule = srcFilePath.startsWith('./node_modules/'); + if (nodeModule) { + packageJsonPath = this.findPackageJsonPath(srcFilePath); + } else { + packageJsonPath = './package.json'; + } + let packageJson = this.parsePackageJson(packageJsonPath); + + // extract license information, overriding it if needed + if (this.options['licenseOverride'] && this.options['licenseOverride'].hasOwnProperty(srcFilePath)) { + srcFileData['license'] = this.spdxToWebLabelsLicense(this.options['licenseOverride'][srcFilePath]); + } else { + srcFileData['license'] = this.extractLicenseInformation(packageJson); + } + // generate static url for downloading non-minified source code + srcFileData['static_url'] = '/weblabels/' + srcFileData['id']; + + // add source file metadata to the webpack chunk + this.chunkToSrcFiles[chunk].push(srcFileData); + // copy non-minified source to output folder + this.copyFileToOutputPath(srcFilePath); + }); + } + }); + + // process additional scripts if needed + if (this.options['additionalScripts']) { + for (let script of Object.keys(this.options['additionalScripts'])) { + this.chunkToSrcFiles[script] = []; + for (let scriptSrc of this.options['additionalScripts'][script]) { + scriptSrc['license'] = this.spdxToWebLabelsLicense(scriptSrc['license']); + scriptSrc['static_url'] = '/weblabels/' + scriptSrc['id']; + this.chunkToSrcFiles[script].push(scriptSrc); + this.copyFileToOutputPath(scriptSrc['path']); + } + } + } + + // generate the weblabels.json file + let weblabelsData = JSON.stringify(this.chunkToSrcFiles); + let weblabelsJsonFile = this.weblabelsOutputDir + '/' + 'weblabels.json'; + fs.writeFileSync(weblabelsJsonFile, weblabelsData); + }); + } + + findPackageJsonPath(srcFilePath) { + let pathSplit = srcFilePath.split('/'); + let packageJsonPath; + for (let i = 3; i < pathSplit.length; ++i) { + packageJsonPath = pathSplit.slice(0, i).join('/') + '/package.json'; + if (fs.existsSync(packageJsonPath)) { + break; + } + } + return packageJsonPath; + } + + parsePackageJson(packageJsonPath) { + if (!this.packageJsonCache.hasOwnProperty(packageJsonPath)) { + let packageJsonStr = fs.readFileSync(packageJsonPath).toString('utf8'); + this.packageJsonCache[packageJsonPath] = JSON.parse(packageJsonStr); + } + return this.packageJsonCache[packageJsonPath]; + } + + spdxToWebLabelsLicense(spdxLicenceId) { + for (let i = 0; i < spdxLicensesMapping.length; ++i) { + if (spdxLicensesMapping[i]['spdx_ids'].indexOf(spdxLicenceId) !== -1) { + let licenseData = Object.assign({}, spdxLicensesMapping[i]); + delete licenseData['spdx_ids']; + delete licenseData['magnet_link']; + return licenseData; + } + } + return undefined; + } + + extractLicenseInformation(packageJson) { + let licenseStr; + if (packageJson.hasOwnProperty('license')) { + licenseStr = packageJson['license']; + } else if (packageJson.hasOwnProperty('licenses')) { + let licenses = packageJson['licenses']; + if (Array.isArray(licenses)) { + let l = []; + licenses.forEach(license => { + l.push(license['type']); + }); + licenseStr = l.join(' 0R '); + } else { + licenseStr = licenses['type']; + } + } + let license = spdxParse(licenseStr)['license']; + let licenseData = this.spdxToWebLabelsLicense(license); + if (licenseData) { + return licenseData; + } else { + return {'name': license}; + } + } + + copyFileToOutputPath(srcFilePath) { + if (!fs.existsSync(this.weblabelsOutputDir)) { + fs.mkdirSync(this.weblabelsOutputDir); + } + let destPath = srcFilePath.replace(/^.\//, ''); + destPath = destPath.replace(/^node_modules\//, ''); + let destPathSplit = destPath.split('/'); + let currentPath = this.weblabelsOutputDir; + for (let i = 0; i < destPathSplit.length - 1; ++i) { + currentPath += '/' + destPathSplit[i]; + if (!fs.existsSync(currentPath)) { + fs.mkdirSync(currentPath); + } + } + fs.copyFileSync(srcFilePath, this.weblabelsOutputDir + '/' + destPath); + } + +}; + +module.exports = GenerateWebLabelsPlugin; diff --git a/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/spdx-licenses-mapping.js b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/spdx-licenses-mapping.js new file mode 100644 --- /dev/null +++ b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/spdx-licenses-mapping.js @@ -0,0 +1,135 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +module.exports = [ + { + 'name': 'Apache License, Version 2.0', + 'url': 'http://www.apache.org/licenses/LICENSE-2.0', + 'magnet_link': 'magnet:?xt=urn:btih:8e4f440f4c65981c5bf93c76d35135ba5064d8b7&dn=apache-2.0.txt', + 'spdx_ids': ['Apache-2.0'] + }, + { + 'name': 'Artistic License 2.0', + 'url': 'http://www.perlfoundation.org/artistic_license_2_0', + 'magnet_link': 'magnet:?xt=urn:btih:54fd2283f9dbdf29466d2df1a98bf8f65cafe314&dn=artistic-2.0.txt', + 'spdx_ids': ['Artistic-2.0'] + }, + { + 'name': 'Boost Software License', + 'url': 'http://www.boost.org/LICENSE_1_0.txt', + 'magnet_link': 'magnet:?xt=urn:btih:89a97c535628232f2f3888c2b7b8ffd4c078cec0&dn=Boost-1.0.txt', + 'spdx_ids': ['BSL-1.0'] + }, + { + 'name': 'BSD 3-Clause License', + 'url': 'http://opensource.org/licenses/BSD-3-Clause', + 'magnet_link': 'magnet:?xt=urn:btih:c80d50af7d3db9be66a4d0a86db0286e4fd33292&dn=bsd-3-clause.txt', + 'spdx_ids': ['BSD-3-Clause'] + }, + { + 'name': 'CPAL 1.0', + 'url': 'http://opensource.org/licenses/cpal_1.0', + 'magnet_link': 'magnet:?xt=urn:btih:84143bc45939fc8fa42921d619a95462c2031c5c&dn=cpal-1.0.txt', + 'spdx_ids': ['CPAL-1.0'] + }, + { + 'name': 'Creative Commons CC0 1.0 Universal', + 'url': 'http://creativecommons.org/publicdomain/zero/1.0/legalcode', + 'magnet_link': 'magnet:?xt=urn:btih:90dc5c0be029de84e523b9b3922520e79e0e6f08&dn=cc0.txt', + 'spdx_ids': ['CC0-1.0'] + }, + { + 'name': 'Eclipse Public License 1.0', + 'url': 'http://www.eclipse.org/legal/epl-v10.html', + 'magnet_link': 'magnet:?xt=urn:btih:4c6a2ad0018cd461e9b0fc44e1b340d2c1828b22&dn=epl-1.0.txt', + 'spdx_ids': ['EPL-1.0'] + }, + { + 'name': 'Expat License', + 'url': 'http://www.jclark.com/xml/copying.txt', + 'magnet_link': 'magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt', + 'spdx_ids': ['MIT'] + }, + { + 'name': 'FreeBSD License', + 'url': 'http://www.freebsd.org/copyright/freebsd-license.html', + 'magnet_link': 'magnet:?xt=urn:btih:87f119ba0b429ba17a44b4bffcab33165ebdacc0&dn=freebsd.txt', + 'spdx_ids': ['BSD-2-Clause-FreeBSD'] + }, + { + 'name': 'GNU General Public License (GPL) version 2', + 'url': 'http://www.gnu.org/licenses/gpl-2.0.html', + 'magnet_link': 'magnet:?xt=urn:btih:cf05388f2679ee054f2beb29a391d25f4e673ac3&dn=gpl-2.0.txt', + 'spdx_ids': ['GPL-2.0-only', 'GPL-2.0-or-later', 'GPL-2.0+', 'GPL-2.0'] + }, + { + 'name': 'GNU General Public License (GPL) version 3', + 'url': 'http://www.gnu.org/licenses/gpl-3.0.html', + 'magnet_link': 'magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt', + 'spdx_ids': ['GPL-3.0-only', 'GPL-3.0-or-later', 'GPL-3.0+', 'GPL-3.0'] + }, + { + 'name': 'GNU Lesser General Public License, version 2.1', + 'url': 'http://www.gnu.org/licenses/lgpl-2.1.html', + 'magnet_link': 'magnet:?xt=urn:btih:5de60da917303dbfad4f93fb1b985ced5a89eac2&dn=lgpl-2.1.txt', + 'spdx_ids': ['LGPL-2.1-only', 'LGPL-2.1-or-later', 'LGPL-2.1+', 'LGPL-2.1'] + }, + { + 'name': 'GNU Lesser General Public License, version 3', + 'url': 'http://www.gnu.org/licenses/lgpl-3.0.html', + 'magnet_link': 'magnet:?xt=urn:btih:0ef1b8170b3b615170ff270def6427c317705f85&dn=lgpl-3.0.txt', + 'spdx_ids': ['LGPL-3.0-only', 'LGPL-3.0-or-later', 'LGPL-3.0+', 'LGPL-3.0'] + }, + { + 'name': 'GNU Affero General Public License, version 3', + 'url': 'http://www.gnu.org/licenses/agpl-3.0.html', + 'magnet_link': 'magnet:?xt=urn:btih:0b31508aeb0634b347b8270c7bee4d411b5d4109&dn=agpl-3.0.txt', + 'spdx_ids': ['AGPL-3.0-only', 'AGPL-3.0-or-later', 'AGPL-3.0+', 'AGPL-3.0'] + }, + { + 'name': 'The ISC License', + 'url': 'https://www.isc.org/downloads/software-support-policy/isc-license/', + 'magnet_link': 'magnet:?xt=urn:btih:b8999bbaf509c08d127678643c515b9ab0836bae&dn=ISC.txt', + 'spdx_ids': ['ISC'] + }, + { + 'name': 'Mozilla Public License 2.0', + 'url': 'http://www.mozilla.org/MPL/2.0', + 'magnet_link': 'magnet:?xt=urn:btih:3877d6d54b3accd4bc32f8a48bf32ebc0901502a&dn=mpl-2.0.txt', + 'spdx_ids': ['MPL-2.0'] + }, + { + 'name': 'Universal Permissive License', + 'url': 'https://oss.oracle.com/licenses/upl/', + 'magnet_link': 'magnet:?xt=urn:btih:478974f4d41c3fa84c4befba25f283527fad107d&dn=upl-1.0.txt', + 'spdx_ids': ['UPL-1.0'] + }, + { + 'name': 'WTFPL', + 'url': 'http://www.wtfpl.net/txt/copying/', + 'magnet_link': 'magnet:?xt=urn:btih:723febf9f6185544f57f0660a41489c7d6b4931b&dn=wtfpl.txt', + 'spdx_ids': ['WTFPL'] + }, + { + 'name': 'Unlicense', + 'url': 'http://unlicense.org/UNLICENSE', + 'magnet_link': 'magnet:?xt=urn:btih:5ac446d35272cc2e4e85e4325b146d0b7ca8f50c&dn=unlicense.txt', + 'spdx_ids': ['Unlicense'] + }, + { + 'name': 'X11 License', + 'url': 'http://www.xfree86.org/3.3.6/COPYRIGHT2.html#3', + 'magnet_link': 'magnet:?xt=urn:btih:5305d91886084f776adcf57509a648432709a7c7&dn=x11.txt', + 'spdx_ids': ['X11'] + }, + { + 'name': 'XFree86 License', + 'url': 'http://www.xfree86.org/current/LICENSE4.html', + 'magnet_link': 'magnet:?xt=urn:btih:12f2ec9e8de2a3b0002a33d518d6010cc8ab2ae9&dn=xfree86.txt', + 'spdx_ids': ['XFree86-1.1'] + } +]; diff --git a/swh/web/assets/config/webpack.config.development.js b/swh/web/assets/config/webpack.config.development.js --- a/swh/web/assets/config/webpack.config.development.js +++ b/swh/web/assets/config/webpack.config.development.js @@ -17,6 +17,7 @@ const MiniCssExtractPlugin = require('mini-css-extract-plugin'); const RemoveSourceMapUrlPlugin = require('./webpack-plugins/remove-source-map-url-webpack-plugin'); const CopyWebpackPlugin = require('copy-webpack-plugin'); +const GenerateWebLabelsPlugin = require('./webpack-plugins/generate-weblabels-webpack-plugin'); // are we running webpack-dev-server ? const isDevServer = process.argv.find(v => v.includes('webpack-dev-server')); @@ -134,166 +135,167 @@ }, // module import configuration module: { - rules: [{ + rules: [ + { // Preprocess all js files with eslint for consistent code style // and avoid bad js development practices. - enforce: 'pre', - test: /\.js$/, - exclude: /node_modules/, - use: [{ - loader: 'eslint-loader', - options: { - configFile: path.join(__dirname, '.eslintrc'), - emitWarning: true - } - }] - }, - { + enforce: 'pre', + test: /\.js$/, + exclude: /node_modules/, + use: [{ + loader: 'eslint-loader', + options: { + configFile: path.join(__dirname, '.eslintrc'), + emitWarning: true + } + }] + }, + { // Use babel-loader in order to use es6 syntax in js files // but also advanced js features like async/await syntax. // All code get transpiled to es5 in order to be executed // in a large majority of browsers. - test: /\.js$/, - exclude: /node_modules/, - use: [ - { - loader: 'cache-loader' - }, - { - loader: 'babel-loader', - options: { - presets: [ - // use env babel presets to benefit from es6 syntax - ['@babel/preset-env', { - // Do not transform es6 module syntax to another module type - // in order to benefit from dead code elimination (aka tree shaking) - // when running webpack in production mode - 'loose': true, - 'modules': false - }] - ], - plugins: [ + test: /\.js$/, + exclude: /node_modules/, + use: [ + { + loader: 'cache-loader' + }, + { + loader: 'babel-loader', + options: { + presets: [ + // use env babel presets to benefit from es6 syntax + ['@babel/preset-env', { + // Do not transform es6 module syntax to another module type + // in order to benefit from dead code elimination (aka tree shaking) + // when running webpack in production mode + 'loose': true, + 'modules': false + }] + ], + plugins: [ // use babel transform-runtime plugin in order to use aync/await syntax - ['@babel/plugin-transform-runtime', { - 'corejs': 2, - 'regenerator': true - }], - // use other babel plugins to benefit from advanced js features (es2017) - '@babel/plugin-syntax-dynamic-import' - ] + ['@babel/plugin-transform-runtime', { + 'corejs': 2, + 'regenerator': true + }], + // use other babel plugins to benefit from advanced js features (es2017) + '@babel/plugin-syntax-dynamic-import' + ] + } + }] + }, + // expose jquery to the global context as $ and jQuery when importing it + { + test: require.resolve('jquery'), + use: [{ + loader: 'expose-loader', + options: 'jQuery' + }, { + loader: 'expose-loader', + options: '$' + }] + }, + // expose highlightjs to the global context as hljs when importing it + { + test: require.resolve('highlight.js'), + use: [{ + loader: 'expose-loader', + options: 'hljs' + }] + }, + { + test: require.resolve('js-cookie'), + use: [{ + loader: 'expose-loader', + options: 'Cookies' + }] + }, + // css import configuration: + // - first process it with postcss + // - then extract it to a dedicated file associated to each bundle + { + test: /\.css$/, + use: cssLoaders + }, + // sass import configuration: + // - generate css with sass-loader + // - process it with postcss + // - then extract it to a dedicated file associated to each bundle + { + test: /\.scss$/, + use: cssLoaders.concat([ + { + loader: 'sass-loader', + options: { + sourceMap: !isDevServer + } + } + ]) + }, + // less import configuration: + // - generate css with less-loader + // - process it with postcss + // - then extract it to a dedicated file associated to each bundle + { + test: /\.less$/, + use: cssLoaders.concat([ + { + loader: 'less-loader', + options: { + sourceMap: !isDevServer + } + } + ]) + }, + // web fonts import configuration + { + test: /\.(woff|woff2)(\?v=\d+\.\d+\.\d+)?$/, + use: [{ + loader: 'file-loader', + options: { + name: '[name].[ext]', + outputPath: 'fonts/' } }] - }, - // expose jquery to the global context as $ and jQuery when importing it - { - test: require.resolve('jquery'), - use: [{ - loader: 'expose-loader', - options: 'jQuery' }, { - loader: 'expose-loader', - options: '$' - }] - }, - // expose highlightjs to the global context as hljs when importing it - { - test: require.resolve('highlight.js'), - use: [{ - loader: 'expose-loader', - options: 'hljs' - }] - }, - { - test: require.resolve('js-cookie'), - use: [{ - loader: 'expose-loader', - options: 'Cookies' - }] - }, - // css import configuration: - // - first process it with postcss - // - then extract it to a dedicated file associated to each bundle - { - test: /\.css$/, - use: cssLoaders - }, - // sass import configuration: - // - generate css with sass-loader - // - process it with postcss - // - then extract it to a dedicated file associated to each bundle - { - test: /\.scss$/, - use: cssLoaders.concat([ - { - loader: 'sass-loader', + test: /\.ttf(\?v=\d+\.\d+\.\d+)?$/, + use: [{ + loader: 'file-loader', options: { - sourceMap: !isDevServer + name: '[name].[ext]', + outputPath: 'fonts/' } - } - ]) - }, - // less import configuration: - // - generate css with less-loader - // - process it with postcss - // - then extract it to a dedicated file associated to each bundle - { - test: /\.less$/, - use: cssLoaders.concat([ - { - loader: 'less-loader', + }] + }, { + test: /\.eot(\?v=\d+\.\d+\.\d+)?$/, + use: [{ + loader: 'file-loader', options: { - sourceMap: !isDevServer + name: '[name].[ext]', + outputPath: 'fonts/' } - } - ]) - }, - // web fonts import configuration - { - test: /\.(woff|woff2)(\?v=\d+\.\d+\.\d+)?$/, - use: [{ - loader: 'file-loader', - options: { - name: '[name].[ext]', - outputPath: 'fonts/' - } - }] - }, { - test: /\.ttf(\?v=\d+\.\d+\.\d+)?$/, - use: [{ - loader: 'file-loader', - options: { - name: '[name].[ext]', - outputPath: 'fonts/' - } - }] - }, { - test: /\.eot(\?v=\d+\.\d+\.\d+)?$/, - use: [{ - loader: 'file-loader', - options: { - name: '[name].[ext]', - outputPath: 'fonts/' - } - }] - }, { - test: /\.svg(\?v=\d+\.\d+\.\d+)?$/, - use: [{ - loader: 'file-loader', - options: { - name: '[name].[ext]', - outputPath: 'fonts/' - } - }] - }, { - test: /\.otf(\?v=\d+\.\d+\.\d+)?$/, - use: [{ - loader: 'file-loader', - options: { - name: '[name].[ext]', - outputPath: 'fonts/' - } - }] - } + }] + }, { + test: /\.svg(\?v=\d+\.\d+\.\d+)?$/, + use: [{ + loader: 'file-loader', + options: { + name: '[name].[ext]', + outputPath: 'fonts/' + } + }] + }, { + test: /\.otf(\?v=\d+\.\d+\.\d+)?$/, + use: [{ + loader: 'file-loader', + options: { + name: '[name].[ext]', + outputPath: 'fonts/' + } + }] + } ], // tell webpack to not parse minified pdfjs file to speedup build process noParse: [path.resolve(nodeModules, 'pdfjs-dist/build/pdf.min.js')] @@ -346,7 +348,30 @@ new CopyWebpackPlugin([{ from: path.resolve(nodeModules, 'pdfjs-dist/build/pdf.worker.min.js'), to: path.resolve(__dirname, '../../static/js/') - }]) + }]), + new GenerateWebLabelsPlugin({ + exclude: ['mini-css-extract-plugin', + 'bootstrap-loader'], + srcReplace: { + './node_modules/pdfjs-dist/build/pdf.min.js': + './node_modules/pdfjs-dist/build/pdf.js', + './node_modules/admin-lte/dist/js/adminlte.min.js': + './node_modules/admin-lte/dist/js/adminlte.js' + }, + licenseOverride: { + './swh/web/assets/src/utils/highlightjs-line-numbers.js': 'MIT', + './swh/web/assets/src/utils/jquery.tabSlideOut.js': 'GPL-3.0' + }, + additionalScripts: { + 'js/pdf.worker.min.js': [ + { + 'id': 'pdfjs-dist/build/pdf.worker.js', + 'path': './node_modules/pdfjs-dist/build/pdf.worker.js', + 'license': 'Apache-2.0' + } + ] + } + }) ], // webpack optimizations optimization: { diff --git a/swh/web/assets/config/webpack.config.production.js b/swh/web/assets/config/webpack.config.production.js --- a/swh/web/assets/config/webpack.config.production.js +++ b/swh/web/assets/config/webpack.config.production.js @@ -38,5 +38,8 @@ }) ]; +// prevent modules concatenation for generating weblabels +webpackProdConfig.optimization.concatenateModules = false; + // webpack production configuration module.exports = webpackProdConfig; diff --git a/swh/web/common/swh_templatetags.py b/swh/web/common/swh_templatetags.py --- a/swh/web/common/swh_templatetags.py +++ b/swh/web/common/swh_templatetags.py @@ -166,3 +166,17 @@ If the origin type is savable or not """ return origin_type in get_savable_origin_types() + + +@register.filter +def split(value, arg): + """Django template filter to split a string. + + Args: + value (str): the string to split + arg (str): the split separator + + Returns: + list: the splitted string parts + """ + return value.split(arg) diff --git a/swh/web/templates/coverage.html b/swh/web/templates/coverage.html --- a/swh/web/templates/coverage.html +++ b/swh/web/templates/coverage.html @@ -7,6 +7,7 @@ +{% load js_reverse %} {% load static %} {% load render_bundle from webpack_loader %} @@ -18,7 +19,31 @@ Software Heritage archive coverage {% render_bundle 'vendors' %} {% render_bundle 'webapp' %} - + + @@ -41,6 +66,7 @@ + JavaScript license information {% if count_origins %} + + + {% block header %}{% endblock %} @@ -30,6 +56,7 @@ + @@ -172,7 +199,8 @@ Terms of use: Archive access - API, - Contact. + Contact, + JavaScript license information
diff --git a/swh/web/urls.py b/swh/web/urls.py --- a/swh/web/urls.py +++ b/swh/web/urls.py @@ -3,10 +3,13 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +import json + from django.conf import settings from django.conf.urls import ( url, include, handler400, handler403, handler404, handler500 ) +from django.contrib.staticfiles import finders from django.contrib.staticfiles.views import serve from django.shortcuts import render from django.views.generic.base import RedirectView @@ -30,6 +33,12 @@ return render(request, "homepage.html") +def jslicenses(request): + jslicenses_data = json.load(open(finders.find('weblabels/weblabels.json'))) + return render(request, "jslicenses.html", + {'jslicenses_data': jslicenses_data}) + + urlpatterns = [ url(r'^admin/', include('swh.web.admin.urls')), url(r'^favicon\.ico$', favicon_view), @@ -39,7 +48,8 @@ url(r'^jsreverse/$', urls_js, name='js_reverse'), url(r'^(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$', swh_id_browse, name='browse-swh-id'), - url(r'^coverage/$', swh_coverage, name='swh-coverage') + url(r'^coverage/$', swh_coverage, name='swh-coverage'), + url(r'^jslicenses/$', jslicenses, name='jslicenses'), ] diff --git a/yarn.lock b/yarn.lock --- a/yarn.lock +++ b/yarn.lock @@ -7585,7 +7585,7 @@ resolved "https://registry.yarnpkg.com/rgba-regex/-/rgba-regex-1.0.0.tgz#43374e2e2ca0968b0ef1523460b7d730ff22eeb3" integrity sha1-QzdOLiyglosO8VI0YLfXMP8i7rM= -rimraf@2, rimraf@2.6.3, rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@~2.6.2: +rimraf@2, rimraf@2.6.3, rimraf@^2.2.8, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3, rimraf@~2.6.2: version "2.6.3" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==