diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "license": "AGPL-3.0-or-later", "dependencies": { "admin-lte": "^3.0.0-alpha", + "ansi_up": "^4.0.3", "bootstrap": "^4.3.1", "bootstrap-year-calendar-bs4": "^1.0.0", "clipboard": "^2.0.4", @@ -28,6 +29,7 @@ "iframe-resizer": "^4.1.1", "jquery": "^3.4.0", "js-cookie": "^2.2.0", + "notebookjs": "^0.4.2", "object-fit-images": "^3.2.4", "octicons": "^8.5.0", "open-iconic": "^1.1.1", @@ -80,6 +82,7 @@ "robotstxt-webpack-plugin": "^5.0.0", "sass-loader": "^7.1.0", "schema-utils": "^1.0.0", + "script-loader": "^0.7.2", "spdx-expression-parse": "^3.0.0", "style-loader": "^0.23.1", "stylelint": "^10.0.1", diff --git a/swh/web/assets/config/.eslintrc b/swh/web/assets/config/.eslintrc --- a/swh/web/assets/config/.eslintrc +++ b/swh/web/assets/config/.eslintrc @@ -40,7 +40,9 @@ "__STATIC__": false, "Image": false, "Cookies": false, - "grecaptcha": false + "grecaptcha": false, + "nb": false, + "MathJax": false }, "rules": { diff --git a/swh/web/assets/config/mathjax-js-files.js b/swh/web/assets/config/mathjax-js-files.js new file mode 100644 --- /dev/null +++ b/swh/web/assets/config/mathjax-js-files.js @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +function getLoadedMathJaxJsFilesInfo(mathJaxJsFiles, cdnPrefix, unpackedLocationPrefix) { + let ret = {}; + for (let mathJaxJsFile of mathJaxJsFiles) { + ret[cdnPrefix + mathJaxJsFile] = [{ + 'id': mathJaxJsFile, + 'path': unpackedLocationPrefix + mathJaxJsFile, + 'spdxLicenseExpression': 'Apache-2.0', + 'licenseFilePath': '' + }]; + } + return ret; +} + +let cdnPrefix = 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/'; + +let unpackedLocationPrefix = 'https://raw.githubusercontent.com/mathjax/MathJax/2.7.5/unpacked/'; + +let mathJaxJsFiles = [ + 'MathJax.js', + 'config/TeX-AMS_HTML.js', + 'extensions/MathMenu.js', + 'jax/element/mml/optable/BasicLatin.js', + 'jax/output/HTML-CSS/jax.js', + 'jax/output/HTML-CSS/fonts/TeX/fontdata.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/Main.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/BBBold.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/GeneralPunctuation.js', + 'jax/output/HTML-CSS/autoload/mtable.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/MiscTechnical.js', + 'jax/output/HTML-CSS/fonts/TeX/Typewriter/Regular/Main.js', + 'jax/element/mml/optable/MathOperators.js', + 'jax/output/HTML-CSS/autoload/multiline.js', + 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/MathOperators.js' +]; + +module.exports = getLoadedMathJaxJsFilesInfo(mathJaxJsFiles, cdnPrefix, unpackedLocationPrefix); diff --git a/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js --- a/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js +++ b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js @@ -99,6 +99,12 @@ } } + // remove webpack loader call if any + let loaderEndPos = srcFilePath.indexOf('!'); + if (loaderEndPos !== -1) { + srcFilePath = srcFilePath.slice(loaderEndPos + 1); + } + // iterate on all chunks containing the module mod.chunks.forEach(chunk => { @@ -171,8 +177,12 @@ // process additional scripts if needed if (this.options['additionalScripts']) { for (let script of Object.keys(this.options['additionalScripts'])) { + let scriptFilesData = this.options['additionalScripts'][script]; + if (script.indexOf('://') === -1) { + script = stats.publicPath + script; + } this.chunkJsAssetToSrcFiles[script] = []; - for (let scriptSrc of this.options['additionalScripts'][script]) { + for (let scriptSrc of scriptFilesData) { let scriptSrcData = {'id': scriptSrc['id']}; let licenceFilePath = scriptSrc['licenseFilePath']; let parsedSpdxLicenses = this.parseSpdxLicenseExpression(scriptSrc['spdxLicenseExpression'], @@ -182,7 +192,11 @@ scriptSrcData['licenses'].forEach(license => { license['copy_url'] = licenseCopyUrl; }); - scriptSrcData['src_url'] = stats.publicPath + path.join(this.weblabelsDirName, scriptSrc['id']); + if (scriptSrc['path'].indexOf('://') === -1) { + scriptSrcData['src_url'] = stats.publicPath + path.join(this.weblabelsDirName, scriptSrc['id']); + } else { + scriptSrcData['src_url'] = scriptSrc['path']; + } this.chunkJsAssetToSrcFiles[script].push(scriptSrcData); this.copyFileToOutputPath(scriptSrc['path']); } @@ -341,7 +355,7 @@ } copyFileToOutputPath(srcFilePath, ext = '') { - if (this.copiedFiles.has(srcFilePath)) { + if (this.copiedFiles.has(srcFilePath) || srcFilePath.indexOf('://') !== -1) { return; } let destPath = this.cleanupPath(srcFilePath); diff --git a/swh/web/assets/config/webpack.config.development.js b/swh/web/assets/config/webpack.config.development.js --- a/swh/web/assets/config/webpack.config.development.js +++ b/swh/web/assets/config/webpack.config.development.js @@ -18,6 +18,7 @@ const FixSwhSourceMapsPlugin = require('./webpack-plugins/fix-swh-source-maps-webpack-plugin'); const CopyWebpackPlugin = require('copy-webpack-plugin'); const GenerateWebLabelsPlugin = require('./webpack-plugins/generate-weblabels-webpack-plugin'); +const loadedMathJaxJsFiles = require('./mathjax-js-files'); // are we running webpack-dev-server ? const isDevServer = process.argv.find(v => v.includes('webpack-dev-server')); @@ -369,17 +370,20 @@ 'licenseFilePath': './swh/web/assets/src/thirdparty/jquery.tabSlideOut/LICENSE' } }, - additionalScripts: { - 'js/pdf.worker.min.js': [ - { - 'id': 'pdfjs-dist/build/pdf.worker.js', - 'path': './node_modules/pdfjs-dist/build/pdf.worker.js', - 'spdxLicenseExpression': 'Apache-2.0', - 'licenseFilePath': './node_modules/pdfjs-dist/LICENSE' + additionalScripts: Object.assign( + { + 'js/pdf.worker.min.js': [ + { + 'id': 'pdfjs-dist/build/pdf.worker.js', + 'path': './node_modules/pdfjs-dist/build/pdf.worker.js', + 'spdxLicenseExpression': 'Apache-2.0', + 'licenseFilePath': './node_modules/pdfjs-dist/LICENSE' - } - ] - } + } + ] + }, + loadedMathJaxJsFiles + ) }) ], // webpack optimizations diff --git a/swh/web/assets/src/bundles/webapp/index.js b/swh/web/assets/src/bundles/webapp/index.js --- a/swh/web/assets/src/bundles/webapp/index.js +++ b/swh/web/assets/src/bundles/webapp/index.js @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information @@ -21,3 +21,5 @@ export * from './code-highlighting'; export * from './readme-rendering'; export * from './pdf-rendering'; +export * from './notebook-rendering'; +export * from './xss-filtering'; diff --git a/swh/web/assets/src/bundles/webapp/notebook-rendering.js b/swh/web/assets/src/bundles/webapp/notebook-rendering.js new file mode 100644 --- /dev/null +++ b/swh/web/assets/src/bundles/webapp/notebook-rendering.js @@ -0,0 +1,168 @@ +/** + * Copyright (C) 2019 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +import 'script-loader!notebookjs'; +import AnsiUp from 'ansi_up'; +import './notebook.css'; + +const ansiup = new AnsiUp(); +ansiup.escape_for_html = false; + +function escapeHTML(text) { + text = text.replace(//g, '>'); + return text; +} + +function unescapeHTML(text) { + text = text.replace(/</g, '<'); + text = text.replace(/>/g, '>'); + return text; +} + +function escapeLaTeX(text) { + + let blockMath = /\$\$(.+?)\$\$|\\\\\[(.+?)\\\\\]/msg; + let inlineMath = /\$(.+?)\$|\\\\\((.+?)\\\\\)/g; + let latexEnvironment = /\\begin\{([a-z]*\*?)\}(.+?)\\end\{\1\}/msg; + + let mathTextFound = []; + let bm; + while ((bm = blockMath.exec(text)) !== null) { + mathTextFound.push(bm[1]); + } + + let im; + while ((im = inlineMath.exec(text)) !== null) { + mathTextFound.push(im[1]); + } + + let le; + while ((le = latexEnvironment.exec(text)) !== null) { + mathTextFound.push(le[1]); + } + + for (let mathText of mathTextFound) { + // showdown will remove line breaks in LaTex array and + // some escaping sequences when converting md to html. + // So we use the following escaping hacks to keep them in the html + // output and avoid MathJax typesetting errors. + let escapedText = mathText.replace('\\\\', '\\\\\\\\'); + for (let specialLaTexChar of ['{', '}', '#', '%', '&', '_']) { + escapedText = escapedText.replace(new RegExp(`\\\\${specialLaTexChar}`, 'g'), + `\\\\${specialLaTexChar}`); + } + + // some html escaping is also needed + escapedText = escapeHTML(escapedText); + + // hack to prevent showdown to replace _ characters + // by html em tags as it will break some math typesetting + // (setting the literalMidWordUnderscores option is not + // enough as iy only works for _ characters contained in words) + escapedText = escapedText.replace(/_/g, '{@}underscore{@}'); + + if (mathText !== escapedText) { + text = text.replace(mathText, escapedText); + } + } + + return text; +} + +export async function renderNotebook(nbJsonUrl, domElt) { + + let showdown = await import(/* webpackChunkName: "showdown" */ 'utils/showdown'); + + await import(/* webpackChunkName: "highlightjs" */ 'utils/highlightjs'); + + function renderMarkdown(text) { + let converter = new showdown.Converter({ + tables: true, + simplifiedAutoLink: true, + rawHeaderId: true + }); + + // some LaTeX escaping is required to get correct math typesetting + text = escapeLaTeX(text); + + // render markdown + let rendered = converter.makeHtml(text); + + // restore underscores in rendered HTML (see escapeLaTeX function) + rendered = rendered.replace(/{@}underscore{@}/g, '_'); + + return rendered; + } + + function highlightCode(text, preElt, codeElt, lang) { + // no need to unescape text processed by ansiup + if (text.indexOf(' 1MB - 'content_display_max_size': ('int', 1024 * 1024), + 'content_display_max_size': ('int', 5 * 1024 * 1024), 'snapshot_content_max_size': ('int', 1000), 'throttling': ('dict', { 'cache_uri': None, # production: memcached as cache (127.0.0.1:11211) diff --git a/swh/web/templates/includes/content-display.html b/swh/web/templates/includes/content-display.html --- a/swh/web/templates/includes/content-display.html +++ b/swh/web/templates/includes/content-display.html @@ -21,6 +21,9 @@ Content is too large to be displayed (size is greater than {{ max_content_size|filesizeformat }}). {% elif "inode/x-empty" == mimetype %} File is empty + {% elif swh_object_metadata.filename and swh_object_metadata.filename|default:""|slice:"-5:" == "ipynb" %} +
+
{% elif "text/" in mimetype %} 
{{ content }}
@@ -47,6 +50,8 @@