diff --git a/package.json b/package.json
--- a/package.json
+++ b/package.json
@@ -15,6 +15,7 @@
"license": "AGPL-3.0-or-later",
"dependencies": {
"admin-lte": "^3.0.0-alpha",
+ "ansi_up": "^4.0.3",
"bootstrap": "^4.3.1",
"bootstrap-year-calendar-bs4": "^1.0.0",
"clipboard": "^2.0.4",
@@ -28,6 +29,7 @@
"iframe-resizer": "^4.1.1",
"jquery": "^3.4.0",
"js-cookie": "^2.2.0",
+ "notebookjs": "^0.4.2",
"object-fit-images": "^3.2.4",
"octicons": "^8.5.0",
"open-iconic": "^1.1.1",
@@ -80,6 +82,7 @@
"robotstxt-webpack-plugin": "^5.0.0",
"sass-loader": "^7.1.0",
"schema-utils": "^1.0.0",
+ "script-loader": "^0.7.2",
"spdx-expression-parse": "^3.0.0",
"style-loader": "^0.23.1",
"stylelint": "^10.0.1",
diff --git a/swh/web/assets/config/.eslintrc b/swh/web/assets/config/.eslintrc
--- a/swh/web/assets/config/.eslintrc
+++ b/swh/web/assets/config/.eslintrc
@@ -40,7 +40,9 @@
"__STATIC__": false,
"Image": false,
"Cookies": false,
- "grecaptcha": false
+ "grecaptcha": false,
+ "nb": false,
+ "MathJax": false
},
"rules": {
diff --git a/swh/web/assets/config/mathjax-js-files.js b/swh/web/assets/config/mathjax-js-files.js
new file mode 100644
--- /dev/null
+++ b/swh/web/assets/config/mathjax-js-files.js
@@ -0,0 +1,43 @@
+/**
+ * Copyright (C) 2019 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
+function getLoadedMathJaxJsFilesInfo(mathJaxJsFiles, cdnPrefix, unpackedLocationPrefix) {
+ let ret = {};
+ for (let mathJaxJsFile of mathJaxJsFiles) {
+ ret[cdnPrefix + mathJaxJsFile] = [{
+ 'id': mathJaxJsFile,
+ 'path': unpackedLocationPrefix + mathJaxJsFile,
+ 'spdxLicenseExpression': 'Apache-2.0',
+ 'licenseFilePath': ''
+ }];
+ }
+ return ret;
+}
+
+let cdnPrefix = 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/';
+
+let unpackedLocationPrefix = 'https://raw.githubusercontent.com/mathjax/MathJax/2.7.5/unpacked/';
+
+let mathJaxJsFiles = [
+ 'MathJax.js',
+ 'config/TeX-AMS_HTML.js',
+ 'extensions/MathMenu.js',
+ 'jax/element/mml/optable/BasicLatin.js',
+ 'jax/output/HTML-CSS/jax.js',
+ 'jax/output/HTML-CSS/fonts/TeX/fontdata.js',
+ 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/Main.js',
+ 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/BBBold.js',
+ 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/GeneralPunctuation.js',
+ 'jax/output/HTML-CSS/autoload/mtable.js',
+ 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/MiscTechnical.js',
+ 'jax/output/HTML-CSS/fonts/TeX/Typewriter/Regular/Main.js',
+ 'jax/element/mml/optable/MathOperators.js',
+ 'jax/output/HTML-CSS/autoload/multiline.js',
+ 'jax/output/HTML-CSS/fonts/TeX/AMS/Regular/MathOperators.js'
+];
+
+module.exports = getLoadedMathJaxJsFilesInfo(mathJaxJsFiles, cdnPrefix, unpackedLocationPrefix);
diff --git a/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js
--- a/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js
+++ b/swh/web/assets/config/webpack-plugins/generate-weblabels-webpack-plugin/index.js
@@ -99,6 +99,12 @@
}
}
+ // remove webpack loader call if any
+ let loaderEndPos = srcFilePath.indexOf('!');
+ if (loaderEndPos !== -1) {
+ srcFilePath = srcFilePath.slice(loaderEndPos + 1);
+ }
+
// iterate on all chunks containing the module
mod.chunks.forEach(chunk => {
@@ -171,8 +177,12 @@
// process additional scripts if needed
if (this.options['additionalScripts']) {
for (let script of Object.keys(this.options['additionalScripts'])) {
+ let scriptFilesData = this.options['additionalScripts'][script];
+ if (!script.startsWith('http:') && !script.startsWith('https:')) {
+ script = stats.publicPath + script;
+ }
this.chunkJsAssetToSrcFiles[script] = [];
- for (let scriptSrc of this.options['additionalScripts'][script]) {
+ for (let scriptSrc of scriptFilesData) {
let scriptSrcData = {'id': scriptSrc['id']};
let licenceFilePath = scriptSrc['licenseFilePath'];
let parsedSpdxLicenses = this.parseSpdxLicenseExpression(scriptSrc['spdxLicenseExpression'],
@@ -182,7 +192,11 @@
scriptSrcData['licenses'].forEach(license => {
license['copy_url'] = licenseCopyUrl;
});
- scriptSrcData['src_url'] = stats.publicPath + path.join(this.weblabelsDirName, scriptSrc['id']);
+ if (!scriptSrc['path'].startsWith('http:') && !scriptSrc['path'].startsWith('https:')) {
+ scriptSrcData['src_url'] = stats.publicPath + path.join(this.weblabelsDirName, scriptSrc['id']);
+ } else {
+ scriptSrcData['src_url'] = scriptSrc['path'];
+ }
this.chunkJsAssetToSrcFiles[script].push(scriptSrcData);
this.copyFileToOutputPath(scriptSrc['path']);
}
@@ -341,7 +355,9 @@
}
copyFileToOutputPath(srcFilePath, ext = '') {
- if (this.copiedFiles.has(srcFilePath)) {
+ if (this.copiedFiles.has(srcFilePath) ||
+ srcFilePath.startsWith('http:') ||
+ srcFilePath.startsWith('https:')) {
return;
}
let destPath = this.cleanupPath(srcFilePath);
diff --git a/swh/web/assets/config/webpack.config.development.js b/swh/web/assets/config/webpack.config.development.js
--- a/swh/web/assets/config/webpack.config.development.js
+++ b/swh/web/assets/config/webpack.config.development.js
@@ -18,6 +18,7 @@
const FixSwhSourceMapsPlugin = require('./webpack-plugins/fix-swh-source-maps-webpack-plugin');
const CopyWebpackPlugin = require('copy-webpack-plugin');
const GenerateWebLabelsPlugin = require('./webpack-plugins/generate-weblabels-webpack-plugin');
+const loadedMathJaxJsFiles = require('./mathjax-js-files');
// are we running webpack-dev-server ?
const isDevServer = process.argv.find(v => v.includes('webpack-dev-server'));
@@ -369,17 +370,20 @@
'licenseFilePath': './swh/web/assets/src/thirdparty/jquery.tabSlideOut/LICENSE'
}
},
- additionalScripts: {
- 'js/pdf.worker.min.js': [
- {
- 'id': 'pdfjs-dist/build/pdf.worker.js',
- 'path': './node_modules/pdfjs-dist/build/pdf.worker.js',
- 'spdxLicenseExpression': 'Apache-2.0',
- 'licenseFilePath': './node_modules/pdfjs-dist/LICENSE'
+ additionalScripts: Object.assign(
+ {
+ 'js/pdf.worker.min.js': [
+ {
+ 'id': 'pdfjs-dist/build/pdf.worker.js',
+ 'path': './node_modules/pdfjs-dist/build/pdf.worker.js',
+ 'spdxLicenseExpression': 'Apache-2.0',
+ 'licenseFilePath': './node_modules/pdfjs-dist/LICENSE'
- }
- ]
- }
+ }
+ ]
+ },
+ loadedMathJaxJsFiles
+ )
})
],
// webpack optimizations
diff --git a/swh/web/assets/src/bundles/webapp/index.js b/swh/web/assets/src/bundles/webapp/index.js
--- a/swh/web/assets/src/bundles/webapp/index.js
+++ b/swh/web/assets/src/bundles/webapp/index.js
@@ -1,5 +1,5 @@
/**
- * Copyright (C) 2018 The Software Heritage developers
+ * Copyright (C) 2018-2019 The Software Heritage developers
* See the AUTHORS file at the top-level directory of this distribution
* License: GNU Affero General Public License version 3, or any later version
* See top-level LICENSE file for more information
@@ -21,3 +21,5 @@
export * from './code-highlighting';
export * from './readme-rendering';
export * from './pdf-rendering';
+export * from './notebook-rendering';
+export * from './xss-filtering';
diff --git a/swh/web/assets/src/bundles/webapp/notebook-rendering.js b/swh/web/assets/src/bundles/webapp/notebook-rendering.js
new file mode 100644
--- /dev/null
+++ b/swh/web/assets/src/bundles/webapp/notebook-rendering.js
@@ -0,0 +1,152 @@
+/**
+ * Copyright (C) 2019 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
+/* eslint-disable */
+
+import 'script-loader!notebookjs';
+import AnsiUp from 'ansi_up';
+import './notebook.css';
+
+const ansiup = new AnsiUp();
+
+function escapeLaTeX(text) {
+
+ let blockMath = /\$\$(.+?)\$\$|\\\\\[(.+?)\\\\\]/msg;
+ let inlineMath = /\$(.+?)\$|\\\\\((.+?)\\\\\)/g;
+ let latexEnvironment = /\\begin\{([a-z]*\*?)\}(.+?)\\end\{\1\}/msg;
+
+ let mathTextFound = [];
+ let bm;
+ while ((bm = blockMath.exec(text)) !== null) {
+ mathTextFound.push(bm[1]);
+ }
+
+ let im;
+ while ((im = inlineMath.exec(text)) !== null) {
+ mathTextFound.push(im[1]);
+ }
+
+ let le;
+ while ((le = latexEnvironment.exec(text)) !== null) {
+ mathTextFound.push(le[1]);
+ }
+
+ for (let mathText of mathTextFound) {
+ // showdown will remove line breaks in LaTex array and
+ // some escaping sequences when converting md to html.
+ // So we use the following escaping hacks to keep them in the html
+ // output and avoid MathJax typesetting errors.
+ let escapedText = mathText.replace('\\\\', '\\\\\\\\');
+ for (let specialLaTexChar of ['{', '}', '#', '%', '&', '_']) {
+ escapedText = escapedText.replace(new RegExp(`\\\\${specialLaTexChar}`, 'g'),
+ `\\\\${specialLaTexChar}`);
+ }
+
+ // some html escaping is also needed
+ escapedText = escapedText.replace(//g, '>');
+
+ if (mathText !== escapedText) {
+ text = text.replace(mathText, escapedText);
+ }
+ }
+
+ return text;
+}
+
+export async function renderNotebook(nbJsonUrl, domElt) {
+
+ let showdown = await import(/* webpackChunkName: "showdown" */ 'utils/showdown');
+
+ await import(/* webpackChunkName: "highlightjs" */ 'utils/highlightjs');
+
+ function renderMarkdown(text) {
+ let converter = new showdown.Converter({
+ tables: true
+ });
+
+ // some LaTeX escaping is required to get correct math typesetting
+ text = escapeLaTeX(text);
+
+ // hack to prevent showdown to replace _ characters
+ // by html em tags as it will break some math typesetting
+ // (setting the literalMidWordUnderscores option is not
+ // enough as iy only works for _ characters contained in words)
+ text = text.replace(/_/g, '{@}underscore{@}');
+
+ // render markdown
+ let rendered = converter.makeHtml(text);
+
+ // restore underscores in rendered HTML
+ rendered = rendered.replace(/{@}underscore{@}/g, '_');
+
+ return rendered;
+ }
+
+ function highlightCode(text, preElt, codeElt, lang) {
+ if (lang) {
+ return hljs.highlight(lang, text).value;
+ } else {
+ return text;
+ }
+ }
+
+ function renderAnsi(text) {
+ return ansiup.ansi_to_html(text);
+ }
+
+ nb.markdown = renderMarkdown;
+ nb.highlighter = highlightCode;
+ nb.ansi = renderAnsi;
+
+ function initMathJax() {
+
+ // same config as in nbviewer
+ window.MathJax = {
+ TeX: {
+ equationNumbers: {
+ autoNumber: 'AMS',
+ useLabelIds: true
+ }
+ },
+ tex2jax: {
+ inlineMath: [ ['$', '$'], ['\\(', '\\)'] ],
+ displayMath: [ ['$$', '$$'], ['\\[', '\\]'] ],
+ processEscapes: true,
+ processEnvironments: true
+ },
+ displayAlign: 'center',
+ 'HTML-CSS': {
+ styles: {'.MathJax_Display': {'margin': 0}},
+ linebreaks: { automatic: true }
+ }
+ };
+
+ // MathJax is not easily webpackable in its current version
+ // (https://github.com/mathjax/MathJax/issues/1629)
+ // and is quite a monster regarding the number of files to distribute.
+ // So we will load it through a CDN for commodity of use here.
+ let head = document.getElementsByTagName('head')[0];
+ let script = document.createElement('script');
+ script.type = 'text/javascript';
+ script.src = 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS_HTML';
+ head.appendChild(script);
+ }
+
+ fetch(nbJsonUrl)
+ .then(response => response.json())
+ .then(nbJson => {
+ // parse the notebook
+ let notebook = nb.parse(nbJson);
+ // render it to HTML and apply XSS filtering
+ let rendered = swh.webapp.filterXSS(notebook.render());
+ // insert rendered notebook in the DOM
+ $(domElt).append(rendered);
+ // load MathJax library for math typesetting
+ initMathJax();
+ });
+}
diff --git a/swh/web/assets/src/bundles/webapp/notebook.css b/swh/web/assets/src/bundles/webapp/notebook.css
new file mode 100644
--- /dev/null
+++ b/swh/web/assets/src/bundles/webapp/notebook.css
@@ -0,0 +1,148 @@
+/**
+ * Copyright (C) 2019 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
+.nb-notebook {
+ line-height: 1.5;
+ padding-left: 6em;
+ overflow-y: hidden;
+}
+
+.nb-stdout,
+.nb-stderr {
+ white-space: pre-wrap;
+ margin: 1em 0;
+ padding: 0.1em 0.5em;
+}
+
+.nb-stderr {
+ background-color: #faa;
+}
+
+.nb-cell + .nb-cell {
+ margin-top: 0.5em;
+}
+
+.nb-output table {
+ border: 1px solid #000;
+ border-collapse: collapse;
+}
+
+.nb-output th {
+ font-weight: bold;
+}
+
+.nb-output th,
+.nb-output td {
+ border: 1px solid #000;
+ padding: 0.25em;
+ text-align: left;
+ vertical-align: middle;
+ border-collapse: collapse;
+}
+
+.nb-cell {
+ position: relative;
+}
+
+.nb-raw-cell {
+ white-space: pre-wrap;
+ background-color: #f5f2f0;
+ font-family: Consolas, Monaco, 'Andale Mono', monospace;
+ padding: 1em;
+ margin: 0.5em 0;
+}
+
+.nb-input {
+ border: 1px solid #cfcfcf;
+ border-radius: 2px;
+ background: #f7f7f7;
+ margin: 0.4em;
+ padding: 0;
+}
+
+.nb-notebook pre {
+ margin: 0.4em !important;
+ border: none;
+ padding: 0;
+ background-color: transparent;
+ min-height: 1rem;
+}
+
+.nb-output {
+ min-height: 1em;
+ width: 100%;
+ overflow-x: auto;
+ border-right: 1px dotted #ccc;
+}
+
+.nb-output img {
+ max-width: 100%;
+}
+
+.nb-output::before,
+.nb-input::before {
+ position: absolute;
+ font-family: monospace;
+ color: #999;
+ left: -7.5em;
+ width: 7em;
+ text-align: right;
+ font-size: large;
+}
+
+.nb-input::before {
+ content: "In [" attr(data-prompt-number) "]:";
+ color: #303f9f;
+}
+
+.nb-output::before {
+ content: "Out [" attr(data-prompt-number) "]:";
+ color: #d84315;
+}
+
+.nb-notebook div[style="max-height:1000px;max-width:1500px;overflow:auto;"] {
+ max-height: none !important;
+}
+
+.nb-latex-output .MathJax_Display {
+ text-align: left !important;
+ padding-left: 0.5rem;
+}
+
+.nb-image-output {
+ padding-left: 0.5rem;
+}
+
+.nb-markdown-cell {
+ margin: 0.4em;
+}
+
+@media screen and (max-width: 600px) {
+ .nb-notebook {
+ padding-left: 0;
+ }
+
+ .nb-input {
+ margin-top: 2em !important;
+ }
+
+ .nb-output::before,
+ .nb-input::before {
+ text-align: left;
+ }
+
+ .nb-input::before {
+ left: 0.5em;
+ top: -2em;
+ }
+
+ .nb-output::before {
+ position: relative;
+ left: 0.5em;
+ top: 0;
+ }
+}
diff --git a/swh/web/config.py b/swh/web/config.py
--- a/swh/web/config.py
+++ b/swh/web/config.py
@@ -37,7 +37,7 @@
'port': ('int', 5004),
'secret_key': ('string', 'development key'),
# do not display code highlighting for content > 1MB
- 'content_display_max_size': ('int', 1024 * 1024),
+ 'content_display_max_size': ('int', 5 * 1024 * 1024),
'snapshot_content_max_size': ('int', 1000),
'throttling': ('dict', {
'cache_uri': None, # production: memcached as cache (127.0.0.1:11211)
diff --git a/swh/web/templates/includes/content-display.html b/swh/web/templates/includes/content-display.html
--- a/swh/web/templates/includes/content-display.html
+++ b/swh/web/templates/includes/content-display.html
@@ -21,6 +21,9 @@
Content is too large to be displayed (size is greater than {{ max_content_size|filesizeformat }}).
{% elif "inode/x-empty" == mimetype %}
File is empty
+ {% elif swh_object_metadata.filename and swh_object_metadata.filename|default:""|slice:"-5:" == "ipynb" %}
+
+
{% elif "text/" in mimetype %}
{{ content }}
@@ -47,6 +50,8 @@