diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "d3": "^6.2.0", "datatables.net-responsive-bs4": "^2.2.6", "dompurify": "^2.2.2", - "highlight.js": "^10.3.2", + "highlight.js": "^10.4.0", "highlightjs-line-numbers.js": "^2.8.0", "html-encoder-decoder": "^1.3.9", "iframe-resizer": "^4.2.11", @@ -84,6 +84,7 @@ "file-loader": "^6.2.0", "imports-loader": "^1.2.0", "istanbul-lib-coverage": "^3.0.0", + "json-stable-stringify": "^1.0.1", "mini-css-extract-plugin": "^1.3.1", "mocha": "^8.2.1", "mocha-junit-reporter": "^2.0.0", diff --git a/static/json/highlightjs-languages.json b/static/json/highlightjs-languages.json new file mode 100644 --- /dev/null +++ b/static/json/highlightjs-languages.json @@ -0,0 +1,382 @@ +{ + "languages": [ + "1c", + "abnf", + "accesslog", + "actionscript", + "ada", + "angelscript", + "apache", + "applescript", + "arcade", + "arduino", + "armasm", + "xml", + "asciidoc", + "aspectj", + "autohotkey", + "autoit", + "avrasm", + "awk", + "axapta", + "bash", + "basic", + "bnf", + "brainfuck", + "c-like", + "c", + "cal", + "capnproto", + "ceylon", + "clean", + "clojure", + "clojure-repl", + "cmake", + "coffeescript", + "coq", + "cos", + "cpp", + "crmsh", + "crystal", + "csharp", + "csp", + "css", + "d", + "markdown", + "dart", + "delphi", + "diff", + "django", + "dns", + "dockerfile", + "dos", + "dsconfig", + "dts", + "dust", + "ebnf", + "elixir", + "elm", + "ruby", + "erb", + "erlang-repl", + "erlang", + "excel", + "fix", + "flix", + "fortran", + "fsharp", + "gams", + "gauss", + "gcode", + "gherkin", + "glsl", + "gml", + "go", + "golo", + "gradle", + "groovy", + "haml", + "handlebars", + "haskell", + "haxe", + "hsp", + "htmlbars", + "http", + "hy", + "inform7", + "ini", + "irpf90", + "isbl", + "java", + "javascript", + "jboss-cli", + "json", + "julia", + "julia-repl", + "kotlin", + "lasso", + "latex", + "ldif", + "leaf", + "less", + "lisp", + "livecodeserver", + "livescript", + "llvm", + "lsl", + "lua", + "makefile", + "mathematica", + "matlab", + "maxima", + "mel", + "mercury", + "mipsasm", + "mizar", + "perl", + "mojolicious", + "monkey", + "moonscript", + "n1ql", + "nginx", + "nim", + "nix", + "node-repl", + "nsis", + "objectivec", + "ocaml", + "openscad", + "oxygene", + "parser3", + "pf", + "pgsql", + "php", + "php-template", + "plaintext", + "pony", + "powershell", + "processing", + "profile", + "prolog", + "properties", + "protobuf", + "puppet", + "purebasic", + "python", + "python-repl", + "q", + "qml", + "r", + "reasonml", + "rib", + "roboconf", + "routeros", + "rsl", + "ruleslanguage", + "rust", + "sas", + "scala", + "scheme", + "scilab", + "scss", + "shell", + "smali", + "smalltalk", + "sml", + "sqf", + "sql", + "stan", + "stata", + "step21", + "stylus", + "subunit", + "swift", + "taggerscript", + "yaml", + "tap", + "tcl", + "thrift", + "tp", + "twig", + "typescript", + "vala", + "vbnet", + "vbscript", + "vbscript-html", + "verilog", + "vhdl", + "vim", + "x86asm", + "xl", + "xquery", + "zephir" + ], + "languages_aliases": { + "GML": "gml", + "SAS": "sas", + "TeX": "latex", + "YAML": "yaml", + "ado": "stata", + "adoc": "asciidoc", + "ahk": "autohotkey", + "apacheconf": "apache", + "arcade": "arcade", + "arm": "armasm", + "as": "actionscript", + "asc": "angelscript", + "atom": "xml", + "bat": "dos", + "bf": "brainfuck", + "bind": "dns", + "c": "c", + "c#": "csharp", + "c++": "cpp", + "capnp": "capnproto", + "cc": "cpp", + "cjs": "javascript", + "clean": "clean", + "clj": "clojure", + "cls": "cos", + "cmake.in": "cmake", + "cmd": "dos", + "coffee": "coffeescript", + "console": "shell", + "cos": "cos", + "cr": "crystal", + "craftcms": "twig", + "crm": "crmsh", + "cs": "csharp", + "cson": "coffeescript", + "cxx": "cpp", + "dcl": "clean", + "dfm": "delphi", + "do": "stata", + "docker": "dockerfile", + "dpr": "delphi", + "dst": "dust", + "erl": "erlang", + "f90": "fortran", + "f95": "fortran", + "feature": "gherkin", + "freepascal": "delphi", + "fs": "fsharp", + "gemspec": "ruby", + "gml": "gml", + "gms": "gams", + "golang": "go", + "graph": "roboconf", + "gss": "gauss", + "gyp": "python", + "h": "c", + "h++": "cpp", + "hbs": "htmlbars", + "hh": "cpp", + "hpp": "cpp", + "hs": "haskell", + "html": "xml", + "html.handlebars": "htmlbars", + "html.hbs": "htmlbars", + "htmlbars": "htmlbars", + "https": "http", + "hx": "haxe", + "hxx": "cpp", + "hylang": "hy", + "i7": "inform7", + "iced": "coffeescript", + "icl": "clean", + "ino": "arduino", + "instances": "roboconf", + "ipython": "python", + "irb": "ruby", + "isbl": "isbl", + "jinja": "django", + "js": "javascript", + "jsp": "java", + "jsx": "javascript", + "k": "q", + "kdb": "q", + "kt": "kotlin", + "lassoscript": "lasso", + "lazarus": "delphi", + "lfm": "delphi", + "lpr": "delphi", + "ls": "livescript", + "m": "mercury", + "mak": "makefile", + "md": "markdown", + "mikrotik": "routeros", + "mips": "mipsasm", + "mjs": "javascript", + "mk": "makefile", + "mkd": "markdown", + "mkdown": "markdown", + "ml": "sml", + "mm": "objectivec", + "mma": "mathematica", + "moo": "mercury", + "moon": "moonscript", + "nc": "gcode", + "nginxconf": "nginx", + "nim": "nim", + "nixos": "nix", + "obj-c": "objectivec", + "obj-c++": "objectivec", + "objc": "objectivec", + "objective-c++": "objectivec", + "osascript": "applescript", + "p21": "step21", + "pas": "delphi", + "pascal": "delphi", + "patch": "diff", + "pb": "purebasic", + "pbi": "purebasic", + "pcmk": "crmsh", + "pf.conf": "pf", + "php": "php", + "php3": "php", + "php4": "php", + "php5": "php", + "php6": "php", + "php7": "php", + "php8": "php", + "pl": "perl", + "plist": "xml", + "pm": "perl", + "podspec": "ruby", + "postgres": "pgsql", + "postgresql": "pgsql", + "pp": "puppet", + "ps": "powershell", + "ps1": "powershell", + "py": "python", + "pycon": "python-repl", + "qt": "qml", + "rb": "ruby", + "re": "reasonml", + "routeros": "routeros", + "rs": "rust", + "rss": "xml", + "sas": "sas", + "scad": "openscad", + "sci": "scilab", + "sh": "bash", + "smali": "smali", + "sqf": "sqf", + "st": "smalltalk", + "stanfuncs": "stan", + "step": "step21", + "stp": "step21", + "styl": "stylus", + "sv": "verilog", + "svg": "xml", + "svh": "verilog", + "tao": "xl", + "tex": "latex", + "text": "plaintext", + "thor": "ruby", + "tk": "tcl", + "toml": "ini", + "ts": "typescript", + "txt": "plaintext", + "v": "verilog", + "vb": "vbnet", + "vbs": "vbscript", + "wildfly-cli": "jboss-cli", + "wl": "mathematica", + "wsf": "xml", + "x++": "axapta", + "xhtml": "xml", + "xjb": "xml", + "xls": "excel", + "xlsx": "excel", + "xpath": "xquery", + "xq": "xquery", + "xsd": "xml", + "xsl": "xml", + "yaml": "yaml", + "yml": "yaml", + "zep": "zephir", + "zone": "dns", + "zsh": "bash" + } +} \ No newline at end of file diff --git a/swh/web/assets/config/webpack-plugins/dump-highlightjs-languages-data-plugin.js b/swh/web/assets/config/webpack-plugins/dump-highlightjs-languages-data-plugin.js new file mode 100644 --- /dev/null +++ b/swh/web/assets/config/webpack-plugins/dump-highlightjs-languages-data-plugin.js @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2020 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const hljs = require('highlight.js'); +var stringify = require('json-stable-stringify'); + +// Simple webpack plugin to dump JSON data related to the set of +// programming languages supported by the highlightjs library. +// The JSON file is saved into swh-web static folder and will +// be consumed by the backend django application. + +class DumpHighlightjsLanguagesDataPlugin { + + apply(compiler) { + compiler.hooks.done.tap('DumpHighlightjsLanguagesDataPlugin', statsObj => { + const outputPath = statsObj.compilation.compiler.outputPath; + const hljsDataFile = path.join(outputPath, 'json/highlightjs-languages.json'); + const languages = hljs.listLanguages(); + const hljsLanguagesData = {'languages': languages}; + const languageAliases = {}; + for (let language of languages) { + const languageData = hljs.getLanguage(language); + if (!languageData.hasOwnProperty('aliases')) { + continue; + } + for (let alias of languageData.aliases) { + languageAliases[alias] = language; + languageAliases[alias.toLowerCase()] = language; + } + } + hljsLanguagesData['languages_aliases'] = languageAliases; + fs.writeFileSync(hljsDataFile, stringify(hljsLanguagesData, {space: 4})); + }); + } + +}; + +module.exports = DumpHighlightjsLanguagesDataPlugin; diff --git a/swh/web/assets/config/webpack.config.development.js b/swh/web/assets/config/webpack.config.development.js --- a/swh/web/assets/config/webpack.config.development.js +++ b/swh/web/assets/config/webpack.config.development.js @@ -21,6 +21,7 @@ const GenerateWebLabelsPlugin = require('./webpack-plugins/generate-weblabels-webpack-plugin'); const ProgressBarPlugin = require('progress-bar-webpack-plugin'); const FixWebpackStatsFormatPlugin = require('./webpack-plugins/fix-webpack-stats-format-plugin'); +const DumpHighlightjsLanguagesDataPlugin = require('./webpack-plugins/dump-highlightjs-languages-data-plugin'); // are we running webpack-dev-server ? const isDevServer = process.argv.find(v => v.includes('serve')); @@ -354,7 +355,7 @@ new CleanWebpackPlugin({ cleanOnceBeforeBuildPatterns: ['**/*', '!xml', '!xml/*', '!img', '!img/*', '!img/logos', '!img/logos/*', '!img/icons', - '!img/icons/*'] + '!img/icons/*', '!json', '!json/*'] }), // needed in order to use django_webpack_loader new BundleTracker({ @@ -459,7 +460,8 @@ format: chalk.cyan.bold('webpack build of swh-web assets') + ' [:bar] ' + chalk.green.bold(':percent') + ' (:elapsed seconds)', width: 50 }), - new FixWebpackStatsFormatPlugin() + new FixWebpackStatsFormatPlugin(), + new DumpHighlightjsLanguagesDataPlugin() ], // webpack optimizations optimization: { diff --git a/swh/web/common/highlightjs.py b/swh/web/common/highlightjs.py --- a/swh/web/common/highlightjs.py +++ b/swh/web/common/highlightjs.py @@ -4,398 +4,32 @@ # See top-level LICENSE file for more information import functools +import json from typing import Dict from pygments.lexers import get_all_lexers, get_lexer_for_filename import sentry_sdk -# set of languages ids that can be highlighted -# by highlight.js library -_hljs_languages = set( - [ - "1c", - "abnf", - "accesslog", - "actionscript", - "ada", - "angelscript", - "apache", - "applescript", - "arcade", - "arduino", - "armasm", - "asciidoc", - "aspectj", - "autohotkey", - "autoit", - "avrasm", - "awk", - "axapta", - "bash", - "basic", - "bnf", - "brainfuck", - "cal", - "capnproto", - "ceylon", - "clean", - "clojure", - "clojure-repl", - "cmake", - "coffeescript", - "coq", - "cos", - "cpp", - "crmsh", - "crystal", - "cs", - "csp", - "css", - "d", - "dart", - "delphi", - "diff", - "django", - "dns", - "dockerfile", - "dos", - "dsconfig", - "dts", - "dust", - "ebnf", - "elixir", - "elm", - "erb", - "erlang", - "erlang-repl", - "excel", - "fix", - "flix", - "fortran", - "fsharp", - "gams", - "gauss", - "gcode", - "gherkin", - "glsl", - "gml", - "go", - "golo", - "gradle", - "groovy", - "haml", - "handlebars", - "haskell", - "haxe", - "hsp", - "htmlbars", - "http", - "hy", - "inform7", - "ini", - "irpf90", - "isbl", - "java", - "javascript", - "jboss-cli", - "json", - "julia", - "julia-repl", - "kotlin", - "lasso", - "ldif", - "leaf", - "less", - "lisp", - "livecodeserver", - "livescript", - "llvm", - "lsl", - "lua", - "makefile", - "markdown", - "mathematica", - "matlab", - "maxima", - "mel", - "mercury", - "mipsasm", - "mizar", - "mojolicious", - "monkey", - "moonscript", - "n1ql", - "nginx", - "nimrod", - "nix", - "nsis", - "objectivec", - "ocaml", - "openscad", - "oxygene", - "parser3", - "perl", - "pf", - "pgsql", - "php", - "plaintext", - "pony", - "powershell", - "processing", - "profile", - "prolog", - "properties", - "protobuf", - "puppet", - "purebasic", - "python", - "q", - "qml", - "r", - "reasonml", - "rib", - "roboconf", - "routeros", - "rsl", - "ruby", - "ruleslanguage", - "rust", - "sas", - "scala", - "scheme", - "scilab", - "scss", - "shell", - "smali", - "smalltalk", - "sml", - "sqf", - "sql", - "stan", - "stata", - "step21", - "stylus", - "subunit", - "swift", - "taggerscript", - "tap", - "tcl", - "tex", - "thrift", - "tp", - "twig", - "typescript", - "vala", - "vbnet", - "vbscript", - "vbscript-html", - "verilog", - "vhdl", - "vim", - "x86asm", - "xl", - "xml", - "xquery", - "yaml", - "zephir", - ] -) +from django.contrib.staticfiles.finders import find +with open(str(find("json/highlightjs-languages.json")), "r") as _hljs_languages_file: + _hljs_languages_data = json.load(_hljs_languages_file) + +# set of languages ids that can be highlighted by highlight.js library +_hljs_languages = set(_hljs_languages_data["languages"]) # languages aliases defined in highlight.js _hljs_languages_aliases = { - "ado": "stata", - "adoc": "asciidoc", - "ahk": "autohotkey", - "aj": "aspectj", - "apacheconf": "apache", - "arm": "armasm", - "as": "actionscript", - "asc": "asciidoc", - "atom": "xml", - "bas": "basic", - "bat": "dos", - "bf": "brainfuck", - "bind": "dns", + **_hljs_languages_data["languages_aliases"], + "ml": "ocaml", "bsl": "1c", - "c-al": "cal", - "c": "cpp", - "c++": "cpp", - "capnp": "capnproto", - "cc": "cpp", - "clj": "clojure", - "cls": "cos", - "cmake.in": "cmake", - "cmd": "dos", - "coffee": "coffeescript", - "console": "shell", - "cr": "crystal", - "craftcms": "twig", - "crm": "crmsh", - "csharp": "cs", - "cson": "coffeescript", - "dcl": "clean", - "dfm": "delphi", - "do": "stata", - "docker": "dockerfile", - "dpr": "delphi", - "dst": "dust", - "dtsi": "dts", "ep": "mojolicious", - "erl": "erlang", - "ex": "elixir", - "exs": "elixir", - "f90": "fortran", - "f95": "fortran", - "feature": "gherkin", - "freepascal": "delphi", - "fs": "fsharp", - "fsx": "fsharp", - "gemspec": "ruby", - "GML": "gml", - "gms": "gams", - "golang": "go", - "graph": "roboconf", - "gss": "gauss", - "gyp": "python", - "h": "cpp", - "h++": "cpp", - "hbs": "handlebars", - "hpp": "cpp", - "hs": "haskell", - "html": "xml", - "html.handlebars": "handlebars", - "html.hbs": "handlebars", - "https": "http", - "hx": "haxe", - "hylang": "hy", - "i7": "inform7", - "i7x": "inform7", - "iced": "coffeescript", - "icl": "clean", - "ino": "arduino", - "instances": "roboconf", - "ipynb": "json", - "irb": "ruby", - "jinja": "django", - "js": "javascript", - "jsp": "java", - "jsx": "javascript", - "k": "q", - "kdb": "q", - "kt": "kotlin", - "lassoscript": "lasso", - "lazarus": "delphi", "lc": "livecode", - "lfm": "delphi", - "ll": "llvm", - "lpr": "delphi", - "ls": "livescript", - "m": "matlab", - "mak": "makefile", - "md": "markdown", - "mikrotik": "routeros", - "mips": "mipsasm", - "mk": "monkey", - "mkd": "markdown", - "mkdown": "markdown", - "ml": "ocaml", - "mli": "ocaml", - "mm": "objectivec", - "mma": "mathematica", - "moo": "mercury", - "moon": "moonscript", - "nav": "cal", - "nb": "mathematica", - "nc": "gcode", - "nginxconf": "nginx", - "ni": "inform7", - "nim": "nimrod", - "nixos": "nix", - "nsi": "nsis", - "obj-c": "objectivec", - "objc": "objectivec", - "osascript": "applescript", - "osl": "rsl", "p": "parser3", - "p21": "step21", - "pas": "delphi", - "pascal": "delphi", - "patch": "diff", - "pb": "purebasic", - "pbi": "purebasic", - "pcmk": "crmsh", "pde": "processing", - "pf.conf": "pf", - "php3": "php", - "php4": "php", - "php5": "php", - "php6": "php", - "php7": "php", - "pl": "perl", - "plist": "xml", - "pm": "perl", - "podspec": "ruby", - "postgres": "pgsql", - "postgresql": "pgsql", - "pp": "puppet", - "proto": "protobuf", - "ps": "powershell", - "ps1": "powershell", - "psd1": "powershell", - "psm1": "powershell", - "py": "python", - "qt": "qml", - "rb": "ruby", - "re": "reasonml", - "rei": "reasonml", - "rs": "rust", "rsc": "routeros", - "rss": "xml", - "rst": "nohighlight", "s": "armasm", - "SAS": "sas", - "scad": "openscad", - "sci": "scilab", - "scm": "scheme", - "sh": "bash", - "sig": "sml", "sl": "rsl", - "st": "smalltalk", - "step": "step21", - "stp": "step21", - "styl": "stylus", - "sv": "verilog", - "svh": "verilog", - "tao": "xl", - "thor": "ruby", - "tk": "tcl", - "toml": "ini", - "ts": "typescript", - "txt": "nohighlight", - "v": "coq", - "vb": "vbnet", - "vbs": "vbscript", - "vhd": "vhdl", - "wildfly-cli": "jboss-cli", - "wl": "mathematica", - "wls": "mathematica", - "xhtml": "xml", - "xjb": "xml", - "xls": "excel", - "xlsx": "excel", - "xpath": "xquery", - "xpo": "axapta", - "xpp": "axapta", - "xq": "xquery", - "xqy": "xquery", - "xsd": "xml", - "xsl": "xml", - "YAML": "yaml", - "yml": "yaml", - "zep": "zephir", - "zone": "dns", - "zsh": "bash", } # dictionary mapping pygment lexers to hljs languages @@ -404,7 +38,7 @@ # dictionary mapping mime types to hljs languages _mime_type_to_hljs_language = { - "text/x-c": "cpp", + "text/x-c": "c", "text/x-c++": "cpp", "text/x-msdos-batch": "dos", "text/x-lisp": "lisp", diff --git a/swh/web/tests/common/test_highlightjs.py b/swh/web/tests/common/test_highlightjs.py --- a/swh/web/tests/common/test_highlightjs.py +++ b/swh/web/tests/common/test_highlightjs.py @@ -12,7 +12,7 @@ assert lang is None lang = highlightjs.get_hljs_language_from_mime_type("text/x-c") - assert lang == "cpp" + assert lang == "c" lang = highlightjs.get_hljs_language_from_mime_type("text/x-c++") assert lang == "cpp" @@ -27,7 +27,7 @@ assert lang == "dos" lang = highlightjs.get_hljs_language_from_mime_type("text/x-tex") - assert lang == "tex" + assert lang == "latex" lang = highlightjs.get_hljs_language_from_mime_type("text/x-lisp") assert lang == "lisp" @@ -49,9 +49,9 @@ for filename, language in ( ("foo", None), - ("foo.h", "cpp"), - ("foo.c", "cpp"), - ("foo.c.in", "cpp"), + ("foo.h", "c"), + ("foo.c", "c"), + ("foo.c.in", "c"), ("foo.cpp", "cpp"), ("foo.pl", "perl"), ("foo.py", "python"), diff --git a/yarn.lock b/yarn.lock --- a/yarn.lock +++ b/yarn.lock @@ -6731,10 +6731,10 @@ resolved "https://registry.yarnpkg.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz#4c06fccb4602fe2602b3c93df82d7e7dbf1a8a8e" integrity sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ== -highlight.js@^10.3.2: - version "10.3.2" - resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-10.3.2.tgz#135fd3619a00c3cbb8b4cd6dbc78d56bfcbc46f1" - integrity sha512-3jRT7OUYsVsKvukNKZCtnvRcFyCJqSEIuIMsEybAXRiFSwpt65qjPd/Pr+UOdYt7WJlt+lj3+ypUsHiySBp/Jw== +highlight.js@^10.4.0: + version "10.4.0" + resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-10.4.0.tgz#ef3ce475e5dfa7a48484260b49ea242ddab823a0" + integrity sha512-EfrUGcQ63oLJbj0J0RI9ebX6TAITbsDBLbsjr881L/X5fMO9+oadKzEF21C7R3ULKG6Gv3uoab2HiqVJa/4+oA== highlightjs-line-numbers.js@^2.8.0: version "2.8.0" @@ -7830,6 +7830,13 @@ resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE= +json-stable-stringify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz#9a759d39c5f2ff503fd5300646ed445f88c4f9af" + integrity sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8= + dependencies: + jsonify "~0.0.0" + json-stable-stringify@~0.0.0: version "0.0.1" resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-0.0.1.tgz#611c23e814db375527df851193db59dd2af27f45"