diff --git a/assets/config/webpack.config.development.js b/assets/config/webpack.config.development.js --- a/assets/config/webpack.config.development.js +++ b/assets/config/webpack.config.development.js @@ -26,6 +26,7 @@ // are we running webpack-dev-server ? const isDevServer = process.argv.find(v => v.includes('serve')) !== undefined; // webpack-dev-server configuration +const host = '0.0.0.0'; const devServerPort = 3000; const devServerPublicPath = 'http://localhost:' + devServerPort + '/static/'; // set publicPath according if we are using webpack-dev-server to serve @@ -97,19 +98,8 @@ devtool: isDevServer ? 'eval' : 'source-map', // webpack-dev-server configuration devServer: { - client: { - logging: 'warn', - overlay: { - warnings: true, - errors: true - }, - progress: true - }, - devMiddleware: { - publicPath: devServerPublicPath, - stats: 'errors-only' - }, - host: '0.0.0.0', + clientLogLevel: 'warning', + host: host, port: devServerPort, // enable to serve static assets not managed by webpack static: { @@ -145,10 +135,16 @@ alias: { 'pdfjs-dist': 'pdfjs-dist/build/pdf.min.js' }, + // for web-tree-sitter + fallback: { + 'path': false, + 'fs': false + }, // configure base paths for resolving modules with webpack modules: [ 'node_modules', - path.resolve(__dirname, '../src') + path.resolve(__dirname, '../src'), + path.resolve(__dirname, '../../../swh-search/query_language/') ] }, stats: 'errors-warnings', @@ -240,6 +236,34 @@ } }] }, + { + test: require.resolve('js-cookie'), + use: [{ + loader: 'expose-loader', + options: { + exposes: { + globalName: 'Cookies', + override: true + } + } + }] + }, + // import .wasm files (for web-tree-sitter) + // web-tree-sitter tries to load static/js/tree-sitter.wasm + // so don't create new path 'wasm/' for .wasm resources. + // otherwise, web-tree-sitter will have to be patched + // in order to use the new path. + { + test: /\.wasm$/, + type: 'javascript/auto', + use: [{ + loader: 'file-loader', + options: { + name: '[name].[ext]', + outputPath: 'js/' + } + }] + }, // css import configuration: // - first process it with postcss // - then extract it to a dedicated file associated to each bundle diff --git a/assets/src/bundles/browse/origin-search.js b/assets/src/bundles/browse/origin-search.js --- a/assets/src/bundles/browse/origin-search.js +++ b/assets/src/bundles/browse/origin-search.js @@ -6,7 +6,7 @@ */ import {handleFetchError, isArchivedOrigin} from 'utils/functions'; - +import {initAutocomplete} from 'utils/search-ql-autocomplete'; const limit = 100; const linksPrev = []; let linkNext = null; @@ -193,6 +193,17 @@ export function initOriginSearch() { $(document).ready(() => { + const inputBox = document.querySelector('#swh-origins-url-patterns'); + const submitBtn = document.querySelector('#swh-search-submit'); + const validQueryCallback = (isValid) => { + submitBtn.disabled = !isValid; + // if (!isValid) + // inputBox.classList.add('invalid'); + // else + // inputBox.classList.remove('invalid'); + }; + initAutocomplete(inputBox, validQueryCallback); + $('#swh-search-origins').submit(event => { event.preventDefault(); if (event.target.checkValidity()) { diff --git a/assets/src/utils/autocomplete.css b/assets/src/utils/autocomplete.css new file mode 100644 --- /dev/null +++ b/assets/src/utils/autocomplete.css @@ -0,0 +1,44 @@ +/* the container must be positioned relative: */ +.autocomplete { + position: relative; + display: inline-block; +} + +input.invalid { + outline: none !important; + /* border: 2px solid red; */ +} + +/* position the autocomplete items to be the same width as the container: */ +.autocomplete-items { + position: absolute; + border: 1px solid #d4d4d4; + width: 200px; + /* overflow-y: scroll; */ + border-top: none; + z-index: 99998; + /* z-index: 99999; is taken by swh-top-bar */ + top: 100%; + left: 0; + right: 0; +} + +.autocomplete-items div { + padding: 3px; + padding-left: 5px; + /* font-size: 15px; */ + cursor: pointer; + background-color: #fff; + /* border-bottom: 1px solid #d4d4d4; */ +} + +/* when hovering an item: */ +.autocomplete-items div:hover { + background-color: #e9e9e9; +} + +/* when navigating through the items using the arrow keys: */ +.autocomplete-active { + background-color: #E20026 !important; + color: #fff; +} diff --git a/assets/src/utils/autocomplete.js b/assets/src/utils/autocomplete.js new file mode 100644 --- /dev/null +++ b/assets/src/utils/autocomplete.js @@ -0,0 +1,140 @@ +import 'utils/autocomplete.css'; + +export class Autocomplete { + constructor(params) { + const {inputBox, suggestions} = params; + this.inputBox = inputBox; + this.suggestions = suggestions; + this.currentIndex = -1; + + this.autocompleteList = document.createElement('div'); + this.autocompleteList.setAttribute('class', 'autocomplete-items'); + this.inputBox.parentNode.appendChild(this.autocompleteList); + + this.initListeners(); + } + + initListeners() { + this.inputBox.addEventListener('focus', this.updateLists.bind(this)); + this.inputBox.addEventListener('input', this.updateLists.bind(this)); + + this.inputBox.addEventListener('keydown', (e) => { + if (e.keyCode === 40) { // down + this.currentIndex++; + this.addActive(); + } else if (e.keyCode === 38) { // up + this.currentIndex--; + this.addActive(); + } else if (e.keyCode === 13) { // enter + e.preventDefault(); + if (this.currentIndex > -1) { + // Simulate a click on the "active" item: + if (this.autocompleteList) this.autocompleteList.children[this.currentIndex].click(); + } + } else if (e.keyCode === 27) { // escape + this.removeAllItems(e.target); + } + }); + + document.addEventListener('click', (e) => { this.removeAllItems(e.target); }); + } + + updateLists() { + const inputValue = this.inputBox.value; + + const tokens = inputValue.split(); + const lastToken = tokens[tokens.length - 1]; + const lastChar = lastToken[lastToken.length - 1]; + + /* close any already open lists of autocompleted values */ + this.removeAllItems(); + + this.currentIndex = -1; + + const suggestions = this.suggestions.filter(s => (s.indexOf(lastToken) >= 0 || lastChar === ' ')); + + suggestions.slice(0, 10).forEach(suggestion => { + const itemDiv = document.createElement('div'); + if (lastChar === ' ') { + itemDiv.innerHTML = suggestion; + } else { + const indexOfLastToken = suggestion.indexOf(lastToken); + + itemDiv.innerHTML = suggestion.substr(0, indexOfLastToken) + + '' + + suggestion.substr(indexOfLastToken, lastToken.length) + + '' + + suggestion.substr( + indexOfLastToken + lastToken.length, suggestion.length - (lastToken.length - 2) + ); + + } + + itemDiv.setAttribute('data-value', suggestion); + + const suggestionClick = (e) => { + const toInsert = e.target.getAttribute('data-value'); + + const oldValue = this.inputBox.value; + const tokens = oldValue.split(); + const lastToken = tokens[tokens.length - 1]; + const lastChar = lastToken[lastToken.length - 1]; + + let newValue = ''; + + if (lastChar === ' ' || oldValue === '') { + newValue = oldValue + toInsert; + } else { + // const position = this.inputBox.selectionStart; + const queryWithoutLastToken = tokens.slice(0, tokens.length - 2).join(' '); + newValue = queryWithoutLastToken + ((queryWithoutLastToken !== '') ? ' ' : '') + toInsert; + } + + this.inputBox.value = newValue; + this.inputBox.blur(); + this.inputBox.focus(); + // this.inputBox.dispatchEvent(new Event('input')) + }; + + itemDiv.addEventListener('click', suggestionClick.bind(this)); + + this.autocompleteList.appendChild(itemDiv); + }); + + if (suggestions?.length) { + // Select first element on each update + this.currentIndex = 0; + this.addActive(); + } + } + + addActive() { + // a function to classify an item as "active": + if (!this.autocompleteList) return false; + // start by removing the "active" class on all items: + const n = this.autocompleteList.childElementCount; + this.removeActive(); + if (this.currentIndex >= n) this.currentIndex = 0; + if (this.currentIndex < 0) this.currentIndex = (n - 1); + // add class "autocomplete-active": + this.autocompleteList.children[this.currentIndex].classList.add('autocomplete-active'); + } + + removeActive() { + /* a function to remove the "active" class from all autocomplete items */ + Array.from(this.autocompleteList.children).forEach(autocompleteItem => { + autocompleteItem.classList.remove('autocomplete-active'); + }); + } + + removeAllItems(element) { + /* + close all autocomplete lists in the document, + except the one passed as an argument + */ + if (element !== this.inputBox && this.autocompleteList) { + this.autocompleteList.innerHTML = ''; + } + } + +} diff --git a/assets/src/utils/search-ql-autocomplete.js b/assets/src/utils/search-ql-autocomplete.js new file mode 100644 --- /dev/null +++ b/assets/src/utils/search-ql-autocomplete.js @@ -0,0 +1,224 @@ +import '../../../node_modules/web-tree-sitter/tree-sitter.wasm'; +import {Parser} from 'web-tree-sitter'; +import {Autocomplete} from 'utils/autocomplete.js'; +import { + fields, limitField, sortByField, // fields + sortByOptions, visitTypeOptions, // options + equalOp, rangeOp, choiceOp, // operators + AND, OR, TRUE, FALSE // special tokens +} from '../../../../swh-search/query_language/tokens.js'; + +const filterNames = fields.concat(sortByField, limitField); + +const languageSyntax = [ + { + category: 'patternFilter', + field: 'patternField', + operator: 'equalOp', + value: 'patternVal', + suggestion: ['string', '"string"'] + }, + { + category: 'booleanFilter', + field: 'booleanField', + operator: 'equalOp', + value: 'booleanVal', + suggestion: [TRUE, FALSE] + }, + { + category: 'numericFilter', + field: 'numericField', + operator: 'rangeOp', + value: 'numberVal', + suggestion: ['15'] + }, + { + category: 'boundedListFilter', + field: 'visitTypeField', + operator: 'equalOp', + value: 'visitTypeVal', + options: visitTypeOptions, + suggestion: ['['] + }, + { + category: 'unboundedListFilter', + field: 'listField', + operator: 'choiceOp', + value: 'listVal', + options: ['string', '"string"'], + suggestion: ['['] + }, + { + category: 'dateFilter', + field: 'dateField', + operator: 'rangeOp', + value: 'dateVal', + suggestion: ['2000-01-01', '2000-01-01T00:00Z'] + }, + { + category: 'sortBy', + field: 'sortByField', + operator: 'equalOp', + value: 'sortByVal', + options: sortByOptions, + suggestion: ['['] + }, + { + category: 'limit', + field: 'limit', + operator: 'equalOp', + value: 'number', + suggestion: ['50'] + } +]; + +const filterOperators = {equalOp, choiceOp, rangeOp}; + +const findMissingNode = (node) => { + if (node.isMissing()) { + return node; + } + if (node.children.length > 0) { + for (let i = 0; i < node.children.length; i++) { + const missingNode = findMissingNode(node.children[i]); + if (missingNode !== null) { return missingNode; } + } + } + + return null; +}; + +const isWrapperNode = (child, parent) => { + if (!child || !parent) return false; + if (parent.namedChildren.length === 1 && parent.type !== 'ERROR') return true; + return ( + (child.startPosition.column === parent.startPosition.column) && + (child.endPosition.column === parent.endPosition.column) + ); +}; + +const isCategoryNode = (node) => { + if (!node || node === null) return false; + if (node.type === 'ERROR' || languageSyntax.filter(f => f.category === node.type).length > 0) { return true; } + + return false; +}; + +const suggestNextNode = (tree, inputBox) => { + const cursor = inputBox.selectionStart - 1; + const query = inputBox.value; + + let lastTokenIndex = cursor; + // let distFromLastToken = 0; + while (query[lastTokenIndex] === ' ') { + lastTokenIndex--; + // distFromLastToken++; + } + + // if(query === "visit_type = []") debugger; + + const lastTokenPosition = {row: 0, column: lastTokenIndex}; + const lastTokenNode = tree.rootNode.descendantForPosition(lastTokenPosition, lastTokenPosition); + + const missingNode = findMissingNode(tree.rootNode); + + // Find last token node wrapper + let lastTokenNodeWrapper = lastTokenNode; + while (isWrapperNode(lastTokenNodeWrapper, lastTokenNodeWrapper.parent)) { + lastTokenNodeWrapper = lastTokenNodeWrapper.parent; + } + + // Find last token node wrapper sibling + const lastTokenNodeWrapperSibling = lastTokenNodeWrapper.previousSibling; + + // Find current filter category + let currentFilterCategory = lastTokenNode; + while (!isCategoryNode(currentFilterCategory)) { + currentFilterCategory = currentFilterCategory.parent; + } + + console.log(lastTokenNode); + console.log(`LAST NODE: ${lastTokenNode.type}`); + console.log(`LAST NODE ANCESTOR: ${lastTokenNodeWrapper.type}`); + console.log(`LAST NODE ANCESTOR SIBLING: ${lastTokenNodeWrapperSibling?.type}`); + console.log(`LAST CATEGORY: ${currentFilterCategory.type}`); + + // Suggest options for array valued filters + if ((lastTokenNode.type === ',' && lastTokenNodeWrapper.type.indexOf('Val') > 0) || + (lastTokenNode.type === '[' && currentFilterCategory) + ) { + const filter = languageSyntax.filter(f => f.category === currentFilterCategory.type)[0]; + console.log(filter.options); + return filter.options ?? []; + } + if ( + (!tree.rootNode.hasError() && (lastTokenNodeWrapper.type.indexOf('Val') > 0)) || + (lastTokenNode.type === ')' || lastTokenNode.type === ']') + ) { + // Suggest AND/OR + return [AND, OR]; + } + if (missingNode && missingNode !== null) { + // Suggest missing nodes (Automatically suggested by Tree-sitter) + if (missingNode.type === ')') { + return [AND, OR, ')']; + } else if (missingNode.type === ']') { + return [',', ']']; + } + } + + if (lastTokenNode.type === 'ERROR' || + (lastTokenNode.type === '(') || + ((lastTokenNode.type === AND || lastTokenNode.type === OR)) + ) { + // Suggest field names + return filterNames.concat('('); + } else if (languageSyntax.map(f => f.field).includes(lastTokenNode.type)) { + // Suggest operators + const filter = languageSyntax.filter(f => f.field === lastTokenNode.type)[0]; + return filterOperators[filter.operator]; + } else if (lastTokenNode.type in filterOperators) { + // Suggest values + const filter = languageSyntax.filter(f => ( + f.field === lastTokenNodeWrapperSibling.type + ))[0]; + return filter.suggestion; + } + + return []; +}; + +export const initAutocomplete = (inputBox, validQueryCallback) => { + Parser.init().then(async() => { + const parser = new Parser(); + + const swhSearchQL = await Parser.Language.load(`${window.location.origin}/static/swh_ql.wasm`); + parser.setLanguage(swhSearchQL); + + const autocomplete = new Autocomplete( + {inputBox, suggestions: ['('].concat(filterNames)} + ); + + const getSuggestions = (e) => { + // if (e.keycode !== 32) // space + // return; + const tree = parser.parse(inputBox.value); + + if (tree.rootNode.hasError()) { + validQueryCallback(false); + // inputBox.classList.add('invalid'); + } else { + validQueryCallback(true); + // inputBox.classList.remove('invalid'); + } + + console.log(`input(${inputBox.value}) => ${tree.rootNode.toString()}`); + + const suggestions = suggestNextNode(tree, inputBox); + // if (suggestions) + autocomplete.suggestions = suggestions; // .map(item => `${item} `); + }; + + inputBox.addEventListener('keydown', getSuggestions.bind(this)); + }); +}; diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -90,6 +90,7 @@ "typeface-alegreya": "^1.1.13", "typeface-alegreya-sans": "^1.1.13", "waypoints": "^4.0.1", + "web-tree-sitter": "^0.19.4", "whatwg-fetch": "^3.6.2" }, "devDependencies": { diff --git a/swh/__init__.py b/swh/__init__.py --- a/swh/__init__.py +++ b/swh/__init__.py @@ -1,3 +1,3 @@ from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +__path__ = extend_path(__path__, __name__) # type: ignore diff --git a/swh/web/__init__.py b/swh/web/__init__.py --- a/swh/web/__init__.py +++ b/swh/web/__init__.py @@ -1,3 +1,3 @@ from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +__path__ = extend_path(__path__, __name__) # type: ignore diff --git a/swh/web/config.py b/swh/web/config.py --- a/swh/web/config.py +++ b/swh/web/config.py @@ -44,7 +44,8 @@ ), "search_config": ( "dict", - {"metadata_backend": "swh-indexer-storage",}, # or "swh-search" + {"metadata_backend": "swh-indexer-storage", "enable_ql": True}, + # or {"metadata_backend" : "swh-search", ...} ), "log_dir": ("string", "/tmp/swh/log"), "debug": ("bool", False), diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -12,6 +12,7 @@ import sys from typing import Any, Dict +from swh import search from swh.web.auth.utils import OIDC_SWH_WEB_CLIENT_ID from swh.web.config import get_config @@ -135,7 +136,15 @@ if not os.path.exists(STATIC_DIR): # static folder location when developping swh-web STATIC_DIR = os.path.join(PROJECT_DIR, "../../../static") -STATICFILES_DIRS = [STATIC_DIR] + +SEARCH_DIR = os.path.dirname(search.__file__) +# static folder location when swh-search has been installed with pip +SEARCH_STATIC_DIR = os.path.join(SEARCH_DIR, "static") +if not os.path.exists(SEARCH_STATIC_DIR): + # static folder location when developping swh-search + SEARCH_STATIC_DIR = os.path.join(SEARCH_DIR, "../../static") +STATICFILES_DIRS = [STATIC_DIR, SEARCH_STATIC_DIR] + INTERNAL_IPS = ["127.0.0.1"] diff --git a/swh/web/templates/includes/origin-search-form.html b/swh/web/templates/includes/origin-search-form.html --- a/swh/web/templates/includes/origin-search-form.html +++ b/swh/web/templates/includes/origin-search-form.html @@ -10,9 +10,10 @@ + oninput="swh.webapp.validateSWHIDInput(this)" + autofocus required autocomplete="off">
- +
diff --git a/yarn.lock b/yarn.lock --- a/yarn.lock +++ b/yarn.lock @@ -11184,6 +11184,11 @@ dependencies: minimalistic-assert "^1.0.0" +web-tree-sitter@^0.19.4: + version "0.19.4" + resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.19.4.tgz#975076e233204de9063e7a7bda1138c4b454b424" + integrity sha512-8G0xBj05hqZybCqBtW7RPZ/hWEtP3DiLTauQzGJZuZYfVRgw7qj7iaZ+8djNqJ4VPrdOO+pS2dR1JsTbsLxdYg== + webidl-conversions@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff"