Page MenuHomeSoftware Heritage
Files F9340926

No OneTemporary
Actions

View Options

diff --git a/Makefile.local b/Makefile.local
index dfe7d439..5a89d791 100644
--- a/Makefile.local
+++ b/Makefile.local
@@ -1,125 +1,125 @@
TEST_DIRS := ./swh/web/tests
TESTFLAGS = --hypothesis-profile=swh-web-fast
TESTFULL_FLAGS = --hypothesis-profile=swh-web
YARN ?= yarn
SETTINGS_TEST ?= swh.web.settings.tests
SETTINGS_DEV ?= swh.web.settings.development
SETTINGS_PROD = swh.web.settings.production
yarn-install: package.json
- $(YARN) install
+ $(YARN) install --frozen-lockfile
.PHONY: build-webpack-dev
build-webpack-dev: yarn-install
$(YARN) build-dev
.PHONY: build-webpack-test
build-webpack-test: yarn-install
$(YARN) build-test
.PHONY: build-webpack-dev-no-verbose
build-webpack-dev-no-verbose: yarn-install
$(YARN) build-dev >/dev/null
.PHONY: build-webpack-prod
build-webpack-prod: yarn-install
$(YARN) build
.PHONY: run-migrations-dev
run-migrations-dev:
python3 swh/web/manage.py migrate --settings=$(SETTINGS_DEV) -v0 2>/dev/null
.PHONY: run-migrations-prod
run-migrations-prod:
django-admin migrate --settings=$(SETTINGS_PROD) -v0 2>/dev/null
.PHONY: run-migrations-test
run-migrations-test:
rm -f swh-web-test.sqlite3
django-admin migrate --settings=$(SETTINGS_TEST) -v0 2>/dev/null
add-users-test: run-migrations-test
cat swh/web/tests/create_test_admin.py | django-admin shell --settings=$(SETTINGS_TEST)
cat swh/web/tests/create_test_users.py | django-admin shell --settings=$(SETTINGS_TEST)
add-users-dev: run-migrations-dev
cat swh/web/tests/create_test_admin.py | django-admin shell --settings=$(SETTINGS_DEV)
cat swh/web/tests/create_test_users.py | django-admin shell --settings=$(SETTINGS_DEV)
add-users-prod: run-migrations-prod
cat swh/web/tests/create_test_admin.py | django-admin shell --settings=$(SETTINGS_PROD)
cat swh/web/tests/create_test_users.py | django-admin shell --settings=$(SETTINGS_PROD)
.PHONY: clear-memcached
clear-memcached:
echo "flush_all" | nc -q 2 localhost 11211 2>/dev/null
run-django-webpack-devserver: add-users-dev yarn-install
bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT && \
# ensure all child processes will be killed by PGID when exiting \
ps -o pgid= $$$$ | grep -o [0-9]* | xargs pkill -g' SIGINT SIGTERM ERR EXIT; \
$(YARN) start-dev & sleep 10 && cd swh/web && \
python3 manage.py runserver --nostatic --settings=$(SETTINGS_DEV) || exit 1"
run-django-webpack-dev: build-webpack-dev add-users-dev
python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_DEV)
run-django-webpack-prod: build-webpack-prod add-users-prod clear-memcached
python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_PROD)
run-django-server-dev: add-users-dev
python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_DEV)
run-django-server-prod: add-users-prod clear-memcached
python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_PROD)
run-gunicorn-server: add-users-prod clear-memcached
DJANGO_SETTINGS_MODULE=$(SETTINGS_PROD) \
gunicorn --bind 127.0.0.1:5004 \
--threads 2 \
--workers 2 'django.core.wsgi:get_wsgi_application()'
run-django-webpack-memory-storages: build-webpack-dev add-users-test
python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_TEST)
test-full:
$(TEST) $(TESTFULL_FLAGS) $(TEST_DIRS)
.PHONY: test-frontend-cmd
test-frontend-cmd: build-webpack-test add-users-test
bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT && \
jobs -p | xargs -r kill' SIGINT SIGTERM ERR EXIT; \
python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_TEST) & \
sleep 10 && $(YARN) run cypress run --config numTestsKeptInMemory=0 && \
$(YARN) mochawesome && $(YARN) nyc-report"
test-frontend: export CYPRESS_SKIP_SLOW_TESTS=1
test-frontend: test-frontend-cmd
test-frontend-full: export CYPRESS_SKIP_SLOW_TESTS=0
test-frontend-full: test-frontend-cmd
.PHONY: test-frontend-ui-cmd
test-frontend-ui-cmd: add-users-test yarn-install
# ensure all child processes will be killed when hitting Ctrl-C in terminal
# or manually closing the Cypress UI window, killing by PGID seems the only
# reliable way to do it in that case
bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT && \
ps -o pgid= $$$$ | grep -o [0-9]* | xargs pkill -g' SIGINT SIGTERM ERR EXIT; \
$(YARN) start-dev & \
python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_TEST) & \
sleep 10 && $(YARN) run cypress open"
test-frontend-ui: export CYPRESS_SKIP_SLOW_TESTS=1
test-frontend-ui: test-frontend-ui-cmd
test-frontend-full-ui: export CYPRESS_SKIP_SLOW_TESTS=0
test-frontend-full-ui: test-frontend-ui-cmd
# Override default rule to make sure DJANGO env var is properly set. It
# *should* work without any override thanks to the mypy django-stubs plugin,
# but it currently doesn't; see
# https://github.com/typeddjango/django-stubs/issues/166
check-mypy:
DJANGO_SETTINGS_MODULE=$(SETTINGS_DEV) $(MYPY) $(MYPYFLAGS) swh
diff --git a/PKG-INFO b/PKG-INFO
index 54a66efe..99f58111 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,206 +1,206 @@
Metadata-Version: 2.1
Name: swh.web
-Version: 0.0.351
+Version: 0.0.352
Summary: Software Heritage Web UI
Home-page: https://forge.softwareheritage.org/diffusion/DWUI/
Author: Software Heritage developers
Author-email: swh-devel@inria.fr
License: UNKNOWN
Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
Project-URL: Funding, https://www.softwareheritage.org/donate
Project-URL: Source, https://forge.softwareheritage.org/source/swh-web
Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-web/
Platform: UNKNOWN
Classifier: Programming Language :: Python :: 3
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
Classifier: Operating System :: OS Independent
Classifier: Development Status :: 5 - Production/Stable
Classifier: Framework :: Django
Requires-Python: >=3.7
Description-Content-Type: text/markdown
Provides-Extra: testing
License-File: LICENSE
License-File: AUTHORS
# swh-web
This repository holds the development of Software Heritage web applications:
* swh-web API (https://archive.softwareheritage.org/api): enables to query the
content of the archive through HTTP requests and get responses in JSON or
YAML.
* swh-web browse (https://archive.softwareheritage.org/browse): graphical
interface that eases the navigation in the archive.
Documentation about how to use these components but also the details of their
URI schemes can be found in the docs folder. The produced HTML documentation
can be read and browsed at
https://docs.softwareheritage.org/devel/swh-web/index.html.
## Technical details
Those applications are powered by:
* [Django Web Framework](https://www.djangoproject.com/) on the backend side
with the following extensions enabled:
* [django-rest-framework](http://www.django-rest-framework.org/)
* [django-webpack-loader](https://github.com/owais/django-webpack-loader)
* [django-js-reverse](http://django-js-reverse.readthedocs.io/en/latest/)
* [webpack](https://webpack.js.org/) on the frontend side for better static
assets management, including:
* assets dependencies management and retrieval through
[yarn](https://yarnpkg.com/en/)
* linting of custom javascript code (through [eslint](https://eslint.org/))
and stylesheets (through [stylelint](https://stylelint.io/))
* use of [es6](http://es6-features.org) syntax and advanced javascript
feature like [async/await](https://javascript.info/async-await) or
[fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)
thanks to [babel](https://babeljs.io/) (es6 to es5 transpiler and
polyfills provider)
* assets minification (using [terser](https://github.com/terser-js/terser)
and [cssnano](http://cssnano.co/)) but also dead code elimination for
production use
## How to build, run and test
### Backend requirements
First you will need [Python 3](https://www.python.org) and a complete [swh
development
environment](https://forge.softwareheritage.org/source/swh-environment/)
installed.
To run the backend, you need to have the following [Python 3
modules](requirements.txt) installed.
To run the backend tests, the following [Python 3
modules](requirements-test.txt) are also required to be installed.
One easy way to install them is to use the `pip` tool:
```
$ pip install -r requirements.txt -r requirements-test.txt
```
### Frontend requirements
To compile the frontend assets, you need to have
[nodejs](https://nodejs.org/en/) >= 12.0.0 and [yarn](https://yarnpkg.com/en/)
installed. If you are on Debian, you can easily install an up to date nodejs
from the
[nodesource](https://github.com/nodesource/distributions/blob/master/README.md)
repository.
Packages for yarn can be installed by following [these
instructions](https://yarnpkg.com/en/docs/install#debian-stable).
Alternatively, you can install yarn with `npm install yarn`, and add
`YARN=node_modules/yarn/bin/yarn` as argument whenever you run `make`.
Please note that the static assets bundles generated by webpack are not stored
in the git repository. Follow the instructions below in order to generate them
in order to be able to run the frontend part of the web applications.
### Make targets to execute the applications
Below is the list of available make targets that can be executed from the root
directory of swh-web in order to build and/or execute the web applications
under various configurations:
* **run-django-webpack-devserver**: Compile and serve not optimized (without
mignification and dead code elimination) frontend static assets using
[webpack-dev-server](https://github.com/webpack/webpack-dev-server) and run
django server with development settings. This is the recommended target to
use when developing swh-web as it enables automatic reloading of backend and
frontend part of the applications when modifying source files (*.py, *.js,
*.css, *.html).
* **run-django-webpack-dev**: Compile not optimized (no minification, no dead
code elimination) frontend static assets using webpack and run django server
with development settings. This is the recommended target when one only wants
to develop the backend side of the application.
* **run-django-webpack-prod**: Compile optimized (with minification and dead
code elimination) frontend static assets using webpack and run django server
with production settings. This is useful to test the applications in
production mode (with the difference that static assets are served by
django). Production settings notably enable advanced django caching and you
will need to have [memcached](https://memcached.org/) installed for that
feature to work.
* **run-django-server-dev**: Run the django server with development settings
but without compiling frontend static assets through webpack.
* **run-django-server-prod**: Run the django server with production settings
but without compiling frontend static assets through webpack.
* **run-gunicorn-server**: Run the web applications with production settings in
a [gunicorn](http://gunicorn.org/) worker as they will be in real production
environment.
Once one of these targets executed, the web applications can be executed by
pointing your browser to http://localhost:5004.
### Make targets to test the applications
Some make targets are also available to easily execute the backend and frontend
tests of the Software Heritage web applications.
The backend tests are powered by the
[pytest](https://docs.pytest.org/en/latest/) and
[hypothesis](https://hypothesis.readthedocs.io/en/latest/) frameworks while the
frontend ones rely on the use of the [cypress](https://www.cypress.io/) tool.
Below is the exhaustive list of those targets:
* **test**: execute the backend tests using a fast hypothesis profile (only one
input example will be provided for each test)
* **test-full**: execute the backend tests using a slower hypothesis profile
(one hundred of input examples will be provided for each test which helps
spotting possible bugs)
* **test-frontend**: execute the frontend tests using cypress in headless mode
but with some slow test suites disabled
* **test-frontend-full**: execute the frontend tests using cypress in headless
mode with all test suites enabled
* **test-frontend-ui**: execute the frontend tests using the cypress GUI but
with some slow test suites disabled
* **test-frontend-full-ui**: execute the frontend tests using the cypress GUI
with all test suites enabled
### Yarn targets
Below is a list of available yarn targets in order to only execute the frontend
static assets compilation (no web server will be executed):
* **build-dev**: compile not optimized (without mignification and dead code
elimination) frontend static assets and store the results in the
`swh/web/static` folder.
* **build**: compile optimized (with mignification and dead code elimination)
frontend static assets and store the results in the `swh/web/static` folder.
**The build target must be executed prior performing the Debian packaging of
swh-web** in order for the package to contain the optimized assets dedicated to
production environment.
To execute these targets, issue the following command:
```
$ yarn <target_name>
```
diff --git a/static/webpack-stats.json b/static/webpack-stats.json
index 80b45c10..a5b7342c 100644
--- a/static/webpack-stats.json
+++ b/static/webpack-stats.json
@@ -1,771 +1,771 @@
{
"status": "done",
"assets": {
"img/thirdParty/chosen-sprite.png": {
"name": "img/thirdParty/chosen-sprite.png",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/img/thirdParty/chosen-sprite.png",
"publicPath": "/static/img/thirdParty/chosen-sprite.png"
},
"img/thirdParty/chosen-sprite@2x.png": {
"name": "img/thirdParty/chosen-sprite@2x.png",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/img/thirdParty/chosen-sprite@2x.png",
"publicPath": "/static/img/thirdParty/chosen-sprite@2x.png"
},
"fonts/alegreya-latin-400.woff2": {
"name": "fonts/alegreya-latin-400.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-400.woff2",
"publicPath": "/static/fonts/alegreya-latin-400.woff2"
},
"fonts/alegreya-latin-400.woff": {
"name": "fonts/alegreya-latin-400.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-400.woff",
"publicPath": "/static/fonts/alegreya-latin-400.woff"
},
"fonts/alegreya-latin-400italic.woff2": {
"name": "fonts/alegreya-latin-400italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-400italic.woff2",
"publicPath": "/static/fonts/alegreya-latin-400italic.woff2"
},
"fonts/alegreya-latin-400italic.woff": {
"name": "fonts/alegreya-latin-400italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-400italic.woff",
"publicPath": "/static/fonts/alegreya-latin-400italic.woff"
},
"fonts/alegreya-latin-500.woff2": {
"name": "fonts/alegreya-latin-500.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-500.woff2",
"publicPath": "/static/fonts/alegreya-latin-500.woff2"
},
"fonts/alegreya-latin-500.woff": {
"name": "fonts/alegreya-latin-500.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-500.woff",
"publicPath": "/static/fonts/alegreya-latin-500.woff"
},
"fonts/alegreya-latin-500italic.woff2": {
"name": "fonts/alegreya-latin-500italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-500italic.woff2",
"publicPath": "/static/fonts/alegreya-latin-500italic.woff2"
},
"fonts/alegreya-latin-500italic.woff": {
"name": "fonts/alegreya-latin-500italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-500italic.woff",
"publicPath": "/static/fonts/alegreya-latin-500italic.woff"
},
"fonts/alegreya-latin-700.woff2": {
"name": "fonts/alegreya-latin-700.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-700.woff2",
"publicPath": "/static/fonts/alegreya-latin-700.woff2"
},
"fonts/alegreya-latin-700.woff": {
"name": "fonts/alegreya-latin-700.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-700.woff",
"publicPath": "/static/fonts/alegreya-latin-700.woff"
},
- "fonts/alegreya-latin-700italic.woff2": {
- "name": "fonts/alegreya-latin-700italic.woff2",
- "path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-700italic.woff2",
- "publicPath": "/static/fonts/alegreya-latin-700italic.woff2"
- },
"fonts/alegreya-latin-700italic.woff": {
"name": "fonts/alegreya-latin-700italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-700italic.woff",
"publicPath": "/static/fonts/alegreya-latin-700italic.woff"
},
+ "fonts/alegreya-latin-700italic.woff2": {
+ "name": "fonts/alegreya-latin-700italic.woff2",
+ "path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-700italic.woff2",
+ "publicPath": "/static/fonts/alegreya-latin-700italic.woff2"
+ },
"fonts/alegreya-latin-800.woff2": {
"name": "fonts/alegreya-latin-800.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-800.woff2",
"publicPath": "/static/fonts/alegreya-latin-800.woff2"
},
"fonts/alegreya-latin-800.woff": {
"name": "fonts/alegreya-latin-800.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-800.woff",
"publicPath": "/static/fonts/alegreya-latin-800.woff"
},
"fonts/alegreya-latin-800italic.woff2": {
"name": "fonts/alegreya-latin-800italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-800italic.woff2",
"publicPath": "/static/fonts/alegreya-latin-800italic.woff2"
},
"fonts/alegreya-latin-800italic.woff": {
"name": "fonts/alegreya-latin-800italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-800italic.woff",
"publicPath": "/static/fonts/alegreya-latin-800italic.woff"
},
"fonts/alegreya-latin-900.woff2": {
"name": "fonts/alegreya-latin-900.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-900.woff2",
"publicPath": "/static/fonts/alegreya-latin-900.woff2"
},
"fonts/alegreya-latin-900.woff": {
"name": "fonts/alegreya-latin-900.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-900.woff",
"publicPath": "/static/fonts/alegreya-latin-900.woff"
},
"fonts/alegreya-latin-900italic.woff2": {
"name": "fonts/alegreya-latin-900italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-900italic.woff2",
"publicPath": "/static/fonts/alegreya-latin-900italic.woff2"
},
"fonts/alegreya-latin-900italic.woff": {
"name": "fonts/alegreya-latin-900italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-latin-900italic.woff",
"publicPath": "/static/fonts/alegreya-latin-900italic.woff"
},
"fonts/alegreya-sans-latin-100.woff2": {
"name": "fonts/alegreya-sans-latin-100.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-100.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-100.woff2"
},
- "fonts/alegreya-sans-latin-100.woff": {
- "name": "fonts/alegreya-sans-latin-100.woff",
- "path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-100.woff",
- "publicPath": "/static/fonts/alegreya-sans-latin-100.woff"
- },
"fonts/alegreya-sans-latin-100italic.woff2": {
"name": "fonts/alegreya-sans-latin-100italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-100italic.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-100italic.woff2"
},
+ "fonts/alegreya-sans-latin-100.woff": {
+ "name": "fonts/alegreya-sans-latin-100.woff",
+ "path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-100.woff",
+ "publicPath": "/static/fonts/alegreya-sans-latin-100.woff"
+ },
"fonts/alegreya-sans-latin-100italic.woff": {
"name": "fonts/alegreya-sans-latin-100italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-100italic.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-100italic.woff"
},
"fonts/alegreya-sans-latin-300.woff2": {
"name": "fonts/alegreya-sans-latin-300.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-300.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-300.woff2"
},
"fonts/alegreya-sans-latin-300.woff": {
"name": "fonts/alegreya-sans-latin-300.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-300.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-300.woff"
},
"fonts/alegreya-sans-latin-300italic.woff2": {
"name": "fonts/alegreya-sans-latin-300italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-300italic.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-300italic.woff2"
},
"fonts/alegreya-sans-latin-300italic.woff": {
"name": "fonts/alegreya-sans-latin-300italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-300italic.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-300italic.woff"
},
"fonts/alegreya-sans-latin-400.woff2": {
"name": "fonts/alegreya-sans-latin-400.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-400.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-400.woff2"
},
"fonts/alegreya-sans-latin-400.woff": {
"name": "fonts/alegreya-sans-latin-400.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-400.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-400.woff"
},
"fonts/alegreya-sans-latin-400italic.woff2": {
"name": "fonts/alegreya-sans-latin-400italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-400italic.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-400italic.woff2"
},
"fonts/alegreya-sans-latin-400italic.woff": {
"name": "fonts/alegreya-sans-latin-400italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-400italic.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-400italic.woff"
},
"fonts/alegreya-sans-latin-500.woff2": {
"name": "fonts/alegreya-sans-latin-500.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-500.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-500.woff2"
},
"fonts/alegreya-sans-latin-500.woff": {
"name": "fonts/alegreya-sans-latin-500.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-500.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-500.woff"
},
"fonts/alegreya-sans-latin-500italic.woff2": {
"name": "fonts/alegreya-sans-latin-500italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-500italic.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-500italic.woff2"
},
"fonts/alegreya-sans-latin-500italic.woff": {
"name": "fonts/alegreya-sans-latin-500italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-500italic.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-500italic.woff"
},
"fonts/alegreya-sans-latin-700.woff2": {
"name": "fonts/alegreya-sans-latin-700.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-700.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-700.woff2"
},
"fonts/alegreya-sans-latin-700.woff": {
"name": "fonts/alegreya-sans-latin-700.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-700.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-700.woff"
},
"fonts/alegreya-sans-latin-700italic.woff2": {
"name": "fonts/alegreya-sans-latin-700italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-700italic.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-700italic.woff2"
},
"fonts/alegreya-sans-latin-700italic.woff": {
"name": "fonts/alegreya-sans-latin-700italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-700italic.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-700italic.woff"
},
"fonts/alegreya-sans-latin-800.woff2": {
"name": "fonts/alegreya-sans-latin-800.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-800.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-800.woff2"
},
"fonts/alegreya-sans-latin-800.woff": {
"name": "fonts/alegreya-sans-latin-800.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-800.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-800.woff"
},
"fonts/alegreya-sans-latin-800italic.woff2": {
"name": "fonts/alegreya-sans-latin-800italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-800italic.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-800italic.woff2"
},
"fonts/alegreya-sans-latin-800italic.woff": {
"name": "fonts/alegreya-sans-latin-800italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-800italic.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-800italic.woff"
},
"fonts/alegreya-sans-latin-900.woff2": {
"name": "fonts/alegreya-sans-latin-900.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-900.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-900.woff2"
},
"fonts/alegreya-sans-latin-900.woff": {
"name": "fonts/alegreya-sans-latin-900.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-900.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-900.woff"
},
"fonts/alegreya-sans-latin-900italic.woff2": {
"name": "fonts/alegreya-sans-latin-900italic.woff2",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-900italic.woff2",
"publicPath": "/static/fonts/alegreya-sans-latin-900italic.woff2"
},
"fonts/alegreya-sans-latin-900italic.woff": {
"name": "fonts/alegreya-sans-latin-900italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/alegreya-sans-latin-900italic.woff",
"publicPath": "/static/fonts/alegreya-sans-latin-900italic.woff"
},
"fonts/materialdesignicons-webfont.eot?v=6.5.95": {
"name": "fonts/materialdesignicons-webfont.eot?v=6.5.95",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/materialdesignicons-webfont.eot",
"publicPath": "/static/fonts/materialdesignicons-webfont.eot?v=6.5.95"
},
"fonts/materialdesignicons-webfont.eot": {
"name": "fonts/materialdesignicons-webfont.eot",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/materialdesignicons-webfont.eot",
"publicPath": "/static/fonts/materialdesignicons-webfont.eot"
},
"fonts/materialdesignicons-webfont.woff2?v=6.5.95": {
"name": "fonts/materialdesignicons-webfont.woff2?v=6.5.95",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/materialdesignicons-webfont.woff2",
"publicPath": "/static/fonts/materialdesignicons-webfont.woff2?v=6.5.95"
},
"fonts/materialdesignicons-webfont.woff?v=6.5.95": {
"name": "fonts/materialdesignicons-webfont.woff?v=6.5.95",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/materialdesignicons-webfont.woff",
"publicPath": "/static/fonts/materialdesignicons-webfont.woff?v=6.5.95"
},
"fonts/materialdesignicons-webfont.ttf?v=6.5.95": {
"name": "fonts/materialdesignicons-webfont.ttf?v=6.5.95",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/materialdesignicons-webfont.ttf",
"publicPath": "/static/fonts/materialdesignicons-webfont.ttf?v=6.5.95"
},
+ "js/pdf.worker.min.js": {
+ "name": "js/pdf.worker.min.js",
+ "path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/pdf.worker.min.js",
+ "publicPath": "/static/js/pdf.worker.min.js"
+ },
"fonts/MathJax_AMS-Regular.woff": {
"name": "fonts/MathJax_AMS-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_AMS-Regular.woff",
"publicPath": "/static/fonts/MathJax_AMS-Regular.woff"
},
"fonts/MathJax_Calligraphic-Bold.woff": {
"name": "fonts/MathJax_Calligraphic-Bold.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Calligraphic-Bold.woff",
"publicPath": "/static/fonts/MathJax_Calligraphic-Bold.woff"
},
"fonts/MathJax_Calligraphic-Regular.woff": {
"name": "fonts/MathJax_Calligraphic-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Calligraphic-Regular.woff",
"publicPath": "/static/fonts/MathJax_Calligraphic-Regular.woff"
},
"fonts/MathJax_Fraktur-Bold.woff": {
"name": "fonts/MathJax_Fraktur-Bold.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Fraktur-Bold.woff",
"publicPath": "/static/fonts/MathJax_Fraktur-Bold.woff"
},
"fonts/MathJax_Fraktur-Regular.woff": {
"name": "fonts/MathJax_Fraktur-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Fraktur-Regular.woff",
"publicPath": "/static/fonts/MathJax_Fraktur-Regular.woff"
},
"fonts/MathJax_Main-Bold.woff": {
"name": "fonts/MathJax_Main-Bold.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Main-Bold.woff",
"publicPath": "/static/fonts/MathJax_Main-Bold.woff"
},
"fonts/MathJax_Main-Italic.woff": {
"name": "fonts/MathJax_Main-Italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Main-Italic.woff",
"publicPath": "/static/fonts/MathJax_Main-Italic.woff"
},
"fonts/MathJax_Main-Regular.woff": {
"name": "fonts/MathJax_Main-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Main-Regular.woff",
"publicPath": "/static/fonts/MathJax_Main-Regular.woff"
},
"fonts/MathJax_Math-BoldItalic.woff": {
"name": "fonts/MathJax_Math-BoldItalic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Math-BoldItalic.woff",
"publicPath": "/static/fonts/MathJax_Math-BoldItalic.woff"
},
"fonts/MathJax_Math-Italic.woff": {
"name": "fonts/MathJax_Math-Italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Math-Italic.woff",
"publicPath": "/static/fonts/MathJax_Math-Italic.woff"
},
"fonts/MathJax_Math-Regular.woff": {
"name": "fonts/MathJax_Math-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Math-Regular.woff",
"publicPath": "/static/fonts/MathJax_Math-Regular.woff"
},
"fonts/MathJax_SansSerif-Bold.woff": {
"name": "fonts/MathJax_SansSerif-Bold.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_SansSerif-Bold.woff",
"publicPath": "/static/fonts/MathJax_SansSerif-Bold.woff"
},
"fonts/MathJax_SansSerif-Italic.woff": {
"name": "fonts/MathJax_SansSerif-Italic.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_SansSerif-Italic.woff",
"publicPath": "/static/fonts/MathJax_SansSerif-Italic.woff"
},
"fonts/MathJax_SansSerif-Regular.woff": {
"name": "fonts/MathJax_SansSerif-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_SansSerif-Regular.woff",
"publicPath": "/static/fonts/MathJax_SansSerif-Regular.woff"
},
"fonts/MathJax_Script-Regular.woff": {
"name": "fonts/MathJax_Script-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Script-Regular.woff",
"publicPath": "/static/fonts/MathJax_Script-Regular.woff"
},
"fonts/MathJax_Size1-Regular.woff": {
"name": "fonts/MathJax_Size1-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Size1-Regular.woff",
"publicPath": "/static/fonts/MathJax_Size1-Regular.woff"
},
"fonts/MathJax_Size2-Regular.woff": {
"name": "fonts/MathJax_Size2-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Size2-Regular.woff",
"publicPath": "/static/fonts/MathJax_Size2-Regular.woff"
},
"fonts/MathJax_Size3-Regular.woff": {
"name": "fonts/MathJax_Size3-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Size3-Regular.woff",
"publicPath": "/static/fonts/MathJax_Size3-Regular.woff"
},
"fonts/MathJax_Size4-Regular.woff": {
"name": "fonts/MathJax_Size4-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Size4-Regular.woff",
"publicPath": "/static/fonts/MathJax_Size4-Regular.woff"
},
"fonts/MathJax_Typewriter-Regular.woff": {
"name": "fonts/MathJax_Typewriter-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Typewriter-Regular.woff",
"publicPath": "/static/fonts/MathJax_Typewriter-Regular.woff"
},
"fonts/MathJax_Vector-Bold.woff": {
"name": "fonts/MathJax_Vector-Bold.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Vector-Bold.woff",
"publicPath": "/static/fonts/MathJax_Vector-Bold.woff"
},
"fonts/MathJax_Vector-Regular.woff": {
"name": "fonts/MathJax_Vector-Regular.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Vector-Regular.woff",
"publicPath": "/static/fonts/MathJax_Vector-Regular.woff"
},
"fonts/MathJax_Zero.woff": {
"name": "fonts/MathJax_Zero.woff",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/fonts/MathJax_Zero.woff",
"publicPath": "/static/fonts/MathJax_Zero.woff"
},
- "js/pdf.worker.min.js": {
- "name": "js/pdf.worker.min.js",
- "path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/pdf.worker.min.js",
- "publicPath": "/static/js/pdf.worker.min.js"
- },
"robots.txt": {
"name": "robots.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/robots.txt",
"publicPath": "/static/robots.txt"
},
"js/pdf.worker.min.js.LICENSE.txt": {
"name": "js/pdf.worker.min.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/pdf.worker.min.js.LICENSE.txt",
"publicPath": "/static/js/pdf.worker.min.js.LICENSE.txt"
},
"js/admin.dd251562dae83eb019a5.js.LICENSE.txt": {
"name": "js/admin.dd251562dae83eb019a5.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/admin.dd251562dae83eb019a5.js.LICENSE.txt",
"publicPath": "/static/js/admin.dd251562dae83eb019a5.js.LICENSE.txt"
},
"js/auth.7381ff7d9581af98cd05.js.LICENSE.txt": {
"name": "js/auth.7381ff7d9581af98cd05.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/auth.7381ff7d9581af98cd05.js.LICENSE.txt",
"publicPath": "/static/js/auth.7381ff7d9581af98cd05.js.LICENSE.txt"
},
"js/browse.9a45216360db35746a75.js.LICENSE.txt": {
"name": "js/browse.9a45216360db35746a75.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/browse.9a45216360db35746a75.js.LICENSE.txt",
"publicPath": "/static/js/browse.9a45216360db35746a75.js.LICENSE.txt"
},
"js/guided_tour.9734aa27598765424c60.js.LICENSE.txt": {
"name": "js/guided_tour.9734aa27598765424c60.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/guided_tour.9734aa27598765424c60.js.LICENSE.txt",
"publicPath": "/static/js/guided_tour.9734aa27598765424c60.js.LICENSE.txt"
},
"js/revision.1179b4c6429b56108c09.js.LICENSE.txt": {
"name": "js/revision.1179b4c6429b56108c09.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/revision.1179b4c6429b56108c09.js.LICENSE.txt",
"publicPath": "/static/js/revision.1179b4c6429b56108c09.js.LICENSE.txt"
},
"js/save.c2c6031565cbb8c1e19f.js.LICENSE.txt": {
"name": "js/save.c2c6031565cbb8c1e19f.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/save.c2c6031565cbb8c1e19f.js.LICENSE.txt",
"publicPath": "/static/js/save.c2c6031565cbb8c1e19f.js.LICENSE.txt"
},
"js/showdown.c33d171d25f3025a070e.js.LICENSE.txt": {
"name": "js/showdown.c33d171d25f3025a070e.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/showdown.c33d171d25f3025a070e.js.LICENSE.txt",
"publicPath": "/static/js/showdown.c33d171d25f3025a070e.js.LICENSE.txt"
},
"js/vault.a5a1c49ab66115f4fbb2.js.LICENSE.txt": {
"name": "js/vault.a5a1c49ab66115f4fbb2.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/vault.a5a1c49ab66115f4fbb2.js.LICENSE.txt",
"publicPath": "/static/js/vault.a5a1c49ab66115f4fbb2.js.LICENSE.txt"
},
"js/vendors.c6ea77f6ad5100283c95.js.LICENSE.txt": {
"name": "js/vendors.c6ea77f6ad5100283c95.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/vendors.c6ea77f6ad5100283c95.js.LICENSE.txt",
"publicPath": "/static/js/vendors.c6ea77f6ad5100283c95.js.LICENSE.txt"
},
"js/webapp.92fe87aa4aea9253ec8b.js.LICENSE.txt": {
"name": "js/webapp.92fe87aa4aea9253ec8b.js.LICENSE.txt",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/webapp.92fe87aa4aea9253ec8b.js.LICENSE.txt",
"publicPath": "/static/js/webapp.92fe87aa4aea9253ec8b.js.LICENSE.txt"
},
"js/admin.dd251562dae83eb019a5.js": {
"name": "js/admin.dd251562dae83eb019a5.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/admin.dd251562dae83eb019a5.js",
"publicPath": "/static/js/admin.dd251562dae83eb019a5.js"
},
"css/auth.0336a94c2c02b4b2a4f4.css": {
"name": "css/auth.0336a94c2c02b4b2a4f4.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/auth.0336a94c2c02b4b2a4f4.css",
"publicPath": "/static/css/auth.0336a94c2c02b4b2a4f4.css"
},
"js/auth.7381ff7d9581af98cd05.js": {
"name": "js/auth.7381ff7d9581af98cd05.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/auth.7381ff7d9581af98cd05.js",
"publicPath": "/static/js/auth.7381ff7d9581af98cd05.js"
},
"css/browse.6315ef52ed73df532bed.css": {
"name": "css/browse.6315ef52ed73df532bed.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/browse.6315ef52ed73df532bed.css",
"publicPath": "/static/css/browse.6315ef52ed73df532bed.css"
},
"js/browse.9a45216360db35746a75.js": {
"name": "js/browse.9a45216360db35746a75.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/browse.9a45216360db35746a75.js",
"publicPath": "/static/js/browse.9a45216360db35746a75.js"
},
"css/guided_tour.1de8b53f719cea9f465a.css": {
"name": "css/guided_tour.1de8b53f719cea9f465a.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/guided_tour.1de8b53f719cea9f465a.css",
"publicPath": "/static/css/guided_tour.1de8b53f719cea9f465a.css"
},
"js/guided_tour.9734aa27598765424c60.js": {
"name": "js/guided_tour.9734aa27598765424c60.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/guided_tour.9734aa27598765424c60.js",
"publicPath": "/static/js/guided_tour.9734aa27598765424c60.js"
},
"css/origin.6d38fe9d3dc3ac901b6d.css": {
"name": "css/origin.6d38fe9d3dc3ac901b6d.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/origin.6d38fe9d3dc3ac901b6d.css",
"publicPath": "/static/css/origin.6d38fe9d3dc3ac901b6d.css"
},
"js/origin.d7e76b2c1bd67100c6ef.js": {
"name": "js/origin.d7e76b2c1bd67100c6ef.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/origin.d7e76b2c1bd67100c6ef.js",
"publicPath": "/static/js/origin.d7e76b2c1bd67100c6ef.js"
},
"css/revision.4085c2cec17d77feb2b7.css": {
"name": "css/revision.4085c2cec17d77feb2b7.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/revision.4085c2cec17d77feb2b7.css",
"publicPath": "/static/css/revision.4085c2cec17d77feb2b7.css"
},
"js/revision.1179b4c6429b56108c09.js": {
"name": "js/revision.1179b4c6429b56108c09.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/revision.1179b4c6429b56108c09.js",
"publicPath": "/static/js/revision.1179b4c6429b56108c09.js"
},
"js/save.c2c6031565cbb8c1e19f.js": {
"name": "js/save.c2c6031565cbb8c1e19f.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/save.c2c6031565cbb8c1e19f.js",
"publicPath": "/static/js/save.c2c6031565cbb8c1e19f.js"
},
"css/vault.25fc5883f848b48ffa5b.css": {
"name": "css/vault.25fc5883f848b48ffa5b.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/vault.25fc5883f848b48ffa5b.css",
"publicPath": "/static/css/vault.25fc5883f848b48ffa5b.css"
},
"js/vault.a5a1c49ab66115f4fbb2.js": {
"name": "js/vault.a5a1c49ab66115f4fbb2.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/vault.a5a1c49ab66115f4fbb2.js",
"publicPath": "/static/js/vault.a5a1c49ab66115f4fbb2.js"
},
"css/vendors.f3faa32e236cf3b45872.css": {
"name": "css/vendors.f3faa32e236cf3b45872.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/vendors.f3faa32e236cf3b45872.css",
"publicPath": "/static/css/vendors.f3faa32e236cf3b45872.css"
},
"js/vendors.c6ea77f6ad5100283c95.js": {
"name": "js/vendors.c6ea77f6ad5100283c95.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/vendors.c6ea77f6ad5100283c95.js",
"publicPath": "/static/js/vendors.c6ea77f6ad5100283c95.js"
},
"css/webapp.556dd4db8176c80d6de7.css": {
"name": "css/webapp.556dd4db8176c80d6de7.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/webapp.556dd4db8176c80d6de7.css",
"publicPath": "/static/css/webapp.556dd4db8176c80d6de7.css"
},
"js/webapp.92fe87aa4aea9253ec8b.js": {
"name": "js/webapp.92fe87aa4aea9253ec8b.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/webapp.92fe87aa4aea9253ec8b.js",
"publicPath": "/static/js/webapp.92fe87aa4aea9253ec8b.js"
},
"js/d3.9fbc8c4f808d6305db8c.js": {
"name": "js/d3.9fbc8c4f808d6305db8c.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/d3.9fbc8c4f808d6305db8c.js",
"publicPath": "/static/js/d3.9fbc8c4f808d6305db8c.js"
},
"css/highlightjs.ae43064ab38a65a04d81.css": {
"name": "css/highlightjs.ae43064ab38a65a04d81.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/highlightjs.ae43064ab38a65a04d81.css",
"publicPath": "/static/css/highlightjs.ae43064ab38a65a04d81.css"
},
"js/highlightjs.3ab3417de241df41fb53.js": {
"name": "js/highlightjs.3ab3417de241df41fb53.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/highlightjs.3ab3417de241df41fb53.js",
"publicPath": "/static/js/highlightjs.3ab3417de241df41fb53.js"
},
"css/showdown.426fbf6a7a6653fd4cbb.css": {
"name": "css/showdown.426fbf6a7a6653fd4cbb.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/showdown.426fbf6a7a6653fd4cbb.css",
"publicPath": "/static/css/showdown.426fbf6a7a6653fd4cbb.css"
},
"js/showdown.c33d171d25f3025a070e.js": {
"name": "js/showdown.c33d171d25f3025a070e.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/showdown.c33d171d25f3025a070e.js",
"publicPath": "/static/js/showdown.c33d171d25f3025a070e.js"
},
"css/org.6851b70c924e28f6bf51.css": {
"name": "css/org.6851b70c924e28f6bf51.css",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/org.6851b70c924e28f6bf51.css",
"publicPath": "/static/css/org.6851b70c924e28f6bf51.css"
},
"js/org.27a44f19269b37fa5fe6.js": {
"name": "js/org.27a44f19269b37fa5fe6.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/org.27a44f19269b37fa5fe6.js",
"publicPath": "/static/js/org.27a44f19269b37fa5fe6.js"
},
"js/pdfjs.0de3de7f552746c41e31.js": {
"name": "js/pdfjs.0de3de7f552746c41e31.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/pdfjs.0de3de7f552746c41e31.js",
"publicPath": "/static/js/pdfjs.0de3de7f552746c41e31.js"
},
"js/mathjax.82b6ebf86da778cadb7d.js": {
"name": "js/mathjax.82b6ebf86da778cadb7d.js",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/mathjax.82b6ebf86da778cadb7d.js",
"publicPath": "/static/js/mathjax.82b6ebf86da778cadb7d.js"
},
"css/auth.0336a94c2c02b4b2a4f4.css.map": {
"name": "css/auth.0336a94c2c02b4b2a4f4.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/auth.0336a94c2c02b4b2a4f4.css.map",
"publicPath": "/static/css/auth.0336a94c2c02b4b2a4f4.css.map"
},
"css/browse.6315ef52ed73df532bed.css.map": {
"name": "css/browse.6315ef52ed73df532bed.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/browse.6315ef52ed73df532bed.css.map",
"publicPath": "/static/css/browse.6315ef52ed73df532bed.css.map"
},
"css/guided_tour.1de8b53f719cea9f465a.css.map": {
"name": "css/guided_tour.1de8b53f719cea9f465a.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/guided_tour.1de8b53f719cea9f465a.css.map",
"publicPath": "/static/css/guided_tour.1de8b53f719cea9f465a.css.map"
},
- "css/origin.6d38fe9d3dc3ac901b6d.css.map": {
- "name": "css/origin.6d38fe9d3dc3ac901b6d.css.map",
- "path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/origin.6d38fe9d3dc3ac901b6d.css.map",
- "publicPath": "/static/css/origin.6d38fe9d3dc3ac901b6d.css.map"
- },
"css/revision.4085c2cec17d77feb2b7.css.map": {
"name": "css/revision.4085c2cec17d77feb2b7.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/revision.4085c2cec17d77feb2b7.css.map",
"publicPath": "/static/css/revision.4085c2cec17d77feb2b7.css.map"
},
+ "css/origin.6d38fe9d3dc3ac901b6d.css.map": {
+ "name": "css/origin.6d38fe9d3dc3ac901b6d.css.map",
+ "path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/origin.6d38fe9d3dc3ac901b6d.css.map",
+ "publicPath": "/static/css/origin.6d38fe9d3dc3ac901b6d.css.map"
+ },
"css/vault.25fc5883f848b48ffa5b.css.map": {
"name": "css/vault.25fc5883f848b48ffa5b.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/vault.25fc5883f848b48ffa5b.css.map",
"publicPath": "/static/css/vault.25fc5883f848b48ffa5b.css.map"
},
"css/vendors.f3faa32e236cf3b45872.css.map": {
"name": "css/vendors.f3faa32e236cf3b45872.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/vendors.f3faa32e236cf3b45872.css.map",
"publicPath": "/static/css/vendors.f3faa32e236cf3b45872.css.map"
},
"css/webapp.556dd4db8176c80d6de7.css.map": {
"name": "css/webapp.556dd4db8176c80d6de7.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/webapp.556dd4db8176c80d6de7.css.map",
"publicPath": "/static/css/webapp.556dd4db8176c80d6de7.css.map"
},
"css/highlightjs.ae43064ab38a65a04d81.css.map": {
"name": "css/highlightjs.ae43064ab38a65a04d81.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/highlightjs.ae43064ab38a65a04d81.css.map",
"publicPath": "/static/css/highlightjs.ae43064ab38a65a04d81.css.map"
},
"css/showdown.426fbf6a7a6653fd4cbb.css.map": {
"name": "css/showdown.426fbf6a7a6653fd4cbb.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/showdown.426fbf6a7a6653fd4cbb.css.map",
"publicPath": "/static/css/showdown.426fbf6a7a6653fd4cbb.css.map"
},
"css/org.6851b70c924e28f6bf51.css.map": {
"name": "css/org.6851b70c924e28f6bf51.css.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/css/org.6851b70c924e28f6bf51.css.map",
"publicPath": "/static/css/org.6851b70c924e28f6bf51.css.map"
},
"js/admin.dd251562dae83eb019a5.js.map": {
"name": "js/admin.dd251562dae83eb019a5.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/admin.dd251562dae83eb019a5.js.map",
"publicPath": "/static/js/admin.dd251562dae83eb019a5.js.map"
},
"js/auth.7381ff7d9581af98cd05.js.map": {
"name": "js/auth.7381ff7d9581af98cd05.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/auth.7381ff7d9581af98cd05.js.map",
"publicPath": "/static/js/auth.7381ff7d9581af98cd05.js.map"
},
"js/browse.9a45216360db35746a75.js.map": {
"name": "js/browse.9a45216360db35746a75.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/browse.9a45216360db35746a75.js.map",
"publicPath": "/static/js/browse.9a45216360db35746a75.js.map"
},
"js/guided_tour.9734aa27598765424c60.js.map": {
"name": "js/guided_tour.9734aa27598765424c60.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/guided_tour.9734aa27598765424c60.js.map",
"publicPath": "/static/js/guided_tour.9734aa27598765424c60.js.map"
},
"js/origin.d7e76b2c1bd67100c6ef.js.map": {
"name": "js/origin.d7e76b2c1bd67100c6ef.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/origin.d7e76b2c1bd67100c6ef.js.map",
"publicPath": "/static/js/origin.d7e76b2c1bd67100c6ef.js.map"
},
"js/revision.1179b4c6429b56108c09.js.map": {
"name": "js/revision.1179b4c6429b56108c09.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/revision.1179b4c6429b56108c09.js.map",
"publicPath": "/static/js/revision.1179b4c6429b56108c09.js.map"
},
"js/save.c2c6031565cbb8c1e19f.js.map": {
"name": "js/save.c2c6031565cbb8c1e19f.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/save.c2c6031565cbb8c1e19f.js.map",
"publicPath": "/static/js/save.c2c6031565cbb8c1e19f.js.map"
},
"js/vault.a5a1c49ab66115f4fbb2.js.map": {
"name": "js/vault.a5a1c49ab66115f4fbb2.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/vault.a5a1c49ab66115f4fbb2.js.map",
"publicPath": "/static/js/vault.a5a1c49ab66115f4fbb2.js.map"
},
"js/vendors.c6ea77f6ad5100283c95.js.map": {
"name": "js/vendors.c6ea77f6ad5100283c95.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/vendors.c6ea77f6ad5100283c95.js.map",
"publicPath": "/static/js/vendors.c6ea77f6ad5100283c95.js.map"
},
"js/webapp.92fe87aa4aea9253ec8b.js.map": {
"name": "js/webapp.92fe87aa4aea9253ec8b.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/webapp.92fe87aa4aea9253ec8b.js.map",
"publicPath": "/static/js/webapp.92fe87aa4aea9253ec8b.js.map"
},
"js/d3.9fbc8c4f808d6305db8c.js.map": {
"name": "js/d3.9fbc8c4f808d6305db8c.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/d3.9fbc8c4f808d6305db8c.js.map",
"publicPath": "/static/js/d3.9fbc8c4f808d6305db8c.js.map"
},
"js/highlightjs.3ab3417de241df41fb53.js.map": {
"name": "js/highlightjs.3ab3417de241df41fb53.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/highlightjs.3ab3417de241df41fb53.js.map",
"publicPath": "/static/js/highlightjs.3ab3417de241df41fb53.js.map"
},
"js/showdown.c33d171d25f3025a070e.js.map": {
"name": "js/showdown.c33d171d25f3025a070e.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/showdown.c33d171d25f3025a070e.js.map",
"publicPath": "/static/js/showdown.c33d171d25f3025a070e.js.map"
},
"js/org.27a44f19269b37fa5fe6.js.map": {
"name": "js/org.27a44f19269b37fa5fe6.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/org.27a44f19269b37fa5fe6.js.map",
"publicPath": "/static/js/org.27a44f19269b37fa5fe6.js.map"
},
"js/pdfjs.0de3de7f552746c41e31.js.map": {
"name": "js/pdfjs.0de3de7f552746c41e31.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/pdfjs.0de3de7f552746c41e31.js.map",
"publicPath": "/static/js/pdfjs.0de3de7f552746c41e31.js.map"
},
"js/mathjax.82b6ebf86da778cadb7d.js.map": {
"name": "js/mathjax.82b6ebf86da778cadb7d.js.map",
"path": "/var/lib/jenkins/workspace/DWAPPS/pypi-upload/static/js/mathjax.82b6ebf86da778cadb7d.js.map",
"publicPath": "/static/js/mathjax.82b6ebf86da778cadb7d.js.map"
}
},
"chunks": {
"admin": [
"js/admin.dd251562dae83eb019a5.js"
],
"auth": [
"css/auth.0336a94c2c02b4b2a4f4.css",
"js/auth.7381ff7d9581af98cd05.js"
],
"browse": [
"css/browse.6315ef52ed73df532bed.css",
"js/browse.9a45216360db35746a75.js"
],
"guided_tour": [
"css/guided_tour.1de8b53f719cea9f465a.css",
"js/guided_tour.9734aa27598765424c60.js"
],
"origin": [
"css/origin.6d38fe9d3dc3ac901b6d.css",
"js/origin.d7e76b2c1bd67100c6ef.js"
],
"revision": [
"css/revision.4085c2cec17d77feb2b7.css",
"js/revision.1179b4c6429b56108c09.js"
],
"save": [
"js/save.c2c6031565cbb8c1e19f.js"
],
"vault": [
"css/vault.25fc5883f848b48ffa5b.css",
"js/vault.a5a1c49ab66115f4fbb2.js"
],
"vendors": [
"css/vendors.f3faa32e236cf3b45872.css",
"js/vendors.c6ea77f6ad5100283c95.js"
],
"webapp": [
"css/webapp.556dd4db8176c80d6de7.css",
"js/webapp.92fe87aa4aea9253ec8b.js"
]
},
"publicPath": "/static/"
}
\ No newline at end of file
diff --git a/swh.web.egg-info/PKG-INFO b/swh.web.egg-info/PKG-INFO
index 54a66efe..99f58111 100644
--- a/swh.web.egg-info/PKG-INFO
+++ b/swh.web.egg-info/PKG-INFO
@@ -1,206 +1,206 @@
Metadata-Version: 2.1
Name: swh.web
-Version: 0.0.351
+Version: 0.0.352
Summary: Software Heritage Web UI
Home-page: https://forge.softwareheritage.org/diffusion/DWUI/
Author: Software Heritage developers
Author-email: swh-devel@inria.fr
License: UNKNOWN
Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest
Project-URL: Funding, https://www.softwareheritage.org/donate
Project-URL: Source, https://forge.softwareheritage.org/source/swh-web
Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-web/
Platform: UNKNOWN
Classifier: Programming Language :: Python :: 3
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
Classifier: Operating System :: OS Independent
Classifier: Development Status :: 5 - Production/Stable
Classifier: Framework :: Django
Requires-Python: >=3.7
Description-Content-Type: text/markdown
Provides-Extra: testing
License-File: LICENSE
License-File: AUTHORS
# swh-web
This repository holds the development of Software Heritage web applications:
* swh-web API (https://archive.softwareheritage.org/api): enables to query the
content of the archive through HTTP requests and get responses in JSON or
YAML.
* swh-web browse (https://archive.softwareheritage.org/browse): graphical
interface that eases the navigation in the archive.
Documentation about how to use these components but also the details of their
URI schemes can be found in the docs folder. The produced HTML documentation
can be read and browsed at
https://docs.softwareheritage.org/devel/swh-web/index.html.
## Technical details
Those applications are powered by:
* [Django Web Framework](https://www.djangoproject.com/) on the backend side
with the following extensions enabled:
* [django-rest-framework](http://www.django-rest-framework.org/)
* [django-webpack-loader](https://github.com/owais/django-webpack-loader)
* [django-js-reverse](http://django-js-reverse.readthedocs.io/en/latest/)
* [webpack](https://webpack.js.org/) on the frontend side for better static
assets management, including:
* assets dependencies management and retrieval through
[yarn](https://yarnpkg.com/en/)
* linting of custom javascript code (through [eslint](https://eslint.org/))
and stylesheets (through [stylelint](https://stylelint.io/))
* use of [es6](http://es6-features.org) syntax and advanced javascript
feature like [async/await](https://javascript.info/async-await) or
[fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)
thanks to [babel](https://babeljs.io/) (es6 to es5 transpiler and
polyfills provider)
* assets minification (using [terser](https://github.com/terser-js/terser)
and [cssnano](http://cssnano.co/)) but also dead code elimination for
production use
## How to build, run and test
### Backend requirements
First you will need [Python 3](https://www.python.org) and a complete [swh
development
environment](https://forge.softwareheritage.org/source/swh-environment/)
installed.
To run the backend, you need to have the following [Python 3
modules](requirements.txt) installed.
To run the backend tests, the following [Python 3
modules](requirements-test.txt) are also required to be installed.
One easy way to install them is to use the `pip` tool:
```
$ pip install -r requirements.txt -r requirements-test.txt
```
### Frontend requirements
To compile the frontend assets, you need to have
[nodejs](https://nodejs.org/en/) >= 12.0.0 and [yarn](https://yarnpkg.com/en/)
installed. If you are on Debian, you can easily install an up to date nodejs
from the
[nodesource](https://github.com/nodesource/distributions/blob/master/README.md)
repository.
Packages for yarn can be installed by following [these
instructions](https://yarnpkg.com/en/docs/install#debian-stable).
Alternatively, you can install yarn with `npm install yarn`, and add
`YARN=node_modules/yarn/bin/yarn` as argument whenever you run `make`.
Please note that the static assets bundles generated by webpack are not stored
in the git repository. Follow the instructions below in order to generate them
in order to be able to run the frontend part of the web applications.
### Make targets to execute the applications
Below is the list of available make targets that can be executed from the root
directory of swh-web in order to build and/or execute the web applications
under various configurations:
* **run-django-webpack-devserver**: Compile and serve not optimized (without
mignification and dead code elimination) frontend static assets using
[webpack-dev-server](https://github.com/webpack/webpack-dev-server) and run
django server with development settings. This is the recommended target to
use when developing swh-web as it enables automatic reloading of backend and
frontend part of the applications when modifying source files (*.py, *.js,
*.css, *.html).
* **run-django-webpack-dev**: Compile not optimized (no minification, no dead
code elimination) frontend static assets using webpack and run django server
with development settings. This is the recommended target when one only wants
to develop the backend side of the application.
* **run-django-webpack-prod**: Compile optimized (with minification and dead
code elimination) frontend static assets using webpack and run django server
with production settings. This is useful to test the applications in
production mode (with the difference that static assets are served by
django). Production settings notably enable advanced django caching and you
will need to have [memcached](https://memcached.org/) installed for that
feature to work.
* **run-django-server-dev**: Run the django server with development settings
but without compiling frontend static assets through webpack.
* **run-django-server-prod**: Run the django server with production settings
but without compiling frontend static assets through webpack.
* **run-gunicorn-server**: Run the web applications with production settings in
a [gunicorn](http://gunicorn.org/) worker as they will be in real production
environment.
Once one of these targets executed, the web applications can be executed by
pointing your browser to http://localhost:5004.
### Make targets to test the applications
Some make targets are also available to easily execute the backend and frontend
tests of the Software Heritage web applications.
The backend tests are powered by the
[pytest](https://docs.pytest.org/en/latest/) and
[hypothesis](https://hypothesis.readthedocs.io/en/latest/) frameworks while the
frontend ones rely on the use of the [cypress](https://www.cypress.io/) tool.
Below is the exhaustive list of those targets:
* **test**: execute the backend tests using a fast hypothesis profile (only one
input example will be provided for each test)
* **test-full**: execute the backend tests using a slower hypothesis profile
(one hundred of input examples will be provided for each test which helps
spotting possible bugs)
* **test-frontend**: execute the frontend tests using cypress in headless mode
but with some slow test suites disabled
* **test-frontend-full**: execute the frontend tests using cypress in headless
mode with all test suites enabled
* **test-frontend-ui**: execute the frontend tests using the cypress GUI but
with some slow test suites disabled
* **test-frontend-full-ui**: execute the frontend tests using the cypress GUI
with all test suites enabled
### Yarn targets
Below is a list of available yarn targets in order to only execute the frontend
static assets compilation (no web server will be executed):
* **build-dev**: compile not optimized (without mignification and dead code
elimination) frontend static assets and store the results in the
`swh/web/static` folder.
* **build**: compile optimized (with mignification and dead code elimination)
frontend static assets and store the results in the `swh/web/static` folder.
**The build target must be executed prior performing the Debian packaging of
swh-web** in order for the package to contain the optimized assets dedicated to
production environment.
To execute these targets, issue the following command:
```
$ yarn <target_name>
```
diff --git a/swh/web/browse/snapshot_context.py b/swh/web/browse/snapshot_context.py
index c5be4d75..148b29bc 100644
--- a/swh/web/browse/snapshot_context.py
+++ b/swh/web/browse/snapshot_context.py
@@ -1,1330 +1,1330 @@
# Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
# Utility module for browsing the archive in a snapshot context.
from collections import defaultdict
from typing import Any, Dict, List, Optional, Tuple
from django.core.cache import cache
from django.shortcuts import render
from django.utils.html import escape
from swh.model.hashutil import hash_to_bytes
from swh.model.model import Snapshot
from swh.model.swhids import CoreSWHID, ObjectType
from swh.web.browse.utils import (
format_log_entries,
gen_release_link,
gen_revision_link,
gen_revision_log_link,
gen_revision_url,
gen_snapshot_link,
get_directory_entries,
get_readme_to_display,
)
from swh.web.common import archive
from swh.web.common.exc import BadInputExc, NotFoundExc, http_status_code_message
from swh.web.common.identifiers import get_swhids_info
from swh.web.common.origin_visits import get_origin_visit
from swh.web.common.typing import (
DirectoryMetadata,
OriginInfo,
SnapshotBranchInfo,
SnapshotContext,
SnapshotReleaseInfo,
SWHObjectInfo,
)
from swh.web.common.utils import (
format_utc_iso_date,
gen_path_info,
reverse,
swh_object_icons,
)
from swh.web.config import get_config
_empty_snapshot_id = Snapshot(branches={}).id.hex()
def _get_branch(branches, branch_name, snapshot_id):
"""
Utility function to get a specific branch from a snapshot.
Returns None if the branch cannot be found.
"""
filtered_branches = [b for b in branches if b["name"] == branch_name]
if filtered_branches:
return filtered_branches[0]
else:
# case where a large branches list has been truncated
snp = archive.lookup_snapshot(
snapshot_id,
branches_from=branch_name,
branches_count=1,
target_types=["revision", "alias"],
# pull request branches must be browsable even if they are hidden
# by default in branches list
branch_name_exclude_prefix=None,
)
snp_branch, _, _ = process_snapshot_branches(snp)
if snp_branch and snp_branch[0]["name"] == branch_name:
branches.append(snp_branch[0])
return snp_branch[0]
def _get_release(releases, release_name, snapshot_id):
"""
Utility function to get a specific release from a snapshot.
Returns None if the release cannot be found.
"""
filtered_releases = [r for r in releases if r["name"] == release_name]
if filtered_releases:
return filtered_releases[0]
else:
# case where a large branches list has been truncated
try:
# git origins have specific branches for releases
snp = archive.lookup_snapshot(
snapshot_id,
branches_from=f"refs/tags/{release_name}",
branches_count=1,
target_types=["release"],
)
except NotFoundExc:
snp = archive.lookup_snapshot(
snapshot_id,
branches_from=release_name,
branches_count=1,
target_types=["release", "alias"],
)
_, snp_release, _ = process_snapshot_branches(snp)
if snp_release and snp_release[0]["name"] == release_name:
releases.append(snp_release[0])
return snp_release[0]
def _branch_not_found(
branch_type, branch, snapshot_id, snapshot_sizes, origin_info, timestamp, visit_id
):
"""
Utility function to raise an exception when a specified branch/release
can not be found.
"""
if branch_type == "branch":
branch_type = "Branch"
branch_type_plural = "branches"
target_type = "revision"
else:
branch_type = "Release"
branch_type_plural = "releases"
target_type = "release"
if snapshot_id and snapshot_sizes[target_type] == 0:
msg = "Snapshot with id %s has an empty list" " of %s!" % (
snapshot_id,
branch_type_plural,
)
elif snapshot_id:
msg = "%s %s for snapshot with id %s" " not found!" % (
branch_type,
branch,
snapshot_id,
)
elif visit_id and snapshot_sizes[target_type] == 0:
msg = (
"Origin with url %s"
" for visit with id %s has an empty list"
" of %s!" % (origin_info["url"], visit_id, branch_type_plural)
)
elif visit_id:
msg = (
"%s %s associated to visit with"
" id %s for origin with url %s"
" not found!" % (branch_type, branch, visit_id, origin_info["url"])
)
elif snapshot_sizes[target_type] == 0:
msg = (
"Origin with url %s"
" for visit with timestamp %s has an empty list"
" of %s!" % (origin_info["url"], timestamp, branch_type_plural)
)
else:
msg = (
"%s %s associated to visit with"
" timestamp %s for origin with "
"url %s not found!" % (branch_type, branch, timestamp, origin_info["url"])
)
raise NotFoundExc(escape(msg))
def process_snapshot_branches(
snapshot: Dict[str, Any]
) -> Tuple[List[SnapshotBranchInfo], List[SnapshotReleaseInfo], Dict[str, Any]]:
"""
Process a dictionary describing snapshot branches: extract those
targeting revisions and releases, put them in two different lists,
then sort those lists in lexicographical order of the branches' names.
Args:
snapshot: A dict describing a snapshot as returned for instance by
:func:`swh.web.common.archive.lookup_snapshot`
Returns:
A tuple whose first member is the sorted list of branches
targeting revisions, second member the sorted list of branches
targeting releases and third member a dict mapping resolved branch
aliases to their real target.
"""
snapshot_branches = snapshot["branches"]
branches: Dict[str, SnapshotBranchInfo] = {}
branch_aliases: Dict[str, str] = {}
releases: Dict[str, SnapshotReleaseInfo] = {}
revision_to_branch = defaultdict(set)
revision_to_release = defaultdict(set)
release_to_branch = defaultdict(set)
for branch_name, target in snapshot_branches.items():
if not target:
# FIXME: display branches with an unknown target anyway
continue
target_id = target["target"]
target_type = target["target_type"]
if target_type == "revision":
branches[branch_name] = SnapshotBranchInfo(
name=branch_name,
alias=False,
revision=target_id,
date=None,
directory=None,
message=None,
url=None,
)
revision_to_branch[target_id].add(branch_name)
elif target_type == "release":
release_to_branch[target_id].add(branch_name)
elif target_type == "alias":
branch_aliases[branch_name] = target_id
# FIXME: handle pointers to other object types
def _add_release_info(branch, release, alias=False):
releases[branch] = SnapshotReleaseInfo(
name=release["name"],
alias=alias,
branch_name=branch,
date=format_utc_iso_date(release["date"]),
directory=None,
id=release["id"],
message=release["message"],
target_type=release["target_type"],
target=release["target"],
url=None,
)
def _add_branch_info(branch, revision, alias=False):
branches[branch] = SnapshotBranchInfo(
name=branch,
alias=alias,
revision=revision["id"],
directory=revision["directory"],
date=format_utc_iso_date(revision["date"]),
message=revision["message"],
url=None,
)
releases_info = archive.lookup_release_multiple(release_to_branch.keys())
for release in releases_info:
if release is None:
continue
branches_to_update = release_to_branch[release["id"]]
for branch in branches_to_update:
_add_release_info(branch, release)
if release["target_type"] == "revision":
revision_to_release[release["target"]].update(branches_to_update)
revisions = archive.lookup_revision_multiple(
set(revision_to_branch.keys()) | set(revision_to_release.keys())
)
for revision in revisions:
if not revision:
continue
for branch in revision_to_branch[revision["id"]]:
_add_branch_info(branch, revision)
for release_id in revision_to_release[revision["id"]]:
releases[release_id]["directory"] = revision["directory"]
resolved_aliases = {}
for branch_alias, branch_target in branch_aliases.items():
resolved_alias = archive.lookup_snapshot_alias(snapshot["id"], branch_alias)
resolved_aliases[branch_alias] = resolved_alias
if resolved_alias is None:
continue
target_type = resolved_alias["target_type"]
target = resolved_alias["target"]
if target_type == "revision":
revision = archive.lookup_revision(target)
_add_branch_info(branch_alias, revision, alias=True)
elif target_type == "release":
release = archive.lookup_release(target)
_add_release_info(branch_alias, release, alias=True)
if branch_alias in branches:
branches[branch_alias]["name"] = branch_alias
ret_branches = list(sorted(branches.values(), key=lambda b: b["name"]))
ret_releases = list(sorted(releases.values(), key=lambda b: b["name"]))
return ret_branches, ret_releases, resolved_aliases
def get_snapshot_content(
snapshot_id: str,
) -> Tuple[List[SnapshotBranchInfo], List[SnapshotReleaseInfo], Dict[str, Any]]:
"""Returns the lists of branches and releases
associated to a swh snapshot.
That list is put in cache in order to speedup the navigation
in the swh-web/browse ui.
.. warning:: At most 1000 branches contained in the snapshot
will be returned for performance reasons.
Args:
snapshot_id: hexadecimal representation of the snapshot identifier
Returns:
A tuple with three members. The first one is a list of dict describing
the snapshot branches. The second one is a list of dict describing the
snapshot releases. The third one is a dict mapping resolved branch
aliases to their real target.
Raises:
NotFoundExc if the snapshot does not exist
"""
cache_entry_id = "swh_snapshot_%s" % snapshot_id
cache_entry = cache.get(cache_entry_id)
if cache_entry:
return (
cache_entry["branches"],
cache_entry["releases"],
cache_entry.get("aliases", {}),
)
branches: List[SnapshotBranchInfo] = []
releases: List[SnapshotReleaseInfo] = []
aliases: Dict[str, Any] = {}
snapshot_content_max_size = get_config()["snapshot_content_max_size"]
if snapshot_id:
snapshot = archive.lookup_snapshot(
snapshot_id, branches_count=snapshot_content_max_size
)
branches, releases, aliases = process_snapshot_branches(snapshot)
cache.set(
cache_entry_id, {"branches": branches, "releases": releases, "aliases": aliases}
)
return branches, releases, aliases
def get_origin_visit_snapshot(
origin_info: OriginInfo,
visit_ts: Optional[str] = None,
visit_id: Optional[int] = None,
snapshot_id: Optional[str] = None,
) -> Tuple[List[SnapshotBranchInfo], List[SnapshotReleaseInfo], Dict[str, Any]]:
"""Returns the lists of branches and releases associated to an origin for
a given visit.
The visit is expressed by either:
* a snapshot identifier
* a timestamp, if no visit with that exact timestamp is found,
the closest one from the provided timestamp will be used.
If no visit parameter is provided, it returns the list of branches
found for the latest visit.
That list is put in cache in order to speedup the navigation
in the swh-web/browse ui.
.. warning:: At most 1000 branches contained in the snapshot
will be returned for performance reasons.
Args:
origin_info: a dict filled with origin information
visit_ts: an ISO 8601 datetime string to parse
visit_id: visit id for disambiguation in case several visits have
the same timestamp
snapshot_id: if provided, visit associated to the snapshot will be processed
Returns:
A tuple with three members. The first one is a list of dict describing
the origin branches for the given visit.
The second one is a list of dict describing the origin releases
for the given visit. The third one is a dict mapping resolved branch
aliases to their real target.
Raises:
NotFoundExc if the origin or its visit are not found
"""
visit_info = get_origin_visit(origin_info, visit_ts, visit_id, snapshot_id)
return get_snapshot_content(visit_info["snapshot"])
def get_snapshot_context(
snapshot_id: Optional[str] = None,
origin_url: Optional[str] = None,
timestamp: Optional[str] = None,
visit_id: Optional[int] = None,
branch_name: Optional[str] = None,
release_name: Optional[str] = None,
revision_id: Optional[str] = None,
path: Optional[str] = None,
browse_context: str = "directory",
) -> SnapshotContext:
"""
Utility function to compute relevant information when navigating
the archive in a snapshot context. The snapshot is either
referenced by its id or it will be retrieved from an origin visit.
Args:
snapshot_id: hexadecimal representation of a snapshot identifier
origin_url: an origin_url
timestamp: a datetime string for retrieving the closest
visit of the origin
visit_id: optional visit id for disambiguation in case
of several visits with the same timestamp
branch_name: optional branch name set when browsing the snapshot in
that scope (will default to "HEAD" if not provided)
release_name: optional release name set when browsing the snapshot in
that scope
revision_id: optional revision identifier set when browsing the snapshot in
that scope
path: optional path of the object currently browsed in the snapshot
browse_context: indicates which type of object is currently browsed
Returns:
A dict filled with snapshot context information.
Raises:
swh.web.common.exc.NotFoundExc: if no snapshot is found for the visit
of an origin.
"""
assert origin_url is not None or snapshot_id is not None
origin_info = None
visit_info = None
url_args = {}
query_params: Dict[str, Any] = {}
origin_visits_url = None
if origin_url:
if visit_id is not None:
query_params["visit_id"] = visit_id
elif snapshot_id is not None:
query_params["snapshot"] = snapshot_id
origin_info = archive.lookup_origin({"url": origin_url})
visit_info = get_origin_visit(origin_info, timestamp, visit_id, snapshot_id)
formatted_date = format_utc_iso_date(visit_info["date"])
visit_info["formatted_date"] = formatted_date
snapshot_id = visit_info["snapshot"]
if not snapshot_id:
raise NotFoundExc(
"No snapshot associated to the visit of origin "
"%s on %s" % (escape(origin_url), formatted_date)
)
# provided timestamp is not necessarily equals to the one
# of the retrieved visit, so get the exact one in order
# to use it in the urls generated below
if timestamp:
timestamp = visit_info["date"]
branches, releases, aliases = get_origin_visit_snapshot(
origin_info, timestamp, visit_id, snapshot_id
)
query_params["origin_url"] = origin_info["url"]
origin_visits_url = reverse(
"browse-origin-visits", query_params={"origin_url": origin_info["url"]}
)
if timestamp is not None:
query_params["timestamp"] = format_utc_iso_date(
timestamp, "%Y-%m-%dT%H:%M:%SZ"
)
visit_url = reverse("browse-origin-directory", query_params=query_params)
visit_info["url"] = directory_url = visit_url
branches_url = reverse("browse-origin-branches", query_params=query_params)
releases_url = reverse("browse-origin-releases", query_params=query_params)
else:
assert snapshot_id is not None
branches, releases, aliases = get_snapshot_content(snapshot_id)
url_args = {"snapshot_id": snapshot_id}
directory_url = reverse("browse-snapshot-directory", url_args=url_args)
branches_url = reverse("browse-snapshot-branches", url_args=url_args)
releases_url = reverse("browse-snapshot-releases", url_args=url_args)
releases = list(reversed(releases))
snapshot_sizes_cache_id = f"swh_snapshot_{snapshot_id}_sizes"
snapshot_sizes = cache.get(snapshot_sizes_cache_id)
if snapshot_sizes is None:
snapshot_sizes = archive.lookup_snapshot_sizes(snapshot_id)
cache.set(snapshot_sizes_cache_id, snapshot_sizes)
is_empty = (snapshot_sizes["release"] + snapshot_sizes["revision"]) == 0
swh_snp_id = str(
CoreSWHID(object_type=ObjectType.SNAPSHOT, object_id=hash_to_bytes(snapshot_id))
)
if visit_info:
timestamp = format_utc_iso_date(visit_info["date"])
if origin_info:
browse_view_name = f"browse-origin-{browse_context}"
else:
browse_view_name = f"browse-snapshot-{browse_context}"
release_id = None
root_directory = None
snapshot_total_size = snapshot_sizes["release"] + snapshot_sizes["revision"]
if path is not None:
query_params["path"] = path
if snapshot_total_size and revision_id is not None:
# browse specific revision for a snapshot requested
revision = archive.lookup_revision(revision_id)
root_directory = revision["directory"]
branches.append(
SnapshotBranchInfo(
name=revision_id,
alias=False,
revision=revision_id,
directory=root_directory,
date=revision["date"],
message=revision["message"],
url=None,
)
)
query_params["revision"] = revision_id
elif snapshot_total_size and release_name:
# browse specific release for a snapshot requested
release = _get_release(releases, release_name, snapshot_id)
if release is None:
_branch_not_found(
"release",
release_name,
snapshot_id,
snapshot_sizes,
origin_info,
timestamp,
visit_id,
)
else:
if release["target_type"] == "revision":
revision = archive.lookup_revision(release["target"])
root_directory = revision["directory"]
revision_id = release["target"]
elif release["target_type"] == "directory":
root_directory = release["target"]
release_id = release["id"]
query_params["release"] = release_name
elif snapshot_total_size:
head = aliases.get("HEAD")
if branch_name:
# browse specific branch for a snapshot requested
query_params["branch"] = branch_name
branch = _get_branch(branches, branch_name, snapshot_id)
if branch is None:
_branch_not_found(
"branch",
branch_name,
snapshot_id,
snapshot_sizes,
origin_info,
timestamp,
visit_id,
)
else:
branch_name = branch["name"]
revision_id = branch["revision"]
root_directory = branch["directory"]
elif head is not None:
# otherwise, browse branch targeted by the HEAD alias if it exists
if head["target_type"] == "revision":
# HEAD alias targets a revision
head_rev = archive.lookup_revision(head["target"])
branch_name = "HEAD"
revision_id = head_rev["id"]
root_directory = head_rev["directory"]
else:
# HEAD alias targets a release
release_name = archive.lookup_release(head["target"])["name"]
head_rel = _get_release(releases, release_name, snapshot_id)
if head_rel["target_type"] == "revision":
revision = archive.lookup_revision(head_rel["target"])
root_directory = revision["directory"]
revision_id = head_rel["target"]
elif head_rel["target_type"] == "directory":
root_directory = head_rel["target"]
release_id = head_rel["id"]
elif branches:
# fallback to browse first branch otherwise
branch = branches[0]
branch_name = branch["name"]
revision_id = branch["revision"]
root_directory = branch["directory"]
elif releases:
# fallback to browse last release otherwise
release = releases[-1]
if release["target_type"] == "revision":
revision = archive.lookup_revision(release["target"])
root_directory = revision["directory"]
revision_id = release["target"]
elif release["target_type"] == "directory":
root_directory = release["target"]
release_id = release["id"]
release_name = release["name"]
for b in branches:
branch_query_params = dict(query_params)
branch_query_params.pop("release", None)
if b["name"] != b["revision"]:
branch_query_params.pop("revision", None)
branch_query_params["branch"] = b["name"]
b["url"] = reverse(
browse_view_name, url_args=url_args, query_params=branch_query_params
)
for r in releases:
release_query_params = dict(query_params)
release_query_params.pop("branch", None)
release_query_params.pop("revision", None)
release_query_params["release"] = r["name"]
r["url"] = reverse(
browse_view_name, url_args=url_args, query_params=release_query_params,
)
revision_info = None
if revision_id:
try:
revision_info = archive.lookup_revision(revision_id)
except NotFoundExc:
pass
else:
revision_info["date"] = format_utc_iso_date(revision_info["date"])
revision_info["committer_date"] = format_utc_iso_date(
revision_info["committer_date"]
)
if revision_info["message"]:
message_lines = revision_info["message"].split("\n")
revision_info["message_header"] = message_lines[0]
else:
revision_info["message_header"] = ""
snapshot_context = SnapshotContext(
directory_url=directory_url,
branch=branch_name,
branch_alias=branch_name in aliases,
branches=branches,
branches_url=branches_url,
is_empty=is_empty,
origin_info=origin_info,
origin_visits_url=origin_visits_url,
release=release_name,
release_alias=release_name in aliases,
release_id=release_id,
query_params=query_params,
releases=releases,
releases_url=releases_url,
revision_id=revision_id,
revision_info=revision_info,
root_directory=root_directory,
snapshot_id=snapshot_id,
snapshot_sizes=snapshot_sizes,
snapshot_swhid=swh_snp_id,
url_args=url_args,
visit_info=visit_info,
)
if revision_info:
revision_info["revision_url"] = gen_revision_url(revision_id, snapshot_context)
return snapshot_context
def _build_breadcrumbs(snapshot_context: SnapshotContext, path: str):
origin_info = snapshot_context["origin_info"]
url_args = snapshot_context["url_args"]
query_params = dict(snapshot_context["query_params"])
root_directory = snapshot_context["root_directory"]
path_info = gen_path_info(path)
if origin_info:
browse_view_name = "browse-origin-directory"
else:
browse_view_name = "browse-snapshot-directory"
breadcrumbs = []
if root_directory:
query_params.pop("path", None)
breadcrumbs.append(
{
"name": root_directory[:7],
"url": reverse(
browse_view_name, url_args=url_args, query_params=query_params
),
}
)
for pi in path_info:
query_params["path"] = pi["path"]
breadcrumbs.append(
{
"name": pi["name"],
"url": reverse(
browse_view_name, url_args=url_args, query_params=query_params
),
}
)
return breadcrumbs
def _check_origin_url(snapshot_id, origin_url):
if snapshot_id is None and origin_url is None:
raise BadInputExc("An origin URL must be provided as query parameter.")
def browse_snapshot_directory(
request, snapshot_id=None, origin_url=None, timestamp=None, path=None
):
"""
Django view implementation for browsing a directory in a snapshot context.
"""
_check_origin_url(snapshot_id, origin_url)
snapshot_context = get_snapshot_context(
snapshot_id=snapshot_id,
origin_url=origin_url,
timestamp=timestamp,
visit_id=request.GET.get("visit_id"),
path=path,
browse_context="directory",
branch_name=request.GET.get("branch"),
release_name=request.GET.get("release"),
revision_id=request.GET.get("revision"),
)
root_directory = snapshot_context["root_directory"]
sha1_git = root_directory
error_info = {
"status_code": 200,
"description": None,
}
if root_directory and path:
try:
dir_info = archive.lookup_directory_with_path(root_directory, path)
sha1_git = dir_info["target"]
except NotFoundExc as e:
sha1_git = None
error_info["status_code"] = 404
error_info["description"] = f"NotFoundExc: {str(e)}"
dirs = []
files = []
if sha1_git:
dirs, files = get_directory_entries(sha1_git)
origin_info = snapshot_context["origin_info"]
visit_info = snapshot_context["visit_info"]
url_args = snapshot_context["url_args"]
query_params = dict(snapshot_context["query_params"])
revision_id = snapshot_context["revision_id"]
snapshot_id = snapshot_context["snapshot_id"]
if origin_info:
browse_view_name = "browse-origin-directory"
else:
browse_view_name = "browse-snapshot-directory"
breadcrumbs = _build_breadcrumbs(snapshot_context, path)
path = "" if path is None else (path + "/")
for d in dirs:
if d["type"] == "rev":
d["url"] = reverse("browse-revision", url_args={"sha1_git": d["target"]})
else:
query_params["path"] = path + d["name"]
d["url"] = reverse(
browse_view_name, url_args=url_args, query_params=query_params
)
sum_file_sizes = 0
readmes = {}
if origin_info:
browse_view_name = "browse-origin-content"
else:
browse_view_name = "browse-snapshot-content"
for f in files:
query_params["path"] = path + f["name"]
f["url"] = reverse(
browse_view_name, url_args=url_args, query_params=query_params
)
if f["length"] is not None:
sum_file_sizes += f["length"]
if f["name"].lower().startswith("readme"):
readmes[f["name"]] = f["checksums"]["sha1"]
readme_name, readme_url, readme_html = get_readme_to_display(readmes)
if origin_info:
browse_view_name = "browse-origin-log"
else:
browse_view_name = "browse-snapshot-log"
history_url = None
if snapshot_id != _empty_snapshot_id:
query_params.pop("path", None)
history_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params
)
nb_files = None
nb_dirs = None
dir_path = None
if root_directory:
nb_files = len(files)
nb_dirs = len(dirs)
dir_path = "/" + path
swh_objects = []
vault_cooking = {}
revision_found = False
- if sha1_git is None and revision_id is not None:
+ if revision_id is not None:
try:
archive.lookup_revision(revision_id)
except NotFoundExc:
pass
else:
revision_found = True
if sha1_git is not None:
swh_objects.append(
SWHObjectInfo(object_type=ObjectType.DIRECTORY, object_id=sha1_git)
)
vault_cooking.update(
{"directory_context": True, "directory_swhid": f"swh:1:dir:{sha1_git}",}
)
if revision_id is not None and revision_found:
swh_objects.append(
SWHObjectInfo(object_type=ObjectType.REVISION, object_id=revision_id)
)
vault_cooking.update(
{"revision_context": True, "revision_swhid": f"swh:1:rev:{revision_id}",}
)
swh_objects.append(
SWHObjectInfo(object_type=ObjectType.SNAPSHOT, object_id=snapshot_id)
)
visit_date = None
visit_type = None
if visit_info:
visit_date = format_utc_iso_date(visit_info["date"])
visit_type = visit_info["type"]
release_id = snapshot_context["release_id"]
if release_id:
swh_objects.append(
SWHObjectInfo(object_type=ObjectType.RELEASE, object_id=release_id)
)
dir_metadata = DirectoryMetadata(
object_type=ObjectType.DIRECTORY,
object_id=sha1_git,
directory=sha1_git,
nb_files=nb_files,
nb_dirs=nb_dirs,
sum_file_sizes=sum_file_sizes,
root_directory=root_directory,
path=dir_path,
revision=revision_id,
revision_found=revision_found,
release=release_id,
snapshot=snapshot_id,
origin_url=origin_url,
visit_date=visit_date,
visit_type=visit_type,
)
swhids_info = get_swhids_info(swh_objects, snapshot_context, dir_metadata)
dir_path = "/".join([bc["name"] for bc in breadcrumbs]) + "/"
context_found = "snapshot: %s" % snapshot_context["snapshot_id"]
if origin_info:
context_found = "origin: %s" % origin_info["url"]
heading = "Directory - %s - %s - %s" % (
dir_path,
snapshot_context["branch"],
context_found,
)
top_right_link = None
if not snapshot_context["is_empty"] and revision_found:
top_right_link = {
"url": history_url,
"icon": swh_object_icons["revisions history"],
"text": "History",
}
return render(
request,
"browse/directory.html",
{
"heading": heading,
"swh_object_name": "Directory",
"swh_object_metadata": dir_metadata,
"dirs": dirs,
"files": files,
"breadcrumbs": breadcrumbs if root_directory else [],
"top_right_link": top_right_link,
"readme_name": readme_name,
"readme_url": readme_url,
"readme_html": readme_html,
"snapshot_context": snapshot_context,
"vault_cooking": vault_cooking,
"show_actions": True,
"swhids_info": swhids_info,
"error_code": error_info["status_code"],
"error_message": http_status_code_message.get(error_info["status_code"]),
"error_description": error_info["description"],
},
status=error_info["status_code"],
)
PER_PAGE = 100
def browse_snapshot_log(request, snapshot_id=None, origin_url=None, timestamp=None):
"""
Django view implementation for browsing a revision history in a
snapshot context.
"""
_check_origin_url(snapshot_id, origin_url)
snapshot_context = get_snapshot_context(
snapshot_id=snapshot_id,
origin_url=origin_url,
timestamp=timestamp,
visit_id=request.GET.get("visit_id"),
browse_context="log",
branch_name=request.GET.get("branch"),
release_name=request.GET.get("release"),
revision_id=request.GET.get("revision"),
)
revision_id = snapshot_context["revision_id"]
if revision_id is None:
raise NotFoundExc("No revisions history found in the current snapshot context.")
per_page = int(request.GET.get("per_page", PER_PAGE))
offset = int(request.GET.get("offset", 0))
revs_ordering = request.GET.get("revs_ordering", "committer_date")
session_key = "rev_%s_log_ordering_%s" % (revision_id, revs_ordering)
rev_log_session = request.session.get(session_key, None)
rev_log = []
revs_walker_state = None
if rev_log_session:
rev_log = rev_log_session["rev_log"]
revs_walker_state = rev_log_session["revs_walker_state"]
if len(rev_log) < offset + per_page:
revs_walker = archive.get_revisions_walker(
revs_ordering,
revision_id,
max_revs=offset + per_page + 1,
state=revs_walker_state,
)
rev_log += [rev["id"] for rev in revs_walker]
revs_walker_state = revs_walker.export_state()
revs = rev_log[offset : offset + per_page]
revision_log = archive.lookup_revision_multiple(revs)
request.session[session_key] = {
"rev_log": rev_log,
"revs_walker_state": revs_walker_state,
}
origin_info = snapshot_context["origin_info"]
visit_info = snapshot_context["visit_info"]
url_args = snapshot_context["url_args"]
query_params = snapshot_context["query_params"]
snapshot_id = snapshot_context["snapshot_id"]
query_params["per_page"] = per_page
revs_ordering = request.GET.get("revs_ordering", "")
query_params["revs_ordering"] = revs_ordering or None
if origin_info:
browse_view_name = "browse-origin-log"
else:
browse_view_name = "browse-snapshot-log"
prev_log_url = None
if len(rev_log) > offset + per_page:
query_params["offset"] = offset + per_page
prev_log_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params
)
next_log_url = None
if offset != 0:
query_params["offset"] = offset - per_page
next_log_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params
)
revision_log_data = format_log_entries(revision_log, per_page, snapshot_context)
browse_rev_link = gen_revision_link(revision_id)
browse_log_link = gen_revision_log_link(revision_id)
browse_snp_link = gen_snapshot_link(snapshot_id)
revision_metadata = {
"context-independent revision": browse_rev_link,
"context-independent revision history": browse_log_link,
"context-independent snapshot": browse_snp_link,
"snapshot": snapshot_id,
}
if origin_info:
revision_metadata["origin url"] = origin_info["url"]
revision_metadata["origin visit date"] = format_utc_iso_date(visit_info["date"])
revision_metadata["origin visit type"] = visit_info["type"]
swh_objects = [
SWHObjectInfo(object_type=ObjectType.REVISION, object_id=revision_id),
SWHObjectInfo(object_type=ObjectType.SNAPSHOT, object_id=snapshot_id),
]
release_id = snapshot_context["release_id"]
if release_id:
swh_objects.append(
SWHObjectInfo(object_type=ObjectType.RELEASE, object_id=release_id)
)
browse_rel_link = gen_release_link(release_id)
revision_metadata["release"] = release_id
revision_metadata["context-independent release"] = browse_rel_link
swhids_info = get_swhids_info(swh_objects, snapshot_context)
context_found = "snapshot: %s" % snapshot_context["snapshot_id"]
if origin_info:
context_found = "origin: %s" % origin_info["url"]
heading = "Revision history - %s - %s" % (snapshot_context["branch"], context_found)
return render(
request,
"browse/revision-log.html",
{
"heading": heading,
"swh_object_name": "Revisions history",
"swh_object_metadata": revision_metadata,
"revision_log": revision_log_data,
"revs_ordering": revs_ordering,
"next_log_url": next_log_url,
"prev_log_url": prev_log_url,
"breadcrumbs": None,
"top_right_link": None,
"snapshot_context": snapshot_context,
"vault_cooking": None,
"show_actions": True,
"swhids_info": swhids_info,
},
)
def browse_snapshot_branches(
request, snapshot_id=None, origin_url=None, timestamp=None, branch_name_include=None
):
"""
Django view implementation for browsing a list of branches in a snapshot
context.
"""
_check_origin_url(snapshot_id, origin_url)
snapshot_context = get_snapshot_context(
snapshot_id=snapshot_id,
origin_url=origin_url,
timestamp=timestamp,
visit_id=request.GET.get("visit_id"),
)
branches_bc = request.GET.get("branches_breadcrumbs", "")
branches_bc = branches_bc.split(",") if branches_bc else []
branches_from = branches_bc[-1] if branches_bc else ""
origin_info = snapshot_context["origin_info"]
url_args = snapshot_context["url_args"]
query_params = snapshot_context["query_params"]
if origin_info:
browse_view_name = "browse-origin-directory"
else:
browse_view_name = "browse-snapshot-directory"
snapshot = archive.lookup_snapshot(
snapshot_context["snapshot_id"],
branches_from,
PER_PAGE + 1,
target_types=["revision", "alias"],
branch_name_include_substring=branch_name_include,
)
displayed_branches = []
if snapshot:
displayed_branches, _, _ = process_snapshot_branches(snapshot)
for branch in displayed_branches:
rev_query_params = {}
if origin_info:
rev_query_params["origin_url"] = origin_info["url"]
revision_url = reverse(
"browse-revision",
url_args={"sha1_git": branch["revision"]},
query_params=query_params,
)
query_params["branch"] = branch["name"]
directory_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params
)
del query_params["branch"]
branch["revision_url"] = revision_url
branch["directory_url"] = directory_url
if origin_info:
browse_view_name = "browse-origin-branches"
else:
browse_view_name = "browse-snapshot-branches"
prev_branches_url = None
next_branches_url = None
if branches_bc:
query_params_prev = dict(query_params)
query_params_prev["branches_breadcrumbs"] = ",".join(branches_bc[:-1])
prev_branches_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params_prev
)
elif branches_from:
prev_branches_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params
)
if snapshot and snapshot["next_branch"] is not None:
query_params_next = dict(query_params)
next_branch = displayed_branches[-1]["name"]
del displayed_branches[-1]
branches_bc.append(next_branch)
query_params_next["branches_breadcrumbs"] = ",".join(branches_bc)
next_branches_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params_next
)
heading = "Branches - "
if origin_info:
heading += "origin: %s" % origin_info["url"]
else:
heading += "snapshot: %s" % snapshot_id
return render(
request,
"browse/branches.html",
{
"heading": heading,
"swh_object_name": "Branches",
"swh_object_metadata": {},
"top_right_link": None,
"displayed_branches": displayed_branches,
"prev_branches_url": prev_branches_url,
"next_branches_url": next_branches_url,
"snapshot_context": snapshot_context,
"search_string": branch_name_include or "",
},
)
def browse_snapshot_releases(
request,
snapshot_id=None,
origin_url=None,
timestamp=None,
release_name_include=None,
):
"""
Django view implementation for browsing a list of releases in a snapshot
context.
"""
_check_origin_url(snapshot_id, origin_url)
snapshot_context = get_snapshot_context(
snapshot_id=snapshot_id,
origin_url=origin_url,
timestamp=timestamp,
visit_id=request.GET.get("visit_id"),
)
rel_bc = request.GET.get("releases_breadcrumbs", "")
rel_bc = rel_bc.split(",") if rel_bc else []
rel_from = rel_bc[-1] if rel_bc else ""
origin_info = snapshot_context["origin_info"]
url_args = snapshot_context["url_args"]
query_params = snapshot_context["query_params"]
snapshot = archive.lookup_snapshot(
snapshot_context["snapshot_id"],
rel_from,
PER_PAGE + 1,
target_types=["release", "alias"],
branch_name_include_substring=release_name_include,
)
displayed_releases = []
if snapshot:
_, displayed_releases, _ = process_snapshot_branches(snapshot)
for release in displayed_releases:
query_params_tgt = {"snapshot": snapshot_id, "release": release["name"]}
if origin_info:
query_params_tgt["origin_url"] = origin_info["url"]
release_url = reverse(
"browse-release",
url_args={"sha1_git": release["id"]},
query_params=query_params_tgt,
)
target_url = ""
tooltip = (
f"The release {release['name']} targets "
f"{release['target_type']} {release['target']}"
)
if release["target_type"] == "revision":
target_url = reverse(
"browse-revision",
url_args={"sha1_git": release["target"]},
query_params=query_params_tgt,
)
elif release["target_type"] == "directory":
target_url = reverse(
"browse-directory",
url_args={"sha1_git": release["target"]},
query_params=query_params_tgt,
)
elif release["target_type"] == "content":
target_url = reverse(
"browse-content",
url_args={"query_string": release["target"]},
query_params=query_params_tgt,
)
elif release["target_type"] == "release":
target_url = reverse(
"browse-release",
url_args={"sha1_git": release["target"]},
query_params=query_params_tgt,
)
tooltip = (
f"The release {release['name']} "
f"is an alias for release {release['target']}"
)
release["release_url"] = release_url
release["target_url"] = target_url
release["tooltip"] = tooltip
if origin_info:
browse_view_name = "browse-origin-releases"
else:
browse_view_name = "browse-snapshot-releases"
prev_releases_url = None
next_releases_url = None
if rel_bc:
query_params_prev = dict(query_params)
query_params_prev["releases_breadcrumbs"] = ",".join(rel_bc[:-1])
prev_releases_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params_prev
)
elif rel_from:
prev_releases_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params
)
if snapshot and snapshot["next_branch"] is not None:
query_params_next = dict(query_params)
next_rel = displayed_releases[-1]["branch_name"]
del displayed_releases[-1]
rel_bc.append(next_rel)
query_params_next["releases_breadcrumbs"] = ",".join(rel_bc)
next_releases_url = reverse(
browse_view_name, url_args=url_args, query_params=query_params_next
)
heading = "Releases - "
if origin_info:
heading += "origin: %s" % origin_info["url"]
else:
heading += "snapshot: %s" % snapshot_id
return render(
request,
"browse/releases.html",
{
"heading": heading,
"top_panel_visible": False,
"top_panel_collapsible": False,
"swh_object_name": "Releases",
"swh_object_metadata": {},
"top_right_link": None,
"displayed_releases": displayed_releases,
"prev_releases_url": prev_releases_url,
"next_releases_url": next_releases_url,
"snapshot_context": snapshot_context,
"vault_cooking": None,
"show_actions": False,
"search_string": release_name_include or "",
},
)
diff --git a/swh/web/common/converters.py b/swh/web/common/converters.py
index 2deecfcf..74a71d53 100644
--- a/swh/web/common/converters.py
+++ b/swh/web/common/converters.py
@@ -1,409 +1,403 @@
# Copyright (C) 2015-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import datetime
import json
from typing import Any, Dict, Union
from django.core.serializers.json import DjangoJSONEncoder
from swh.core.utils import decode_with_escape
from swh.model import hashutil
-from swh.model.model import RawExtrinsicMetadata, Release, Revision
+from swh.model.model import (
+ RawExtrinsicMetadata,
+ Release,
+ Revision,
+ TimestampWithTimezone,
+)
from swh.model.swhids import ObjectType
from swh.storage.interface import PartialBranches
from swh.web.common.typing import OriginInfo, OriginVisitInfo
def _group_checksums(data):
"""Groups checksums values computed from hash functions used in swh
and stored in data dict under a single entry 'checksums'
"""
if data:
checksums = {}
for hash in hashutil.ALGORITHMS:
if hash in data and data[hash]:
checksums[hash] = data[hash]
del data[hash]
if len(checksums) > 0:
data["checksums"] = checksums
def fmap(f, data):
"""Map f to data at each level.
This must keep the origin data structure type:
- map -> map
- dict -> dict
- list -> list
- None -> None
Args:
f: function that expects one argument.
data: data to traverse to apply the f function.
list, map, dict or bare value.
Returns:
The same data-structure with modified values by the f function.
"""
if data is None:
return data
if isinstance(data, map):
return map(lambda y: fmap(f, y), (x for x in data))
if isinstance(data, list):
return [fmap(f, x) for x in data]
if isinstance(data, tuple):
return tuple(fmap(f, x) for x in data)
if isinstance(data, dict):
return {k: fmap(f, v) for (k, v) in data.items()}
return f(data)
def from_swh(
dict_swh,
hashess={},
bytess={},
dates={},
blacklist={},
removables_if_empty={},
empty_dict={},
empty_list={},
convert={},
convert_fn=lambda x: x,
):
"""Convert from a swh dictionary to something reasonably json
serializable.
Args:
dict_swh: the origin dictionary needed to be transformed
hashess: list/set of keys representing hashes values (sha1, sha256,
sha1_git, etc...) as bytes. Those need to be transformed in
hexadecimal string
bytess: list/set of keys representing bytes values which needs to be
decoded
blacklist: set of keys to filter out from the conversion
convert: set of keys whose associated values need to be converted using
convert_fn
convert_fn: the conversion function to apply on the value of key in
'convert'
The remaining keys are copied as is in the output.
Returns:
dictionary equivalent as dict_swh only with its keys converted.
"""
def convert_hashes_bytes(v):
"""v is supposedly a hash as bytes, returns it converted in hex.
"""
if isinstance(v, bytes):
return hashutil.hash_to_hex(v)
return v
def convert_bytes(v):
"""v is supposedly a bytes string, decode as utf-8.
FIXME: Improve decoding policy.
If not utf-8, break!
"""
if isinstance(v, bytes):
return v.decode("utf-8")
return v
def convert_date(v):
"""
Args:
v (dict or datatime): either:
- a dict with three keys:
- timestamp (dict or integer timestamp)
- - offset
- - negative_utc
+ - offset_bytes
- or, a datetime
We convert it to a human-readable string
"""
if not v:
return v
if isinstance(v, datetime.datetime):
return v.isoformat()
- tz = datetime.timezone(datetime.timedelta(minutes=v["offset"]))
- swh_timestamp = v["timestamp"]
- if isinstance(swh_timestamp, dict):
- date = datetime.datetime.fromtimestamp(swh_timestamp["seconds"], tz=tz)
- else:
- date = datetime.datetime.fromtimestamp(swh_timestamp, tz=tz)
-
- datestr = date.isoformat()
-
- if v["offset"] == 0 and v["negative_utc"]:
- # remove the rightmost + and replace it with a -
- return "-".join(datestr.rsplit("+", 1))
-
- return datestr
+ v = v.copy()
+ if isinstance(v["timestamp"], float):
+ v["timestamp"] = int(v["timestamp"])
+ return TimestampWithTimezone.from_dict(v).to_datetime().isoformat()
if not dict_swh:
return dict_swh
new_dict = {}
for key, value in dict_swh.items():
if key in blacklist or (key in removables_if_empty and not value):
continue
if key in dates:
new_dict[key] = convert_date(value)
elif key in convert:
new_dict[key] = convert_fn(value)
elif isinstance(value, dict):
new_dict[key] = from_swh(
value,
hashess=hashess,
bytess=bytess,
dates=dates,
blacklist=blacklist,
removables_if_empty=removables_if_empty,
empty_dict=empty_dict,
empty_list=empty_list,
convert=convert,
convert_fn=convert_fn,
)
elif key in hashess:
new_dict[key] = fmap(convert_hashes_bytes, value)
elif key in bytess:
try:
new_dict[key] = fmap(convert_bytes, value)
except UnicodeDecodeError:
if "decoding_failures" not in new_dict:
new_dict["decoding_failures"] = [key]
else:
new_dict["decoding_failures"].append(key)
new_dict[key] = fmap(decode_with_escape, value)
elif key in empty_dict and not value:
new_dict[key] = {}
elif key in empty_list and not value:
new_dict[key] = []
else:
new_dict[key] = value
_group_checksums(new_dict)
return new_dict
def from_origin(origin: Dict[str, Any]) -> OriginInfo:
"""Convert from a swh origin to an origin dictionary.
"""
return from_swh(origin, blacklist={"id"})
def from_release(release: Release) -> Dict[str, Any]:
"""Convert from a swh release to a json serializable release dictionary.
Args:
release: A release model object
Returns:
release dictionary with the following keys
- id: hexadecimal sha1 (string)
- revision: hexadecimal sha1 (string)
- comment: release's comment message (string)
- name: release's name (string)
- author: release's author identifier (swh's id)
- synthetic: the synthetic property (boolean)
"""
return from_swh(
release.to_dict(),
hashess={"id", "target"},
bytess={"message", "name", "fullname", "email"},
dates={"date"},
)
class SWHDjangoJSONEncoder(DjangoJSONEncoder):
"""Wrapper around DjangoJSONEncoder to serialize SWH-specific types
found in :class:`swh.web.common.typing.SWHObjectInfo`."""
def default(self, o):
if isinstance(o, ObjectType):
return o.name.lower()
else:
super().default(o)
class SWHMetadataEncoder(json.JSONEncoder):
"""Special json encoder for metadata field which can contain bytes
encoded value.
"""
def default(self, obj):
if isinstance(obj, bytes):
try:
return obj.decode("utf-8")
except UnicodeDecodeError:
# fallback to binary representation to avoid display errors
return repr(obj)
# Let the base class default method raise the TypeError
return json.JSONEncoder.default(self, obj)
def convert_metadata(metadata):
"""Convert json specific dict to a json serializable one.
"""
if metadata is None:
return {}
return json.loads(json.dumps(metadata, cls=SWHMetadataEncoder))
def from_revision(revision: Union[Dict[str, Any], Revision]) -> Dict[str, Any]:
"""Convert swh revision model object to a json serializable revision dictionary.
Args:
revision: revision model object
Returns:
dict: Revision dictionary with the same keys as inputs, except:
- sha1s are in hexadecimal strings (id, directory)
- bytes are decoded in string (author_name, committer_name,
author_email, committer_email)
Remaining keys are left as is
"""
if isinstance(revision, Revision):
revision_d = revision.to_dict()
else:
revision_d = revision
revision_d = from_swh(
revision_d,
hashess={"id", "directory", "parents", "children"},
bytess={"name", "fullname", "email", "extra_headers", "message"},
convert={"metadata"},
convert_fn=convert_metadata,
dates={"date", "committer_date"},
)
if revision_d:
if "parents" in revision_d:
revision_d["merge"] = len(revision_d["parents"]) > 1
return revision_d
def from_raw_extrinsic_metadata(
metadata: Union[Dict[str, Any], RawExtrinsicMetadata]
) -> Dict[str, Any]:
"""Convert RawExtrinsicMetadata model object to a json serializable dictionary.
"""
return from_swh(
metadata.to_dict() if isinstance(metadata, RawExtrinsicMetadata) else metadata,
blacklist={"id", "metadata"},
dates={"discovery_date"},
)
def from_content(content):
"""Convert swh content to serializable content dictionary.
"""
return from_swh(
content,
hashess={"sha1", "sha1_git", "sha256", "blake2s256"},
blacklist={"ctime"},
convert={"status"},
convert_fn=lambda v: "absent" if v == "hidden" else v,
)
def from_person(person):
"""Convert swh person to serializable person dictionary.
"""
return from_swh(person, bytess={"name", "fullname", "email"})
def from_origin_visit(visit: Dict[str, Any]) -> OriginVisitInfo:
"""Convert swh origin_visit to serializable origin_visit dictionary.
"""
ov = from_swh(
visit,
hashess={"target", "snapshot"},
bytess={"branch"},
dates={"date"},
empty_dict={"metadata"},
)
return ov
def from_snapshot(snapshot):
"""Convert swh snapshot to serializable (partial) snapshot dictionary.
"""
sv = from_swh(snapshot, hashess={"id", "target"}, bytess={"next_branch"})
if sv and "branches" in sv:
sv["branches"] = {decode_with_escape(k): v for k, v in sv["branches"].items()}
for k, v in snapshot["branches"].items():
# alias target existing branch names, not a sha1
if v and v["target_type"] == "alias":
branch = decode_with_escape(k)
target = decode_with_escape(v["target"])
sv["branches"][branch]["target"] = target
return sv
def from_partial_branches(branches: PartialBranches):
"""Convert PartialBranches to serializable partial snapshot dictionary
"""
return from_snapshot(
{
"id": branches["id"],
"branches": {
branch_name: branch.to_dict() if branch else None
for (branch_name, branch) in branches["branches"].items()
},
"next_branch": branches["next_branch"],
}
)
def from_directory_entry(dir_entry):
"""Convert swh directory to serializable directory dictionary.
"""
return from_swh(
dir_entry,
hashess={"dir_id", "sha1_git", "sha1", "sha256", "blake2s256", "target"},
bytess={"name"},
removables_if_empty={"sha1", "sha1_git", "sha256", "blake2s256", "status"},
convert={"status"},
convert_fn=lambda v: "absent" if v == "hidden" else v,
)
def from_filetype(content_entry):
"""Convert swh content to serializable dictionary containing keys
'id', 'encoding', and 'mimetype'.
"""
return from_swh(content_entry, hashess={"id"})
diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py
index 3749d134..6696427b 100644
--- a/swh/web/common/origin_save.py
+++ b/swh/web/common/origin_save.py
@@ -1,935 +1,939 @@
# Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from bisect import bisect_right
from datetime import datetime, timedelta, timezone
from functools import lru_cache
from itertools import product
import json
import logging
from typing import Any, Dict, List, Optional, Tuple
from prometheus_client import Gauge
import requests
import sentry_sdk
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from django.core.validators import URLValidator
from django.db.models import Q, QuerySet
from django.utils.html import escape
from swh.scheduler.utils import create_oneshot_task_dict
from swh.web.common import archive
from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc
from swh.web.common.models import (
SAVE_REQUEST_ACCEPTED,
SAVE_REQUEST_PENDING,
SAVE_REQUEST_REJECTED,
SAVE_TASK_FAILED,
SAVE_TASK_NOT_CREATED,
SAVE_TASK_NOT_YET_SCHEDULED,
SAVE_TASK_RUNNING,
SAVE_TASK_SCHEDULED,
SAVE_TASK_SUCCEEDED,
VISIT_STATUS_CREATED,
VISIT_STATUS_ONGOING,
SaveAuthorizedOrigin,
SaveOriginRequest,
SaveUnauthorizedOrigin,
)
from swh.web.common.origin_visits import get_origin_visits
from swh.web.common.typing import (
OriginExistenceCheckInfo,
OriginInfo,
SaveOriginRequestInfo,
)
from swh.web.common.utils import SWH_WEB_METRICS_REGISTRY, parse_iso8601_date_to_utc
from swh.web.config import get_config, scheduler
logger = logging.getLogger(__name__)
# Number of days in the past to lookup for information
MAX_THRESHOLD_DAYS = 30
# Non terminal visit statuses which needs updates
NON_TERMINAL_STATUSES = [
VISIT_STATUS_CREATED,
VISIT_STATUS_ONGOING,
]
def get_origin_save_authorized_urls() -> List[str]:
"""
Get the list of origin url prefixes authorized to be
immediately loaded into the archive (whitelist).
Returns:
list: The list of authorized origin url prefix
"""
return [origin.url for origin in SaveAuthorizedOrigin.objects.all()]
def get_origin_save_unauthorized_urls() -> List[str]:
"""
Get the list of origin url prefixes forbidden to be
loaded into the archive (blacklist).
Returns:
list: the list of unauthorized origin url prefix
"""
return [origin.url for origin in SaveUnauthorizedOrigin.objects.all()]
def can_save_origin(origin_url: str, bypass_pending_review: bool = False) -> str:
"""
Check if a software origin can be saved into the archive.
Based on the origin url, the save request will be either:
* immediately accepted if the url is whitelisted
* rejected if the url is blacklisted
* put in pending state for manual review otherwise
Args:
origin_url (str): the software origin url to check
Returns:
str: the origin save request status, either **accepted**,
**rejected** or **pending**
"""
# origin url may be blacklisted
for url_prefix in get_origin_save_unauthorized_urls():
if origin_url.startswith(url_prefix):
return SAVE_REQUEST_REJECTED
# if the origin url is in the white list, it can be immediately saved
for url_prefix in get_origin_save_authorized_urls():
if origin_url.startswith(url_prefix):
return SAVE_REQUEST_ACCEPTED
# otherwise, the origin url needs to be manually verified if the user
# that submitted it does not have special permission
if bypass_pending_review:
# mark the origin URL as trusted in that case
SaveAuthorizedOrigin.objects.get_or_create(url=origin_url)
return SAVE_REQUEST_ACCEPTED
else:
return SAVE_REQUEST_PENDING
# map visit type to scheduler task
# TODO: do not hardcode the task name here (T1157)
_visit_type_task = {
"git": "load-git",
"hg": "load-hg",
"svn": "load-svn",
"cvs": "load-cvs",
}
_visit_type_task_privileged = {
"archives": "load-archive-files",
}
# map scheduler task status to origin save status
_save_task_status = {
"next_run_not_scheduled": SAVE_TASK_NOT_YET_SCHEDULED,
"next_run_scheduled": SAVE_TASK_SCHEDULED,
"completed": SAVE_TASK_SUCCEEDED,
"disabled": SAVE_TASK_FAILED,
}
# map scheduler task_run status to origin save status
_save_task_run_status = {
"scheduled": SAVE_TASK_SCHEDULED,
"started": SAVE_TASK_RUNNING,
"eventful": SAVE_TASK_SUCCEEDED,
"uneventful": SAVE_TASK_SUCCEEDED,
"failed": SAVE_TASK_FAILED,
"permfailed": SAVE_TASK_FAILED,
"lost": SAVE_TASK_FAILED,
}
@lru_cache()
def get_scheduler_load_task_types() -> List[str]:
task_types = scheduler().get_task_types()
return [t["type"] for t in task_types if t["type"].startswith("load")]
def get_savable_visit_types_dict(privileged_user: bool = False) -> Dict:
"""Returned the supported task types the user has access to.
Args:
privileged_user: Flag to determine if all visit types should be returned or not.
Default to False to only list unprivileged visit types.
Returns:
the dict of supported visit types for the user
"""
if privileged_user:
task_types = {**_visit_type_task, **_visit_type_task_privileged}
else:
task_types = _visit_type_task
# filter visit types according to scheduler load task types if available
try:
load_task_types = get_scheduler_load_task_types()
return {k: v for k, v in task_types.items() if v in load_task_types}
except Exception:
return task_types
def get_savable_visit_types(privileged_user: bool = False) -> List[str]:
"""Return the list of visit types the user can perform save requests on.
Args:
privileged_user: Flag to determine if all visit types should be returned or not.
Default to False to only list unprivileged visit types.
Returns:
the list of saveable visit types
"""
return sorted(list(get_savable_visit_types_dict(privileged_user).keys()))
def _check_visit_type_savable(visit_type: str, privileged_user: bool = False) -> None:
visit_type_tasks = get_savable_visit_types(privileged_user)
if visit_type not in visit_type_tasks:
allowed_visit_types = ", ".join(visit_type_tasks)
raise BadInputExc(
f"Visit of type {visit_type} can not be saved! "
f"Allowed types are the following: {allowed_visit_types}"
)
_validate_url = URLValidator(
schemes=["http", "https", "svn", "git", "rsync", "pserver", "ssh"]
)
def _check_origin_url_valid(origin_url: str) -> None:
try:
_validate_url(origin_url)
except ValidationError:
raise BadInputExc(
"The provided origin url (%s) is not valid!" % escape(origin_url)
)
def origin_exists(origin_url: str) -> OriginExistenceCheckInfo:
"""Check the origin url for existence. If it exists, extract some more useful
information on the origin.
"""
resp = requests.head(origin_url, allow_redirects=True)
exists = resp.ok
content_length: Optional[int] = None
last_modified: Optional[str] = None
if exists:
# Also process X-Archive-Orig-* headers in case the URL targets the
# Internet Archive.
size_ = resp.headers.get(
"Content-Length", resp.headers.get("X-Archive-Orig-Content-Length")
)
content_length = int(size_) if size_ else None
try:
date_str = resp.headers.get(
"Last-Modified", resp.headers.get("X-Archive-Orig-Last-Modified", "")
)
date = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %Z")
last_modified = date.isoformat()
except ValueError:
# if not provided or not parsable as per the expected format, keep it None
pass
return OriginExistenceCheckInfo(
origin_url=origin_url,
exists=exists,
last_modified=last_modified,
content_length=content_length,
)
def _check_origin_exists(url: str) -> OriginExistenceCheckInfo:
"""Ensure an URL exists, if not raise an explicit message."""
metadata = origin_exists(url)
if not metadata["exists"]:
raise BadInputExc(f"The provided url ({escape(url)}) does not exist!")
return metadata
def _get_visit_info_for_save_request(
save_request: SaveOriginRequest,
) -> Tuple[Optional[datetime], Optional[str]]:
"""Retrieve visit information out of a save request
Args:
save_request: Input save origin request to retrieve information for.
Returns:
Tuple of (visit date, optional visit status) for such save request origin
"""
visit_date = None
visit_status = None
time_now = datetime.now(tz=timezone.utc)
time_delta = time_now - save_request.request_date
# stop trying to find a visit date one month after save request submission
# as those requests to storage are expensive and associated loading task
# surely ended up with errors
if time_delta.days <= MAX_THRESHOLD_DAYS:
try:
origin_info = archive.lookup_origin(OriginInfo(url=save_request.origin_url))
origin_visits = get_origin_visits(origin_info)
visit_dates = [parse_iso8601_date_to_utc(v["date"]) for v in origin_visits]
i = bisect_right(visit_dates, save_request.request_date)
if i != len(visit_dates):
visit_date = visit_dates[i]
visit_status = origin_visits[i]["status"]
except Exception as exc:
sentry_sdk.capture_exception(exc)
return visit_date, visit_status
def _check_visit_update_status(
save_request: SaveOriginRequest,
) -> Tuple[Optional[datetime], Optional[str], Optional[str]]:
"""Given a save request, determine whether a save request was successful or failed.
Args:
save_request: Input save origin request to retrieve information for.
Returns:
Tuple of (optional visit date, optional visit status, optional save task status)
for such save request origin
"""
visit_date, visit_status = _get_visit_info_for_save_request(save_request)
loading_task_status = None
if visit_date and visit_status in ("full", "partial"):
# visit has been performed, mark the saving task as succeeded
loading_task_status = SAVE_TASK_SUCCEEDED
elif visit_status in ("created", "ongoing"):
# visit is currently running
loading_task_status = SAVE_TASK_RUNNING
elif visit_status in ("not_found", "failed"):
loading_task_status = SAVE_TASK_FAILED
else:
time_now = datetime.now(tz=timezone.utc)
time_delta = time_now - save_request.request_date
# consider the task as failed if it is still in scheduled state
# 30 days after its submission
if time_delta.days > MAX_THRESHOLD_DAYS:
loading_task_status = SAVE_TASK_FAILED
return visit_date, visit_status, loading_task_status
def _compute_task_loading_status(
task: Optional[Dict[str, Any]] = None, task_run: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
loading_task_status: Optional[str] = None
# First determine the loading task status out of task information
if task:
loading_task_status = _save_task_status[task["status"]]
if task_run:
loading_task_status = _save_task_run_status[task_run["status"]]
return loading_task_status
def _update_save_request_info(
save_request: SaveOriginRequest,
task: Optional[Dict[str, Any]] = None,
task_run: Optional[Dict[str, Any]] = None,
) -> SaveOriginRequestInfo:
"""Update save request information out of the visit status and fallback to the task and
task_run information if the visit status is missing.
Args:
save_request: Save request
task: Associated scheduler task information about the save request
task_run: Most recent run occurrence of the associated task
Returns:
Summary of the save request information updated.
"""
must_save = False
# To determine the save code now request's final status, the visit date must be set
# and the visit status must be a final one. Once they do, the save code now is
# definitely done.
if (
not save_request.visit_date
or not save_request.visit_status
or save_request.visit_status in NON_TERMINAL_STATUSES
):
visit_date, visit_status, loading_task_status = _check_visit_update_status(
save_request
)
if not loading_task_status: # fallback when not provided
loading_task_status = _compute_task_loading_status(task, task_run)
if visit_date != save_request.visit_date:
must_save = True
save_request.visit_date = visit_date
if visit_status != save_request.visit_status:
must_save = True
save_request.visit_status = visit_status
if (
loading_task_status is not None
and loading_task_status != save_request.loading_task_status
):
must_save = True
save_request.loading_task_status = loading_task_status
if must_save:
save_request.save()
return save_request.to_dict()
def create_save_origin_request(
visit_type: str,
origin_url: str,
privileged_user: bool = False,
user_id: Optional[int] = None,
**kwargs,
) -> SaveOriginRequestInfo:
"""Create a loading task to save a software origin into the archive.
This function aims to create a software origin loading task trough the use of the
swh-scheduler component.
First, some checks are performed to see if the visit type and origin url are valid
but also if the the save request can be accepted. For the 'archives' visit type,
this also ensures the artifacts actually exists. If those checks passed, the loading
task is then created. Otherwise, the save request is put in pending or rejected
state.
All the submitted save requests are logged into the swh-web database to keep track
of them.
Args:
visit_type: the type of visit to perform (e.g. git, hg, svn, archives, ...)
origin_url: the url of the origin to save
privileged: Whether the user has some more privilege than other (bypass
review, access to privileged other visit types)
user_id: User identifier (provided when authenticated)
kwargs: Optional parameters (e.g. artifact_url, artifact_filename,
artifact_version)
Raises:
BadInputExc: the visit type or origin url is invalid or inexistent
ForbiddenExc: the provided origin url is blacklisted
Returns:
dict: A dict describing the save request with the following keys:
* **visit_type**: the type of visit to perform
* **origin_url**: the url of the origin
* **save_request_date**: the date the request was submitted
* **save_request_status**: the request status, either **accepted**,
**rejected** or **pending**
* **save_task_status**: the origin loading task status, either
**not created**, **not yet scheduled**, **scheduled**,
**succeed** or **failed**
"""
visit_type_tasks = get_savable_visit_types_dict(privileged_user)
_check_visit_type_savable(visit_type, privileged_user)
_check_origin_url_valid(origin_url)
# if all checks passed so far, we can try and save the origin
save_request_status = can_save_origin(origin_url, privileged_user)
task = None
# if the origin save request is accepted, create a scheduler
# task to load it into the archive
if save_request_status == SAVE_REQUEST_ACCEPTED:
# create a task with high priority
task_kwargs: Dict[str, Any] = {
"priority": "high",
"url": origin_url,
}
if visit_type == "archives":
# extra arguments for that type are required
archives_data = kwargs.get("archives_data", [])
if not archives_data:
raise BadInputExc(
"Artifacts data are missing for the archives visit type."
)
artifacts = []
for artifact in archives_data:
artifact_url = artifact.get("artifact_url")
artifact_version = artifact.get("artifact_version")
if not artifact_url or not artifact_version:
raise BadInputExc("Missing url or version for an artifact to load.")
metadata = _check_origin_exists(artifact_url)
artifacts.append(
{
"url": artifact_url,
"version": artifact_version,
"time": metadata["last_modified"],
"length": metadata["content_length"],
}
)
task_kwargs = dict(**task_kwargs, artifacts=artifacts, snapshot_append=True)
sor = None
# get list of previously submitted save requests (most recent first)
current_sors = list(
SaveOriginRequest.objects.filter(
visit_type=visit_type, origin_url=origin_url
).order_by("-request_date")
)
can_create_task = False
# if no save requests previously submitted, create the scheduler task
if not current_sors:
can_create_task = True
else:
# get the latest submitted save request
sor = current_sors[0]
# if it was in pending state, we need to create the scheduler task
# and update the save request info in the database
if sor.status == SAVE_REQUEST_PENDING:
can_create_task = True
# a task has already been created to load the origin
elif sor.loading_task_id != -1:
# get the scheduler task and its status
tasks = scheduler().get_tasks([sor.loading_task_id])
task = tasks[0] if tasks else None
task_runs = scheduler().get_task_runs([sor.loading_task_id])
task_run = task_runs[0] if task_runs else None
save_request_info = _update_save_request_info(sor, task, task_run)
task_status = save_request_info["save_task_status"]
# create a new scheduler task only if the previous one has been
# already executed
if (
task_status == SAVE_TASK_FAILED
or task_status == SAVE_TASK_SUCCEEDED
):
can_create_task = True
sor = None
else:
can_create_task = False
if can_create_task:
# effectively create the scheduler task
task_dict = create_oneshot_task_dict(
visit_type_tasks[visit_type], **task_kwargs
)
task = scheduler().create_tasks([task_dict])[0]
# pending save request has been accepted
if sor:
sor.status = SAVE_REQUEST_ACCEPTED
sor.loading_task_id = task["id"]
sor.save()
else:
sor = SaveOriginRequest.objects.create(
visit_type=visit_type,
origin_url=origin_url,
status=save_request_status,
loading_task_id=task["id"],
user_ids=f'"{user_id}"' if user_id else None,
)
# save request must be manually reviewed for acceptation
elif save_request_status == SAVE_REQUEST_PENDING:
# check if there is already such a save request already submitted,
# no need to add it to the database in that case
try:
sor = SaveOriginRequest.objects.get(
visit_type=visit_type, origin_url=origin_url, status=save_request_status
)
user_ids = sor.user_ids if sor.user_ids is not None else ""
if user_id is not None and f'"{user_id}"' not in user_ids:
# update user ids list
sor.user_ids = f'{sor.user_ids},"{user_id}"'
sor.save()
# if not add it to the database
except ObjectDoesNotExist:
sor = SaveOriginRequest.objects.create(
visit_type=visit_type,
origin_url=origin_url,
status=save_request_status,
user_ids=f'"{user_id}"' if user_id else None,
)
# origin can not be saved as its url is blacklisted,
# log the request to the database anyway
else:
sor = SaveOriginRequest.objects.create(
visit_type=visit_type,
origin_url=origin_url,
status=save_request_status,
user_ids=f'"{user_id}"' if user_id else None,
)
if save_request_status == SAVE_REQUEST_REJECTED:
raise ForbiddenExc(
(
'The "save code now" request has been rejected '
"because the provided origin url is blacklisted."
)
)
assert sor is not None
return _update_save_request_info(sor, task)
def update_save_origin_requests_from_queryset(
requests_queryset: QuerySet,
) -> List[SaveOriginRequestInfo]:
"""Update all save requests from a SaveOriginRequest queryset, update their status in db
and return the list of impacted save_requests.
Args:
requests_queryset: input SaveOriginRequest queryset
Returns:
list: A list of save origin request info dicts as described in
:func:`swh.web.common.origin_save.create_save_origin_request`
"""
task_ids = []
for sor in requests_queryset:
task_ids.append(sor.loading_task_id)
save_requests = []
if task_ids:
try:
tasks = scheduler().get_tasks(task_ids)
tasks = {task["id"]: task for task in tasks}
task_runs = scheduler().get_task_runs(tasks)
task_runs = {task_run["task"]: task_run for task_run in task_runs}
except Exception:
# allow to avoid mocking api GET responses for /origin/save endpoint when
# running cypress tests as scheduler is not available
tasks = {}
task_runs = {}
for sor in requests_queryset:
sr_dict = _update_save_request_info(
sor, tasks.get(sor.loading_task_id), task_runs.get(sor.loading_task_id),
)
save_requests.append(sr_dict)
return save_requests
def refresh_save_origin_request_statuses() -> List[SaveOriginRequestInfo]:
"""Refresh non-terminal save origin requests (SOR) in the backend.
Non-terminal SOR are requests whose status is **accepted** and their task status are
either **created**, **not yet scheduled**, **scheduled** or **running**.
This shall compute this list of SOR, checks their status in the scheduler and
optionally elasticsearch for their current status. Then update those in db.
Finally, this returns the refreshed information on those SOR.
"""
pivot_date = datetime.now(tz=timezone.utc) - timedelta(days=MAX_THRESHOLD_DAYS)
save_requests = SaveOriginRequest.objects.filter(
# Retrieve accepted request statuses (all statuses)
Q(status=SAVE_REQUEST_ACCEPTED),
# those without the required information we need to update
Q(visit_date__isnull=True)
| Q(visit_status__isnull=True)
| Q(visit_status__in=NON_TERMINAL_STATUSES),
# limit results to recent ones (that is roughly 30 days old at best)
Q(request_date__gte=pivot_date),
)
return (
update_save_origin_requests_from_queryset(save_requests)
if save_requests.count() > 0
else []
)
def get_save_origin_requests(
visit_type: str, origin_url: str
) -> List[SaveOriginRequestInfo]:
"""
Get all save requests for a given software origin.
Args:
visit_type: the type of visit
origin_url: the url of the origin
Raises:
BadInputExc: the visit type or origin url is invalid
swh.web.common.exc.NotFoundExc: no save requests can be found for the
given origin
Returns:
list: A list of save origin requests dict as described in
:func:`swh.web.common.origin_save.create_save_origin_request`
"""
_check_visit_type_savable(visit_type)
_check_origin_url_valid(origin_url)
sors = SaveOriginRequest.objects.filter(
visit_type=visit_type, origin_url=origin_url
)
if sors.count() == 0:
raise NotFoundExc(
f"No save requests found for visit of type {visit_type} "
f"on origin with url {origin_url}."
)
return update_save_origin_requests_from_queryset(sors)
def get_save_origin_task_info(
save_request_id: int, full_info: bool = True
) -> Dict[str, Any]:
"""
Get detailed information about an accepted save origin request
and its associated loading task.
If the associated loading task info is archived and removed
from the scheduler database, returns an empty dictionary.
Args:
save_request_id: identifier of a save origin request
full_info: whether to return detailed info for staff users
Returns:
A dictionary with the following keys:
- **type**: loading task type
- **arguments**: loading task arguments
- **id**: loading task database identifier
- **backend_id**: loading task celery identifier
- **scheduled**: loading task scheduling date
- **ended**: loading task termination date
- **status**: loading task execution status
- **visit_status**: Actual visit status
Depending on the availability of the task logs in the elasticsearch
cluster of Software Heritage, the returned dictionary may also
contain the following keys:
- **name**: associated celery task name
- **message**: relevant log message from task execution
- **duration**: task execution time (only if it succeeded)
- **worker**: name of the worker that executed the task
"""
try:
save_request = SaveOriginRequest.objects.get(id=save_request_id)
except ObjectDoesNotExist:
return {}
task_info: Dict[str, Any] = {}
if save_request.note is not None:
task_info["note"] = save_request.note
try:
task = scheduler().get_tasks([save_request.loading_task_id])
except Exception:
# to avoid mocking GET responses of /save/task/info/ endpoint when running
# cypress tests as scheduler is not available in that case
task = None
task = task[0] if task else None
if task is None:
return task_info
task_run = scheduler().get_task_runs([task["id"]])
task_run = task_run[0] if task_run else None
if task_run is None:
return task_info
task_info.update(task_run)
task_info["type"] = task["type"]
task_info["arguments"] = task["arguments"]
task_info["id"] = task_run["task"]
del task_info["task"]
del task_info["metadata"]
# Enrich the task info with the loading visit status
task_info["visit_status"] = save_request.visit_status
es_workers_index_url = get_config()["es_workers_index_url"]
if not es_workers_index_url:
return task_info
es_workers_index_url += "/_search"
if save_request.visit_date:
min_ts = save_request.visit_date
max_ts = min_ts + timedelta(days=7)
else:
min_ts = save_request.request_date
max_ts = min_ts + timedelta(days=MAX_THRESHOLD_DAYS)
min_ts_unix = int(min_ts.timestamp()) * 1000
max_ts_unix = int(max_ts.timestamp()) * 1000
save_task_status = _save_task_status[task["status"]]
priority = "3" if save_task_status == SAVE_TASK_FAILED else "6"
query = {
"bool": {
"must": [
- {"match_phrase": {"priority": {"query": priority}}},
- {"match_phrase": {"swh_task_id": {"query": task_run["backend_id"]}}},
+ {"match_phrase": {"syslog.priority": {"query": priority}}},
+ {
+ "match_phrase": {
+ "journald.custom.swh_task_id": {"query": task_run["backend_id"]}
+ }
+ },
{
"range": {
"@timestamp": {
"gte": min_ts_unix,
"lte": max_ts_unix,
"format": "epoch_millis",
}
}
},
]
}
}
try:
response = requests.post(
es_workers_index_url,
json={"query": query, "sort": ["@timestamp"]},
timeout=30,
)
results = json.loads(response.text)
if results["hits"]["total"]["value"] >= 1:
task_run_info = results["hits"]["hits"][-1]["_source"]
if "swh_logging_args_runtime" in task_run_info:
duration = task_run_info["swh_logging_args_runtime"]
task_info["duration"] = duration
if "message" in task_run_info:
task_info["message"] = task_run_info["message"]
if "swh_logging_args_name" in task_run_info:
task_info["name"] = task_run_info["swh_logging_args_name"]
elif "swh_task_name" in task_run_info:
task_info["name"] = task_run_info["swh_task_name"]
if "hostname" in task_run_info:
task_info["worker"] = task_run_info["hostname"]
elif "host" in task_run_info:
task_info["worker"] = task_run_info["host"]
except Exception as exc:
logger.warning("Request to Elasticsearch failed\n%s", exc)
sentry_sdk.capture_exception(exc)
if not full_info:
for field in ("id", "backend_id", "worker"):
# remove some staff only fields
task_info.pop(field, None)
if "message" in task_run and "Loading failure" in task_run["message"]:
# hide traceback for non staff users, only display exception
message_lines = task_info["message"].split("\n")
message = ""
for line in message_lines:
if line.startswith("Traceback"):
break
message += f"{line}\n"
message += message_lines[-1]
task_info["message"] = message
return task_info
SUBMITTED_SAVE_REQUESTS_METRIC = "swh_web_submitted_save_requests"
_submitted_save_requests_gauge = Gauge(
name=SUBMITTED_SAVE_REQUESTS_METRIC,
documentation="Number of submitted origin save requests",
labelnames=["status", "visit_type"],
registry=SWH_WEB_METRICS_REGISTRY,
)
ACCEPTED_SAVE_REQUESTS_METRIC = "swh_web_accepted_save_requests"
_accepted_save_requests_gauge = Gauge(
name=ACCEPTED_SAVE_REQUESTS_METRIC,
documentation="Number of accepted origin save requests",
labelnames=["load_task_status", "visit_type"],
registry=SWH_WEB_METRICS_REGISTRY,
)
# Metric on the delay of save code now request per status and visit_type. This is the
# time difference between the save code now is requested and the time it got ingested.
ACCEPTED_SAVE_REQUESTS_DELAY_METRIC = "swh_web_save_requests_delay_seconds"
_accepted_save_requests_delay_gauge = Gauge(
name=ACCEPTED_SAVE_REQUESTS_DELAY_METRIC,
documentation="Save Requests Duration",
labelnames=["load_task_status", "visit_type"],
registry=SWH_WEB_METRICS_REGISTRY,
)
def compute_save_requests_metrics() -> None:
"""Compute Prometheus metrics related to origin save requests:
- Number of submitted origin save requests
- Number of accepted origin save requests
- Save Code Now requests delay between request time and actual time of ingestion
"""
request_statuses = (
SAVE_REQUEST_ACCEPTED,
SAVE_REQUEST_REJECTED,
SAVE_REQUEST_PENDING,
)
load_task_statuses = (
SAVE_TASK_NOT_CREATED,
SAVE_TASK_NOT_YET_SCHEDULED,
SAVE_TASK_SCHEDULED,
SAVE_TASK_SUCCEEDED,
SAVE_TASK_FAILED,
SAVE_TASK_RUNNING,
)
# for metrics, we want access to all visit types
visit_types = get_savable_visit_types(privileged_user=True)
labels_set = product(request_statuses, visit_types)
for labels in labels_set:
_submitted_save_requests_gauge.labels(*labels).set(0)
labels_set = product(load_task_statuses, visit_types)
for labels in labels_set:
_accepted_save_requests_gauge.labels(*labels).set(0)
duration_load_task_statuses = (
SAVE_TASK_FAILED,
SAVE_TASK_SUCCEEDED,
)
for labels in product(duration_load_task_statuses, visit_types):
_accepted_save_requests_delay_gauge.labels(*labels).set(0)
for sor in SaveOriginRequest.objects.all():
if sor.status == SAVE_REQUEST_ACCEPTED:
_accepted_save_requests_gauge.labels(
load_task_status=sor.loading_task_status, visit_type=sor.visit_type,
).inc()
_submitted_save_requests_gauge.labels(
status=sor.status, visit_type=sor.visit_type
).inc()
if (
sor.loading_task_status in (SAVE_TASK_SUCCEEDED, SAVE_TASK_FAILED)
and sor.visit_date is not None
and sor.request_date is not None
):
delay = sor.visit_date.timestamp() - sor.request_date.timestamp()
_accepted_save_requests_delay_gauge.labels(
load_task_status=sor.loading_task_status, visit_type=sor.visit_type,
).inc(delay)
diff --git a/swh/web/tests/api/views/test_release.py b/swh/web/tests/api/views/test_release.py
index f985217b..a847973f 100644
--- a/swh/web/tests/api/views/test_release.py
+++ b/swh/web/tests/api/views/test_release.py
@@ -1,116 +1,104 @@
# Copyright (C) 2015-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from datetime import datetime
+from datetime import datetime, timezone
from swh.model.hashutil import hash_to_bytes, hash_to_hex
-from swh.model.model import (
- ObjectType,
- Person,
- Release,
- Timestamp,
- TimestampWithTimezone,
-)
+from swh.model.model import ObjectType, Person, Release, TimestampWithTimezone
from swh.web.common.utils import reverse
from swh.web.tests.data import random_sha1
from swh.web.tests.utils import check_api_get_responses, check_http_get_response
def test_api_release(api_client, archive_data, release):
url = reverse("api-1-release", url_args={"sha1_git": release})
rv = check_api_get_responses(api_client, url, status_code=200)
expected_release = archive_data.release_get(release)
target_revision = expected_release["target"]
target_url = reverse(
"api-1-revision",
url_args={"sha1_git": target_revision},
request=rv.wsgi_request,
)
expected_release["target_url"] = target_url
assert rv.data == expected_release
def test_api_release_target_type_not_a_revision(
api_client, archive_data, content, directory, release
):
for target_type, target in (
(ObjectType.CONTENT, content),
(ObjectType.DIRECTORY, directory),
(ObjectType.RELEASE, release),
):
if target_type == ObjectType.CONTENT:
target = target["sha1_git"]
sample_release = Release(
author=Person(
email=b"author@company.org",
fullname=b"author <author@company.org>",
name=b"author",
),
- date=TimestampWithTimezone(
- timestamp=Timestamp(
- seconds=int(datetime.now().timestamp()), microseconds=0
- ),
- offset=0,
- negative_utc=False,
- ),
+ date=TimestampWithTimezone.from_datetime(datetime.now(tz=timezone.utc)),
message=b"sample release message",
name=b"sample release",
synthetic=False,
target=hash_to_bytes(target),
target_type=target_type,
)
archive_data.release_add([sample_release])
new_release_id = hash_to_hex(sample_release.id)
url = reverse("api-1-release", url_args={"sha1_git": new_release_id})
rv = check_api_get_responses(api_client, url, status_code=200)
expected_release = archive_data.release_get(new_release_id)
if target_type == ObjectType.CONTENT:
url_args = {"q": "sha1_git:%s" % target}
else:
url_args = {"sha1_git": target}
target_url = reverse(
"api-1-%s" % target_type.value, url_args=url_args, request=rv.wsgi_request
)
expected_release["target_url"] = target_url
assert rv.data == expected_release
def test_api_release_not_found(api_client):
unknown_release_ = random_sha1()
url = reverse("api-1-release", url_args={"sha1_git": unknown_release_})
rv = check_api_get_responses(api_client, url, status_code=404)
assert rv.data == {
"exception": "NotFoundExc",
"reason": "Release with sha1_git %s not found." % unknown_release_,
}
def test_api_release_uppercase(api_client, release):
url = reverse(
"api-1-release-uppercase-checksum", url_args={"sha1_git": release.upper()}
)
resp = check_http_get_response(api_client, url, status_code=302)
redirect_url = reverse(
"api-1-release-uppercase-checksum", url_args={"sha1_git": release}
)
assert resp["location"] == redirect_url
diff --git a/swh/web/tests/browse/views/test_snapshot.py b/swh/web/tests/browse/views/test_snapshot.py
index f7388235..e4e929c8 100644
--- a/swh/web/tests/browse/views/test_snapshot.py
+++ b/swh/web/tests/browse/views/test_snapshot.py
@@ -1,385 +1,427 @@
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import random
import re
import string
from dateutil import parser
from hypothesis import given
import pytest
from django.utils.html import escape
from swh.model.hashutil import hash_to_bytes
from swh.model.model import (
ObjectType,
OriginVisit,
OriginVisitStatus,
Release,
+ Revision,
+ RevisionType,
Snapshot,
SnapshotBranch,
TargetType,
+ TimestampWithTimezone,
)
from swh.storage.utils import now
from swh.web.browse.snapshot_context import process_snapshot_branches
from swh.web.common.utils import reverse
from swh.web.tests.data import random_sha1
from swh.web.tests.django_asserts import assert_contains, assert_not_contains
-from swh.web.tests.strategies import new_origin, visit_dates
+from swh.web.tests.strategies import new_origin, new_person, new_swh_date, visit_dates
from swh.web.tests.utils import check_html_get_response
@pytest.mark.parametrize(
"browse_context,template_used",
[
("log", "revision-log.html"),
("branches", "branches.html"),
("releases", "releases.html"),
],
)
def test_snapshot_browse_with_id(client, browse_context, template_used, snapshot):
url = reverse(
f"browse-snapshot-{browse_context}", url_args={"snapshot_id": snapshot}
)
resp = check_html_get_response(
client, url, status_code=200, template_used=f"browse/{template_used}"
)
assert_contains(resp, f"swh:1:snp:{snapshot}")
@pytest.mark.parametrize("browse_context", ["log", "branches", "releases"])
def test_snapshot_browse_with_id_and_origin(
client, browse_context, archive_data, origin
):
snapshot = archive_data.snapshot_get_latest(origin["url"])
url = reverse(
f"browse-snapshot-{browse_context}",
url_args={"snapshot_id": snapshot["id"]},
query_params={"origin_url": origin["url"]},
)
resp = check_html_get_response(
client, url, status_code=200, template_used="includes/snapshot-context.html"
)
assert_contains(resp, origin["url"])
@pytest.mark.parametrize("browse_context", ["log", "branches", "releases"])
def test_snapshot_browse_with_id_origin_and_timestamp(
client, browse_context, archive_data, origin_with_multiple_visits
):
visit = archive_data.origin_visit_get(origin_with_multiple_visits["url"])[0]
url = reverse(
f"browse-snapshot-{browse_context}",
url_args={"snapshot_id": visit["snapshot"]},
query_params={"origin_url": visit["origin"], "timestamp": visit["date"]},
)
resp = check_html_get_response(
client, url, status_code=200, template_used="includes/snapshot-context.html"
)
requested_time = parser.parse(visit["date"]).strftime("%d %B %Y, %H:%M")
assert_contains(resp, requested_time)
assert_contains(resp, visit["origin"])
@pytest.mark.parametrize("browse_context", ["log", "branches", "releases"])
def test_snapshot_browse_without_id(client, browse_context, archive_data, origin):
url = reverse(
f"browse-snapshot-{browse_context}", query_params={"origin_url": origin["url"]}
)
# This will be redirected to /snapshot/<latest_snapshot_id>/log
resp = check_html_get_response(client, url, status_code=302,)
snapshot = archive_data.snapshot_get_latest(origin["url"])
assert resp.url == reverse(
f"browse-snapshot-{browse_context}",
url_args={"snapshot_id": snapshot["id"]},
query_params={"origin_url": origin["url"]},
)
@pytest.mark.parametrize("browse_context", ["log", "branches", "releases"])
def test_snapshot_browse_without_id_and_origin(client, browse_context):
url = reverse(f"browse-snapshot-{browse_context}")
resp = check_html_get_response(client, url, status_code=400,)
# assert_contains works only with a success response, using re.search instead
assert re.search(
"An origin URL must be provided as a query parameter",
resp.content.decode("utf-8"),
)
def test_snapshot_browse_branches(client, archive_data, origin):
snapshot = archive_data.snapshot_get_latest(origin["url"])
snapshot_sizes = archive_data.snapshot_count_branches(snapshot["id"])
snapshot_content = process_snapshot_branches(snapshot)
_origin_branches_test_helper(
client, origin, snapshot_content, snapshot_sizes, snapshot_id=snapshot["id"]
)
def _origin_branches_test_helper(
client, origin_info, origin_snapshot, snapshot_sizes, snapshot_id
):
query_params = {"origin_url": origin_info["url"], "snapshot": snapshot_id}
url = reverse(
"browse-snapshot-branches",
url_args={"snapshot_id": snapshot_id},
query_params=query_params,
)
resp = check_html_get_response(
client, url, status_code=200, template_used="browse/branches.html"
)
origin_branches = origin_snapshot[0]
origin_releases = origin_snapshot[1]
origin_branches_url = reverse("browse-origin-branches", query_params=query_params)
assert_contains(resp, f'href="{escape(origin_branches_url)}"')
assert_contains(resp, f"Branches ({snapshot_sizes['revision']})")
origin_releases_url = reverse("browse-origin-releases", query_params=query_params)
nb_releases = len(origin_releases)
if nb_releases > 0:
assert_contains(resp, f'href="{escape(origin_releases_url)}">')
assert_contains(resp, f"Releases ({snapshot_sizes['release']})")
assert_contains(resp, '<tr class="swh-branch-entry', count=len(origin_branches))
for branch in origin_branches:
browse_branch_url = reverse(
"browse-origin-directory",
query_params={"branch": branch["name"], **query_params},
)
assert_contains(resp, '<a href="%s">' % escape(browse_branch_url))
browse_revision_url = reverse(
"browse-revision",
url_args={"sha1_git": branch["revision"]},
query_params=query_params,
)
assert_contains(resp, '<a href="%s">' % escape(browse_revision_url))
_check_origin_link(resp, origin_info["url"])
def _check_origin_link(resp, origin_url):
browse_origin_url = reverse(
"browse-origin", query_params={"origin_url": origin_url}
)
assert_contains(resp, f'href="{browse_origin_url}"')
@given(
new_origin(), visit_dates(),
)
def test_snapshot_branches_pagination_with_alias(
client, archive_data, mocker, release, revisions_list, new_origin, visit_dates,
):
"""
When a snapshot contains a branch or a release alias, pagination links
in the branches / releases view should be displayed.
"""
revisions = revisions_list(size=10)
mocker.patch("swh.web.browse.snapshot_context.PER_PAGE", len(revisions) / 2)
snp_dict = {"branches": {}, "id": hash_to_bytes(random_sha1())}
for i in range(len(revisions)):
branch = "".join(random.choices(string.ascii_lowercase, k=8))
snp_dict["branches"][branch.encode()] = {
"target_type": "revision",
"target": hash_to_bytes(revisions[i]),
}
release_name = "".join(random.choices(string.ascii_lowercase, k=8))
snp_dict["branches"][b"RELEASE_ALIAS"] = {
"target_type": "alias",
"target": release_name.encode(),
}
snp_dict["branches"][release_name.encode()] = {
"target_type": "release",
"target": hash_to_bytes(release),
}
archive_data.origin_add([new_origin])
archive_data.snapshot_add([Snapshot.from_dict(snp_dict)])
visit = archive_data.origin_visit_add(
[OriginVisit(origin=new_origin.url, date=visit_dates[0], type="git",)]
)[0]
visit_status = OriginVisitStatus(
origin=new_origin.url,
visit=visit.visit,
date=now(),
status="full",
snapshot=snp_dict["id"],
)
archive_data.origin_visit_status_add([visit_status])
snapshot = archive_data.snapshot_get_latest(new_origin.url)
url = reverse(
"browse-snapshot-branches",
url_args={"snapshot_id": snapshot["id"]},
query_params={"origin_url": new_origin.url},
)
resp = check_html_get_response(
client, url, status_code=200, template_used="browse/branches.html"
)
assert_contains(resp, '<ul class="pagination')
def test_pull_request_branches_filtering(
client, origin_with_pull_request_branches, archive_data
):
origin_url = origin_with_pull_request_branches.url
# check no pull request branches are displayed in the Branches / Releases dropdown
url = reverse("browse-origin-directory", query_params={"origin_url": origin_url})
resp = check_html_get_response(
client, url, status_code=200, template_used="browse/directory.html"
)
assert_not_contains(resp, "refs/pull/")
snapshot = archive_data.snapshot_get_latest(origin_url)
# check no pull request branches are displayed in the branches view
url = reverse(
"browse-snapshot-branches",
url_args={"snapshot_id": snapshot["id"]},
query_params={"origin_url": origin_url},
)
resp = check_html_get_response(
client, url, status_code=200, template_used="browse/branches.html"
)
assert_not_contains(resp, "refs/pull/")
def test_snapshot_browse_releases(client, archive_data, origin):
origin_visits = archive_data.origin_visit_get(origin["url"])
visit = origin_visits[-1]
snapshot = archive_data.snapshot_get(visit["snapshot"])
snapshot_sizes = archive_data.snapshot_count_branches(snapshot["id"])
snapshot_content = process_snapshot_branches(snapshot)
_origin_releases_test_helper(
client, origin, snapshot_content, snapshot_sizes, snapshot_id=visit["snapshot"]
)
def _origin_releases_test_helper(
client, origin_info, origin_snapshot, snapshot_sizes, snapshot_id=None
):
query_params = {"origin_url": origin_info["url"], "snapshot": snapshot_id}
url = reverse(
"browse-snapshot-releases",
url_args={"snapshot_id": snapshot_id},
query_params=query_params,
)
resp = check_html_get_response(
client, url, status_code=200, template_used="browse/releases.html"
)
origin_releases = origin_snapshot[1]
origin_branches_url = reverse("browse-origin-branches", query_params=query_params)
assert_contains(resp, f'href="{escape(origin_branches_url)}"')
assert_contains(resp, f"Branches ({snapshot_sizes['revision']})")
origin_releases_url = reverse("browse-origin-releases", query_params=query_params)
nb_releases = len(origin_releases)
if nb_releases > 0:
assert_contains(resp, f'href="{escape(origin_releases_url)}"')
assert_contains(resp, f"Releases ({snapshot_sizes['release']}")
assert_contains(resp, '<tr class="swh-release-entry', count=nb_releases)
assert_contains(resp, 'title="The release', count=nb_releases)
for release in origin_releases:
query_params["release"] = release["name"]
browse_release_url = reverse(
"browse-release",
url_args={"sha1_git": release["id"]},
query_params=query_params,
)
browse_revision_url = reverse(
"browse-revision",
url_args={"sha1_git": release["target"]},
query_params=query_params,
)
assert_contains(resp, '<a href="%s">' % escape(browse_release_url))
assert_contains(resp, '<a href="%s">' % escape(browse_revision_url))
_check_origin_link(resp, origin_info["url"])
def test_snapshot_content_redirect(client, snapshot):
qry = {"extra-arg": "extra"}
url = reverse(
"browse-snapshot-content", url_args={"snapshot_id": snapshot}, query_params=qry
)
resp = check_html_get_response(client, url, status_code=301)
assert resp.url == reverse(
"browse-content", query_params={**{"snapshot_id": snapshot}, **qry}
)
def test_snapshot_content_legacy_redirect(client, snapshot):
qry = {"extra-arg": "extra"}
url_args = {"snapshot_id": snapshot, "path": "test.txt"}
url = reverse("browse-snapshot-content-legacy", url_args=url_args, query_params=qry)
resp = check_html_get_response(client, url, status_code=301)
assert resp.url == reverse("browse-content", query_params={**url_args, **qry})
def test_browse_snapshot_log_no_revisions(client, archive_data, directory):
release_name = "v1.0.0"
release = Release(
name=release_name.encode(),
message=f"release {release_name}".encode(),
target=hash_to_bytes(directory),
target_type=ObjectType.DIRECTORY,
synthetic=True,
)
archive_data.release_add([release])
snapshot = Snapshot(
branches={
b"HEAD": SnapshotBranch(
target=release_name.encode(), target_type=TargetType.ALIAS
),
release_name.encode(): SnapshotBranch(
target=release.id, target_type=TargetType.RELEASE
),
},
)
archive_data.snapshot_add([snapshot])
snp_url = reverse(
"browse-snapshot-directory", url_args={"snapshot_id": snapshot.id.hex()}
)
log_url = reverse(
"browse-snapshot-log", url_args={"snapshot_id": snapshot.id.hex()}
)
resp = check_html_get_response(
client, snp_url, status_code=200, template_used="browse/directory.html"
)
assert_not_contains(resp, log_url)
resp = check_html_get_response(
client, log_url, status_code=404, template_used="error.html"
)
assert_contains(
resp,
"No revisions history found in the current snapshot context.",
status_code=404,
)
+
+
+@given(new_person(), new_swh_date())
+def test_browse_snapshot_log_when_revisions(
+ client, archive_data, directory, person, date
+):
+
+ revision = Revision(
+ directory=hash_to_bytes(directory),
+ author=person,
+ committer=person,
+ message=b"commit message",
+ date=TimestampWithTimezone.from_datetime(date),
+ committer_date=TimestampWithTimezone.from_datetime(date),
+ synthetic=False,
+ type=RevisionType.GIT,
+ )
+ archive_data.revision_add([revision])
+
+ snapshot = Snapshot(
+ branches={
+ b"HEAD": SnapshotBranch(
+ target=revision.id, target_type=TargetType.REVISION
+ ),
+ },
+ )
+ archive_data.snapshot_add([snapshot])
+
+ snp_url = reverse(
+ "browse-snapshot-directory", url_args={"snapshot_id": snapshot.id.hex()}
+ )
+ log_url = reverse(
+ "browse-snapshot-log", url_args={"snapshot_id": snapshot.id.hex()}
+ )
+
+ resp = check_html_get_response(
+ client, snp_url, status_code=200, template_used="browse/directory.html"
+ )
+ assert_contains(resp, log_url)
diff --git a/swh/web/tests/common/test_converters.py b/swh/web/tests/common/test_converters.py
index 756bbafa..c00062bb 100644
--- a/swh/web/tests/common/test_converters.py
+++ b/swh/web/tests/common/test_converters.py
@@ -1,758 +1,740 @@
# Copyright (C) 2015-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import datetime
import hashlib
from swh.model import hashutil
from swh.model.model import (
ObjectType,
Person,
Release,
Revision,
RevisionType,
- Timestamp,
TimestampWithTimezone,
)
from swh.web.common import converters
def test_fmap():
assert [2, 3, None, 4] == converters.fmap(lambda x: x + 1, [1, 2, None, 3])
assert [11, 12, 13] == list(
converters.fmap(lambda x: x + 10, map(lambda x: x, [1, 2, 3]))
)
assert {"a": 2, "b": 4} == converters.fmap(lambda x: x * 2, {"a": 1, "b": 2})
assert 100 == converters.fmap(lambda x: x * 10, 10)
assert {"a": [2, 6], "b": 4} == converters.fmap(
lambda x: x * 2, {"a": [1, 3], "b": 2}
)
assert converters.fmap(lambda x: x, None) is None
def test_from_swh():
some_input = {
"a": "something",
"b": "someone",
"c": b"sharp-0.3.4.tgz",
"d": hashutil.hash_to_bytes("b04caf10e9535160d90e874b45aa426de762f19f"),
"e": b"sharp.html/doc_002dS_005fISREG.html",
"g": [b"utf-8-to-decode", b"another-one"],
"h": "something filtered",
"i": {"e": b"something"},
"j": {
"k": {
"l": [b"bytes thing", b"another thingy", b""],
"n": "don't care either",
},
"m": "don't care",
},
"o": "something",
"p": b"foo",
"q": {"extra-headers": [["a", b"intact"]]},
"w": None,
"r": {"p": "also intact", "q": "bar"},
"s": {"timestamp": 42, "offset": -420, "negative_utc": None,},
"s1": {
"timestamp": {"seconds": 42, "microseconds": 0},
"offset": -420,
"negative_utc": None,
},
"s2": datetime.datetime(2013, 7, 1, 20, 0, 0, tzinfo=datetime.timezone.utc),
"t": None,
"u": None,
"v": None,
"x": None,
}
expected_output = {
"a": "something",
"b": "someone",
"c": "sharp-0.3.4.tgz",
"d": "b04caf10e9535160d90e874b45aa426de762f19f",
"e": "sharp.html/doc_002dS_005fISREG.html",
"g": ["utf-8-to-decode", "another-one"],
"i": {"e": "something"},
"j": {"k": {"l": ["bytes thing", "another thingy", ""]}},
"p": "foo",
"q": {"extra-headers": [["a", "intact"]]},
"w": {},
"r": {"p": "also intact", "q": "bar"},
"s": "1969-12-31T17:00:42-07:00",
"s1": "1969-12-31T17:00:42-07:00",
"s2": "2013-07-01T20:00:00+00:00",
"u": {},
"v": [],
"x": None,
}
actual_output = converters.from_swh(
some_input,
hashess={"d", "o", "x"},
bytess={"c", "e", "g", "l"},
dates={"s", "s1", "s2"},
blacklist={"h", "m", "n", "o"},
removables_if_empty={"t"},
empty_dict={"u"},
empty_list={"v"},
convert={"p", "q", "w"},
convert_fn=converters.convert_metadata,
)
assert expected_output == actual_output
def test_from_swh_edge_cases_do_no_conversion_if_none_or_not_bytes():
some_input = {"a": "something", "b": None, "c": "someone", "d": None, "e": None}
expected_output = {
"a": "something",
"b": None,
"c": "someone",
"d": None,
"e": None,
}
actual_output = converters.from_swh(
some_input, hashess={"a", "b"}, bytess={"c", "d"}, dates={"e"}
)
assert expected_output == actual_output
def test_from_swh_edge_cases_convert_invalid_utf8_bytes():
some_input = {
"a": "something",
"b": "someone",
"c": b"a name \xff",
"d": b"an email \xff",
}
expected_output = {
"a": "something",
"b": "someone",
"c": "a name \\xff",
"d": "an email \\xff",
"decoding_failures": ["c", "d"],
}
actual_output = converters.from_swh(
some_input, hashess={"a", "b"}, bytess={"c", "d"}
)
for v in ["a", "b", "c", "d"]:
assert expected_output[v] == actual_output[v]
assert len(expected_output["decoding_failures"]) == len(
actual_output["decoding_failures"]
)
for v in expected_output["decoding_failures"]:
assert v in actual_output["decoding_failures"]
def test_from_swh_empty():
assert {} == converters.from_swh({})
def test_from_swh_none():
assert converters.from_swh(None) is None
def test_from_origin():
origin_url = "rsync://ftp.gnu.org/gnu/octave"
origin_input = {
"id": hashlib.sha1(origin_url.encode("utf-8")).digest(),
"url": origin_url,
}
expected_origin = {
"url": origin_url,
}
actual_origin = converters.from_origin(origin_input)
assert actual_origin == expected_origin
def test_from_origin_visit():
snap_hash = "b5f0b7f716735ebffe38505c60145c4fd9da6ca3"
for snap in [snap_hash, None]:
visit = {
"date": {
"timestamp": datetime.datetime(
2015, 1, 1, 22, 0, 0, tzinfo=datetime.timezone.utc
).timestamp(),
"offset": 0,
"negative_utc": False,
},
"origin": 10,
"visit": 100,
"metadata": None,
"status": "full",
"snapshot": hashutil.hash_to_bytes(snap) if snap else snap,
}
expected_visit = {
"date": "2015-01-01T22:00:00+00:00",
"origin": 10,
"visit": 100,
"metadata": {},
"status": "full",
"snapshot": snap_hash if snap else snap,
}
actual_visit = converters.from_origin_visit(visit)
assert actual_visit == expected_visit
def test_from_release():
"""Convert release model object to a dict should be ok"""
- ts = int(
- datetime.datetime(
- 2015, 1, 1, 22, 0, 0, tzinfo=datetime.timezone.utc
- ).timestamp()
- )
release_input = Release(
id=hashutil.hash_to_bytes("aad23fa492a0c5fed0708a6703be875448c86884"),
target=hashutil.hash_to_bytes("5e46d564378afc44b31bb89f99d5675195fbdf67"),
target_type=ObjectType.REVISION,
- date=TimestampWithTimezone(
- timestamp=Timestamp(seconds=ts, microseconds=0),
- offset=0,
- negative_utc=False,
+ date=TimestampWithTimezone.from_datetime(
+ datetime.datetime(2015, 1, 1, 22, 0, 0, tzinfo=datetime.timezone.utc)
),
author=Person(
name=b"author name",
fullname=b"Author Name author@email",
email=b"author@email",
),
name=b"v0.0.1",
message=b"some comment on release",
synthetic=True,
)
expected_release = {
"id": "aad23fa492a0c5fed0708a6703be875448c86884",
"target": "5e46d564378afc44b31bb89f99d5675195fbdf67",
"target_type": "revision",
"date": "2015-01-01T22:00:00+00:00",
"author": {
"name": "author name",
"fullname": "Author Name author@email",
"email": "author@email",
},
"name": "v0.0.1",
"message": "some comment on release",
"target_type": "revision",
"synthetic": True,
}
actual_release = converters.from_release(release_input)
assert actual_release == expected_release
def test_from_revision_model_object():
- ts = int(
- datetime.datetime(
- 2000, 1, 17, 11, 23, 54, tzinfo=datetime.timezone.utc
- ).timestamp()
+ date = TimestampWithTimezone.from_datetime(
+ datetime.datetime(2000, 1, 17, 11, 23, 54, tzinfo=datetime.timezone.utc)
)
revision_input = Revision(
directory=hashutil.hash_to_bytes("7834ef7e7c357ce2af928115c6c6a42b7e2a44e6"),
author=Person(
name=b"Software Heritage",
fullname=b"robot robot@softwareheritage.org",
email=b"robot@softwareheritage.org",
),
committer=Person(
name=b"Software Heritage",
fullname=b"robot robot@softwareheritage.org",
email=b"robot@softwareheritage.org",
),
message=b"synthetic revision message",
- date=TimestampWithTimezone(
- timestamp=Timestamp(seconds=ts, microseconds=0),
- offset=0,
- negative_utc=False,
- ),
- committer_date=TimestampWithTimezone(
- timestamp=Timestamp(seconds=ts, microseconds=0),
- offset=0,
- negative_utc=False,
- ),
+ date=date,
+ committer_date=date,
synthetic=True,
type=RevisionType.TAR,
parents=tuple(
[
hashutil.hash_to_bytes("29d8be353ed3480476f032475e7c244eff7371d5"),
hashutil.hash_to_bytes("30d8be353ed3480476f032475e7c244eff7371d5"),
]
),
extra_headers=((b"gpgsig", b"some-signature"),),
metadata={
"original_artifact": [
{
"archive_type": "tar",
"name": "webbase-5.7.0.tar.gz",
"sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd",
"sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1",
"sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f"
"309d36484e7edf7bb912",
}
],
},
)
expected_revision = {
"id": "a001358278a0d811fe7072463f805da601121c2a",
"directory": "7834ef7e7c357ce2af928115c6c6a42b7e2a44e6",
"author": {
"name": "Software Heritage",
"fullname": "robot robot@softwareheritage.org",
"email": "robot@softwareheritage.org",
},
"committer": {
"name": "Software Heritage",
"fullname": "robot robot@softwareheritage.org",
"email": "robot@softwareheritage.org",
},
"message": "synthetic revision message",
"date": "2000-01-17T11:23:54+00:00",
"committer_date": "2000-01-17T11:23:54+00:00",
"parents": tuple(
[
"29d8be353ed3480476f032475e7c244eff7371d5",
"30d8be353ed3480476f032475e7c244eff7371d5",
]
),
"type": "tar",
"synthetic": True,
"extra_headers": (("gpgsig", "some-signature"),),
"metadata": {
"original_artifact": [
{
"archive_type": "tar",
"name": "webbase-5.7.0.tar.gz",
"sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd",
"sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1",
"sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f"
"309d36484e7edf7bb912",
}
],
},
"merge": True,
}
actual_revision = converters.from_revision(revision_input)
assert actual_revision == expected_revision
def test_from_revision():
ts = datetime.datetime(
2000, 1, 17, 11, 23, 54, tzinfo=datetime.timezone.utc
).timestamp()
revision_input = {
"id": hashutil.hash_to_bytes("18d8be353ed3480476f032475e7c233eff7371d5"),
"directory": hashutil.hash_to_bytes("7834ef7e7c357ce2af928115c6c6a42b7e2a44e6"),
"author": {
"name": b"Software Heritage",
"fullname": b"robot robot@softwareheritage.org",
"email": b"robot@softwareheritage.org",
},
"committer": {
"name": b"Software Heritage",
"fullname": b"robot robot@softwareheritage.org",
"email": b"robot@softwareheritage.org",
},
"message": b"synthetic revision message",
"date": {"timestamp": ts, "offset": 0, "negative_utc": False,},
"committer_date": {"timestamp": ts, "offset": 0, "negative_utc": False,},
"synthetic": True,
"type": "tar",
"parents": [
hashutil.hash_to_bytes("29d8be353ed3480476f032475e7c244eff7371d5"),
hashutil.hash_to_bytes("30d8be353ed3480476f032475e7c244eff7371d5"),
],
"children": [
hashutil.hash_to_bytes("123546353ed3480476f032475e7c244eff7371d5"),
],
"metadata": {
"extra_headers": [["gpgsig", b"some-signature"]],
"original_artifact": [
{
"archive_type": "tar",
"name": "webbase-5.7.0.tar.gz",
"sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd",
"sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1",
"sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f"
"309d36484e7edf7bb912",
}
],
},
}
expected_revision = {
"id": "18d8be353ed3480476f032475e7c233eff7371d5",
"directory": "7834ef7e7c357ce2af928115c6c6a42b7e2a44e6",
"author": {
"name": "Software Heritage",
"fullname": "robot robot@softwareheritage.org",
"email": "robot@softwareheritage.org",
},
"committer": {
"name": "Software Heritage",
"fullname": "robot robot@softwareheritage.org",
"email": "robot@softwareheritage.org",
},
"message": "synthetic revision message",
"date": "2000-01-17T11:23:54+00:00",
"committer_date": "2000-01-17T11:23:54+00:00",
"children": ["123546353ed3480476f032475e7c244eff7371d5"],
"parents": [
"29d8be353ed3480476f032475e7c244eff7371d5",
"30d8be353ed3480476f032475e7c244eff7371d5",
],
"type": "tar",
"synthetic": True,
"metadata": {
"extra_headers": [["gpgsig", "some-signature"]],
"original_artifact": [
{
"archive_type": "tar",
"name": "webbase-5.7.0.tar.gz",
"sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd",
"sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1",
"sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f"
"309d36484e7edf7bb912",
}
],
},
"merge": True,
}
actual_revision = converters.from_revision(revision_input)
assert actual_revision == expected_revision
def test_from_revision_nomerge():
revision_input = {
"id": hashutil.hash_to_bytes("18d8be353ed3480476f032475e7c233eff7371d5"),
"parents": [hashutil.hash_to_bytes("29d8be353ed3480476f032475e7c244eff7371d5")],
}
expected_revision = {
"id": "18d8be353ed3480476f032475e7c233eff7371d5",
"parents": ["29d8be353ed3480476f032475e7c244eff7371d5"],
"merge": False,
}
actual_revision = converters.from_revision(revision_input)
assert actual_revision == expected_revision
def test_from_revision_noparents():
revision_input = {
"id": hashutil.hash_to_bytes("18d8be353ed3480476f032475e7c233eff7371d5"),
"directory": hashutil.hash_to_bytes("7834ef7e7c357ce2af928115c6c6a42b7e2a44e6"),
"author": {
"name": b"Software Heritage",
"fullname": b"robot robot@softwareheritage.org",
"email": b"robot@softwareheritage.org",
},
"committer": {
"name": b"Software Heritage",
"fullname": b"robot robot@softwareheritage.org",
"email": b"robot@softwareheritage.org",
},
"message": b"synthetic revision message",
"date": {
"timestamp": datetime.datetime(
2000, 1, 17, 11, 23, 54, tzinfo=datetime.timezone.utc
).timestamp(),
"offset": 0,
"negative_utc": False,
},
"committer_date": {
"timestamp": datetime.datetime(
2000, 1, 17, 11, 23, 54, tzinfo=datetime.timezone.utc
).timestamp(),
"offset": 0,
"negative_utc": False,
},
"synthetic": True,
"type": "tar",
"children": [
hashutil.hash_to_bytes("123546353ed3480476f032475e7c244eff7371d5"),
],
"metadata": {
"original_artifact": [
{
"archive_type": "tar",
"name": "webbase-5.7.0.tar.gz",
"sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd",
"sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1",
"sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f"
"309d36484e7edf7bb912",
}
]
},
}
expected_revision = {
"id": "18d8be353ed3480476f032475e7c233eff7371d5",
"directory": "7834ef7e7c357ce2af928115c6c6a42b7e2a44e6",
"author": {
"name": "Software Heritage",
"fullname": "robot robot@softwareheritage.org",
"email": "robot@softwareheritage.org",
},
"committer": {
"name": "Software Heritage",
"fullname": "robot robot@softwareheritage.org",
"email": "robot@softwareheritage.org",
},
"message": "synthetic revision message",
"date": "2000-01-17T11:23:54+00:00",
"committer_date": "2000-01-17T11:23:54+00:00",
"children": ["123546353ed3480476f032475e7c244eff7371d5"],
"type": "tar",
"synthetic": True,
"metadata": {
"original_artifact": [
{
"archive_type": "tar",
"name": "webbase-5.7.0.tar.gz",
"sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd",
"sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1",
"sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f"
"309d36484e7edf7bb912",
}
]
},
}
actual_revision = converters.from_revision(revision_input)
assert actual_revision == expected_revision
def test_from_revision_invalid():
revision_input = {
"id": hashutil.hash_to_bytes("18d8be353ed3480476f032475e7c233eff7371d5"),
"directory": hashutil.hash_to_bytes("7834ef7e7c357ce2af928115c6c6a42b7e2a44e6"),
"author": {
"name": b"Software Heritage",
"fullname": b"robot robot@softwareheritage.org",
"email": b"robot@softwareheritage.org",
},
"committer": {
"name": b"Software Heritage",
"fullname": b"robot robot@softwareheritage.org",
"email": b"robot@softwareheritage.org",
},
"message": b"invalid message \xff",
"date": {
"timestamp": datetime.datetime(
2000, 1, 17, 11, 23, 54, tzinfo=datetime.timezone.utc
).timestamp(),
"offset": 0,
"negative_utc": False,
},
"committer_date": {
"timestamp": datetime.datetime(
2000, 1, 17, 11, 23, 54, tzinfo=datetime.timezone.utc
).timestamp(),
"offset": 0,
"negative_utc": False,
},
"synthetic": True,
"type": "tar",
"parents": [
hashutil.hash_to_bytes("29d8be353ed3480476f032475e7c244eff7371d5"),
hashutil.hash_to_bytes("30d8be353ed3480476f032475e7c244eff7371d5"),
],
"children": [
hashutil.hash_to_bytes("123546353ed3480476f032475e7c244eff7371d5"),
],
"metadata": {
"original_artifact": [
{
"archive_type": "tar",
"name": "webbase-5.7.0.tar.gz",
"sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd",
"sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1",
"sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f"
"309d36484e7edf7bb912",
}
]
},
}
expected_revision = {
"id": "18d8be353ed3480476f032475e7c233eff7371d5",
"directory": "7834ef7e7c357ce2af928115c6c6a42b7e2a44e6",
"author": {
"name": "Software Heritage",
"fullname": "robot robot@softwareheritage.org",
"email": "robot@softwareheritage.org",
},
"committer": {
"name": "Software Heritage",
"fullname": "robot robot@softwareheritage.org",
"email": "robot@softwareheritage.org",
},
"message": "invalid message \\xff",
"decoding_failures": ["message"],
"date": "2000-01-17T11:23:54+00:00",
"committer_date": "2000-01-17T11:23:54+00:00",
"children": ["123546353ed3480476f032475e7c244eff7371d5"],
"parents": [
"29d8be353ed3480476f032475e7c244eff7371d5",
"30d8be353ed3480476f032475e7c244eff7371d5",
],
"type": "tar",
"synthetic": True,
"metadata": {
"original_artifact": [
{
"archive_type": "tar",
"name": "webbase-5.7.0.tar.gz",
"sha1": "147f73f369733d088b7a6fa9c4e0273dcd3c7ccd",
"sha1_git": "6a15ea8b881069adedf11feceec35588f2cfe8f1",
"sha256": "401d0df797110bea805d358b85bcc1ced29549d3d73f"
"309d36484e7edf7bb912",
}
]
},
"merge": True,
}
actual_revision = converters.from_revision(revision_input)
assert actual_revision == expected_revision
def test_from_content_none():
assert converters.from_content(None) is None
def test_from_content():
content_input = {
"sha1": hashutil.hash_to_bytes("5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5"),
"sha256": hashutil.hash_to_bytes(
"39007420ca5de7cb3cfc15196335507e" "e76c98930e7e0afa4d2747d3bf96c926"
),
"blake2s256": hashutil.hash_to_bytes(
"49007420ca5de7cb3cfc15196335507e" "e76c98930e7e0afa4d2747d3bf96c926"
),
"sha1_git": hashutil.hash_to_bytes("40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03"),
"ctime": "something-which-is-filtered-out",
"data": b"data in bytes",
"length": 10,
"status": "hidden",
}
# 'status' is filtered
expected_content = {
"checksums": {
"sha1": "5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5",
"sha256": "39007420ca5de7cb3cfc15196335507ee76c98"
"930e7e0afa4d2747d3bf96c926",
"blake2s256": "49007420ca5de7cb3cfc15196335507ee7"
"6c98930e7e0afa4d2747d3bf96c926",
"sha1_git": "40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03",
},
"data": b"data in bytes",
"length": 10,
"status": "absent",
}
actual_content = converters.from_content(content_input)
assert actual_content == expected_content
def test_from_person():
person_input = {
"id": 10,
"anything": "else",
"name": b"bob",
"fullname": b"bob bob@alice.net",
"email": b"bob@foo.alice",
}
expected_person = {
"id": 10,
"anything": "else",
"name": "bob",
"fullname": "bob bob@alice.net",
"email": "bob@foo.alice",
}
actual_person = converters.from_person(person_input)
assert actual_person == expected_person
def test_from_directory_entries():
dir_entries_input = {
"sha1": hashutil.hash_to_bytes("5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5"),
"sha256": hashutil.hash_to_bytes(
"39007420ca5de7cb3cfc15196335507e" "e76c98930e7e0afa4d2747d3bf96c926"
),
"sha1_git": hashutil.hash_to_bytes("40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03"),
"blake2s256": hashutil.hash_to_bytes(
"685395c5dc57cada459364f0946d3dd45bad5fcbab" "c1048edb44380f1d31d0aa"
),
"target": hashutil.hash_to_bytes("40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03"),
"dir_id": hashutil.hash_to_bytes("40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03"),
"name": b"bob",
"type": 10,
"status": "hidden",
}
expected_dir_entries = {
"checksums": {
"sha1": "5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5",
"sha256": "39007420ca5de7cb3cfc15196335507ee76c98"
"930e7e0afa4d2747d3bf96c926",
"sha1_git": "40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03",
"blake2s256": "685395c5dc57cada459364f0946d3dd45bad5f"
"cbabc1048edb44380f1d31d0aa",
},
"target": "40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03",
"dir_id": "40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03",
"name": "bob",
"type": 10,
"status": "absent",
}
actual_dir_entries = converters.from_directory_entry(dir_entries_input)
assert actual_dir_entries == expected_dir_entries
def test_from_filetype():
content_filetype = {
"id": hashutil.hash_to_bytes("5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5"),
"encoding": "utf-8",
"mimetype": "text/plain",
}
expected_content_filetype = {
"id": "5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5",
"encoding": "utf-8",
"mimetype": "text/plain",
}
actual_content_filetype = converters.from_filetype(content_filetype)
assert actual_content_filetype == expected_content_filetype

File Metadata

Mime Type
text/x-diff
Expires
Fri, Jul 4, 11:22 AM (3 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3250777

Event Timeline

About · Developer information · Legal