diff --git a/swh/web/api/apiresponse.py b/swh/web/api/apiresponse.py
index 44f09d593..e1c212ae5 100644
--- a/swh/web/api/apiresponse.py
+++ b/swh/web/api/apiresponse.py
@@ -1,180 +1,180 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import json
from rest_framework.response import Response
from swh.storage.exc import StorageDBError, StorageAPIError
from swh.web.common.exc import NotFoundExc, ForbiddenExc
from swh.web.common.utils import shorten_path, gen_path_info
from swh.web.api import utils
def compute_link_header(rv, options):
"""Add Link header in returned value results.
Args:
rv (dict): dictionary with keys:
- headers: potential headers with 'link-next' and 'link-prev'
keys
- results: containing the result to return
options (dict): the initial dict to update with result if any
Returns:
dict: dictionary with optional keys 'link-next' and 'link-prev'
"""
link_headers = []
if 'headers' not in rv:
return {}
rv_headers = rv['headers']
if 'link-next' in rv_headers:
link_headers.append('<%s>; rel="next"' % (
rv_headers['link-next']))
if 'link-prev' in rv_headers:
link_headers.append('<%s>; rel="previous"' % (
rv_headers['link-prev']))
if link_headers:
link_header_str = ','.join(link_headers)
headers = options.get('headers', {})
headers.update({
'Link': link_header_str
})
return headers
return {}
def filter_by_fields(request, data):
"""Extract a request parameter 'fields' if it exists to permit the filtering on
he data dict's keys.
If such field is not provided, returns the data as is.
"""
fields = request.query_params.get('fields')
if fields:
fields = set(fields.split(','))
data = utils.filter_field_keys(data, fields)
return data
def transform(rv):
"""Transform an eventual returned value with multiple layer of
information with only what's necessary.
If the returned value rv contains the 'results' key, this is the
associated value which is returned.
Otherwise, return the initial dict without the potential 'headers'
key.
"""
if 'results' in rv:
return rv['results']
if 'headers' in rv:
rv.pop('headers')
return rv
def make_api_response(request, data, doc_data={}, options={}):
"""Generates an API response based on the requested mimetype.
Args:
request: a DRF Request object
data: raw data to return in the API response
doc_data: documentation data for HTML response
options: optionnal data that can be used to generate the response
Returns:
a DRF Response a object
"""
if data:
options['headers'] = compute_link_header(data, options)
data = transform(data)
data = filter_by_fields(request, data)
doc_env = doc_data
headers = {}
if 'headers' in options:
doc_env['headers_data'] = options['headers']
headers = options['headers']
# get request status code
doc_env['status_code'] = options.get('status', 200)
response_args = {'status': doc_env['status_code'],
'headers': headers,
'content_type': request.accepted_media_type}
# when requesting HTML, typically when browsing the API through its
# documented views, we need to enrich the input data with documentation
# related ones and inform DRF that we request HTML template rendering
if request.accepted_media_type == 'text/html':
if data:
data = json.dumps(data, sort_keys=True,
indent=4,
separators=(',', ': '))
doc_env['response_data'] = data
doc_env['request'] = {
'path': request.path,
'method': request.method,
'absolute_uri': request.build_absolute_uri()
}
doc_env['heading'] = shorten_path(str(request.path))
if 'route' in doc_env:
doc_env['endpoint_path'] = gen_path_info(doc_env['route'])
response_args['data'] = doc_env
response_args['template_name'] = 'apidoc.html'
# otherwise simply return the raw data and let DRF picks
# the correct renderer (JSON or YAML)
else:
response_args['data'] = data
return Response(**response_args)
def error_response(request, error, doc_data):
"""Private function to create a custom error response.
Args:
request: a DRF Request object
error: the exception that caused the error
doc_data: documentation data for HTML response
"""
error_code = 400
if isinstance(error, NotFoundExc):
error_code = 404
elif isinstance(error, ForbiddenExc):
error_code = 403
elif isinstance(error, StorageDBError):
error_code = 503
elif isinstance(error, StorageAPIError):
error_code = 503
error_opts = {'status': error_code}
error_data = {
'exception': error.__class__.__name__,
'reason': str(error),
}
return make_api_response(request, error_data, doc_data,
options=error_opts)
diff --git a/swh/web/api/apiurls.py b/swh/web/api/apiurls.py
index 4e3b8fe75..459586904 100644
--- a/swh/web/api/apiurls.py
+++ b/swh/web/api/apiurls.py
@@ -1,126 +1,126 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import re
from rest_framework.decorators import api_view
from swh.web.common.urlsindex import UrlsIndex
from swh.web.common.throttling import throttle_scope
class APIUrls(UrlsIndex):
"""
Class to manage API documentation URLs.
- Indexes all routes documented using apidoc's decorators.
- Tracks endpoint/request processing method relationships for use in
generating related urls in API documentation
"""
_apidoc_routes = {}
_method_endpoints = {}
scope = 'api'
@classmethod
def get_app_endpoints(cls):
return cls._apidoc_routes
@classmethod
def get_method_endpoints(cls, f):
if f.__name__ not in cls._method_endpoints:
cls._method_endpoints[f.__name__] = cls.group_routes_by_method(f)
return cls._method_endpoints[f.__name__]
@classmethod
def group_routes_by_method(cls, f):
"""
Group URL endpoints according to their processing method.
Returns:
A dict where keys are the processing method names, and values are
the routes that are bound to the key method.
"""
rules = []
for urlp in cls.get_url_patterns():
endpoint = urlp.callback.__name__
if endpoint != f.__name__:
continue
method_names = urlp.callback.http_method_names
url_rule = urlp.regex.pattern.replace('^', '/').replace('$', '')
url_rule_params = re.findall('\([^)]+\)', url_rule)
for param in url_rule_params:
param_name = re.findall('<(.*)>', param)
param_name = param_name[0] if len(param_name) > 0 else None
if param_name and hasattr(f, 'doc_data') and f.doc_data['args']: # noqa
param_index = \
next(i for (i, d) in enumerate(f.doc_data['args'])
if d['name'] == param_name)
if param_index is not None:
url_rule = url_rule.replace(
param, '<' +
f.doc_data['args'][param_index]['name'] +
': ' + f.doc_data['args'][param_index]['type'] +
'>').replace('.*', '')
rule_dict = {'rule': '/api' + url_rule,
'name': urlp.name,
'methods': {method.upper() for method in method_names}
}
rules.append(rule_dict)
return rules
@classmethod
def add_route(cls, route, docstring, **kwargs):
"""
Add a route to the self-documenting API reference
"""
route_view_name = route[1:-1].replace('/', '-')
if route not in cls._apidoc_routes:
d = {'docstring': docstring,
'route_view_name': route_view_name}
for k, v in kwargs.items():
d[k] = v
cls._apidoc_routes[route] = d
class api_route(object): # noqa: N801
"""
Decorator to ease the registration of an API endpoint
using the Django REST Framework.
Args:
url_pattern: the url pattern used by DRF to identify the API route
view_name: the name of the API view associated to the route used to
reverse the url
methods: array of HTTP methods supported by the API route
"""
def __init__(self, url_pattern=None, view_name=None,
methods=['GET', 'HEAD', 'OPTIONS'],
throttle_scope='swh_api',
api_version='1'):
super().__init__()
self.url_pattern = '^' + api_version + url_pattern + '$'
self.view_name = view_name
self.methods = methods
self.throttle_scope = throttle_scope
def __call__(self, f):
# create a DRF view from the wrapped function
@api_view(self.methods)
@throttle_scope(self.throttle_scope)
def api_view_f(*args, **kwargs):
return f(*args, **kwargs)
# small hacks for correctly generating API endpoints index doc
api_view_f.__name__ = f.__name__
api_view_f.http_method_names = self.methods
# register the route and its view in the endpoints index
APIUrls.add_url_pattern(self.url_pattern, api_view_f,
self.view_name)
return f
diff --git a/swh/web/api/urls.py b/swh/web/api/urls.py
index 03071b5ac..de2e0101c 100644
--- a/swh/web/api/urls.py
+++ b/swh/web/api/urls.py
@@ -1,19 +1,19 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import swh.web.api.views.content # noqa
import swh.web.api.views.directory # noqa
import swh.web.api.views.entity # noqa
import swh.web.api.views.origin # noqa
import swh.web.api.views.person # noqa
import swh.web.api.views.release # noqa
import swh.web.api.views.revision # noqa
import swh.web.api.views.snapshot # noqa
import swh.web.api.views.stat # noqa
import swh.web.api.views.vault # noqa
from swh.web.api.apiurls import APIUrls
urlpatterns = APIUrls.get_url_patterns()
diff --git a/swh/web/assets/config/bootstrap-pre-customize.scss b/swh/web/assets/config/bootstrap-pre-customize.scss
index 8acd2c99b..94607475a 100644
--- a/swh/web/assets/config/bootstrap-pre-customize.scss
+++ b/swh/web/assets/config/bootstrap-pre-customize.scss
@@ -1,23 +1,30 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// override some global boostrap sass variables before generating stylesheets
// global text colors and fonts
$body-color: rgba(0, 0, 0, 0.55);
$font-family-sans-serif: "Alegreya Sans", sans-serif;
$link-color: rgba(0, 0, 0, 0.75);
$code-color: #c7254e;
// headings
$headings-line-height: 1.1;
$headings-color: #e20026;
$headings-font-family: "Alegreya Sans", sans-serif;
// remove the ugly box shadow from bootstrap 4.x
$input-btn-focus-width: 0;
// dropdown menu padding
$dropdown-padding-y: 0.25rem;
$dropdown-item-padding-x: 0;
$dropdown-item-padding-y: 0;
// card header padding
$card-spacer-y: 0.5rem;
diff --git a/swh/web/assets/config/webpack-plugins/remove-source-map-url-webpack-plugin.js b/swh/web/assets/config/webpack-plugins/remove-source-map-url-webpack-plugin.js
index 145046858..1ee77566f 100644
--- a/swh/web/assets/config/webpack-plugins/remove-source-map-url-webpack-plugin.js
+++ b/swh/web/assets/config/webpack-plugins/remove-source-map-url-webpack-plugin.js
@@ -1,49 +1,56 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// Adapted and ported to Webpack 4 from
// https://github.com/rbarilani/remove-source-map-url-webpack-plugin
// Enable to remove source map url annotation embedded in vendor modules.
// The purpose is to remove warnings in js console after bundling them with webpack.
'use strict';
class RemoveSourceMapURLWebpackPlugin {
constructor(opts) {
this.options = opts || {};
this.options.test = this.options.test || /\.js($|\?)/i;
}
testKey(key) {
if (this.options.test instanceof RegExp) {
return this.options.test.test(key);
}
if (typeof this.options.test === 'string') {
return this.options.test === key;
}
if (typeof this.options.test === 'function') {
return this.options.test(key);
}
throw new Error(`remove-source-map-url: Invalid "test" option. May be a RegExp (tested against asset key), a string containing the key, a function(key): bool`);
}
apply(compiler) {
compiler.hooks.afterCompile.tap('RemoveSourceMapUrlPlugin', compilation => {
Object.keys(compilation.assets).filter(key => {
return this.testKey(key);
}).forEach(key => {
let asset = compilation.assets[key];
let source = asset.source().replace(/# sourceMappingURL=(.*\.map)/g, '# $1');
compilation.assets[key] = Object.assign(asset, {
source: () => {
return source;
}
});
});
});
}
};
module.exports = RemoveSourceMapURLWebpackPlugin;
diff --git a/swh/web/assets/config/webpack.config.development.js b/swh/web/assets/config/webpack.config.development.js
index d7d833100..93e366f6e 100644
--- a/swh/web/assets/config/webpack.config.development.js
+++ b/swh/web/assets/config/webpack.config.development.js
@@ -1,331 +1,338 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// webpack configuration for compiling static assets in development mode
// import required node modules and webpack plugins
const fs = require('fs');
const path = require('path');
const webpack = require('webpack');
const BundleTracker = require('webpack-bundle-tracker');
const RobotstxtPlugin = require('robotstxt-webpack-plugin').default;
const CleanWebpackPlugin = require('clean-webpack-plugin');
const MiniCssExtractPlugin = require('mini-css-extract-plugin');
const RemoveSourceMapUrlPlugin = require('./webpack-plugins/remove-source-map-url-webpack-plugin');
// are we running webpack-dev-server ?
const isDevServer = process.argv.find(v => v.includes('webpack-dev-server'));
// webpack-dev-server configuration
const devServerPort = 3000;
const devServerPublicPath = 'http://localhost:' + devServerPort + '/static/';
// set publicPath according if we are using webpack-dev-server to serve
// our assets or not
const publicPath = isDevServer ? devServerPublicPath : '/static/';
// collect all bundles we want to produce with webpack
var bundles = {};
const bundlesDir = path.join(__dirname, '../src/bundles');
fs.readdirSync(bundlesDir).forEach(file => {
bundles[file] = ['bundles/' + file + '/index.js'];
});
// common loaders for css related assets (css, sass, less)
let cssLoaders = [
MiniCssExtractPlugin.loader,
{
loader: 'cache-loader'
},
{
loader: 'css-loader',
options: {
sourceMap: !isDevServer
}
},
{
loader: 'postcss-loader',
options: {
ident: 'postcss',
sourceMap: !isDevServer,
plugins: [
// lint swh-web stylesheets
require('stylelint')({
'config': {
'extends': 'stylelint-config-standard',
'rules': {
'indentation': 4,
'font-family-no-missing-generic-family-keyword': null,
'no-descending-specificity': null
},
'ignoreFiles': 'node_modules/**/*.css'
}
}),
// automatically add vendor prefixes to css rules
require('autoprefixer')()
]
}
}
];
// webpack development configuration
module.exports = {
// use caching to speedup incremental builds
cache: true,
// set mode to development
mode: 'development',
// use eval source maps when using webpack-dev-server for quick debugging,
// otherwise generate source map files (more expensive)
devtool: isDevServer ? 'eval' : 'source-map',
// webpack-dev-server configuration
devServer: {
clientLogLevel: 'warning',
port: devServerPort,
publicPath: devServerPublicPath,
// enable to serve static assets not managed by webpack
contentBase: path.resolve('./swh/web/'),
// we do not use hot reloading here (as a framework like React needs to be used in order to fully benefit from that feature)
// and prefere to fully reload the frontend application in the browser instead
hot: false,
inline: true,
historyApiFallback: true,
headers: {
'Access-Control-Allow-Origin': '*'
},
compress: true,
stats: {
colors: true
}
},
// set entries to the bundles we want to produce
entry: bundles,
// assets output configuration
output: {
path: path.resolve('./swh/web/static/'),
filename: 'js/[name].js',
chunkFilename: 'js/[name].js',
publicPath: publicPath,
// each bundle will be compiled as a umd module with its own namespace
// in order to easily use them in django templates
library: ['swh', '[name]'],
libraryTarget: 'umd'
},
// module resolving configuration
resolve: {
// alias highlightjs-line-numbers to its minified version
// as there is some issues when using the unminified one (even without webpack processing)
alias: {
'highlightjs-line-numbers.js': 'highlightjs-line-numbers.js/dist/highlightjs-line-numbers.min.js'
},
// configure base paths for resolving modules with webpack
modules: [
'node_modules',
path.resolve(__dirname, '../src')
]
},
// module import configuration
module: {
rules: [{
// Preprocess all js files with eslint for consistent code style
// and avoid bad js development practices.
enforce: 'pre',
test: /\.js$/,
exclude: /node_modules/,
use: [{
loader: 'eslint-loader',
options: {
configFile: path.join(__dirname, '.eslintrc'),
emitWarning: true
}
}]
},
{
// Use babel-loader in order to use es6 syntax in js files
// but also advanced js features like async/await syntax.
// All code get transpiled to es5 in order to be executed
// in a large majority of browsers.
test: /\.js$/,
exclude: /node_modules/,
use: [
{
loader: 'cache-loader'
},
{
loader: 'babel-loader',
options: {
presets: [
// use env babel presets to benefit from es6 syntax
['env', {
// Do not transform es6 module syntax to another module type
// in order to benefit from dead code elimination (aka tree shaking)
// when running webpack in production mode
'loose': true,
'modules': false
}],
// use stage-0 babel presets to benfit from advanced js features (es2017)
'stage-0'
],
plugins: [
// use babel transform-runtime plugin in order to use aync/await syntax
['transform-runtime', {
'polyfill': false,
'regenerator': true
}]
]
}
}]
},
// expose jquery to the global context as $ and jQuery when importing it
{
test: require.resolve('jquery'),
use: [{
loader: 'expose-loader',
options: 'jQuery'
}, {
loader: 'expose-loader',
options: '$'
}]
},
// expose highlightjs to the global context as hljs when importing it
{
test: require.resolve('highlightjs'),
use: [{
loader: 'expose-loader',
options: 'hljs'
}]
},
// css import configuration:
// - first process it with postcss
// - then extract it to a dedicated file associated to each bundle
{
test: /\.css$/,
use: cssLoaders
},
// sass import configuration:
// - generate css with sass-loader
// - process it with postcss
// - then extract it to a dedicated file associated to each bundle
{
test: /\.scss$/,
use: cssLoaders.concat([
{
loader: 'sass-loader',
options: {
sourceMap: !isDevServer
}
}
])
},
// less import configuration:
// - generate css with less-loader
// - process it with postcss
// - then extract it to a dedicated file associated to each bundle
{
test: /\.less$/,
use: cssLoaders.concat([
{
loader: 'less-loader',
options: {
sourceMap: !isDevServer
}
}
])
},
// web fonts import configuration
{
test: /\.(woff|woff2)(\?v=\d+\.\d+\.\d+)?$/,
use: [{
loader: 'file-loader',
options: {
name: '[name].[ext]',
outputPath: 'fonts/'
}
}]
}, {
test: /\.ttf(\?v=\d+\.\d+\.\d+)?$/,
use: [{
loader: 'file-loader',
options: {
name: '[name].[ext]',
outputPath: 'fonts/'
}
}]
}, {
test: /\.eot(\?v=\d+\.\d+\.\d+)?$/,
use: [{
loader: 'file-loader',
options: {
name: '[name].[ext]',
outputPath: 'fonts/'
}
}]
}, {
test: /\.svg(\?v=\d+\.\d+\.\d+)?$/,
use: [{
loader: 'file-loader',
options: {
name: '[name].[ext]',
outputPath: 'fonts/'
}
}]
}
]
},
// webpack plugins
plugins: [
// cleanup previously generated assets
new CleanWebpackPlugin(['static/css', 'static/js', 'static/fonts', 'static/*.*'], {
root: path.resolve(__dirname, '../../')
}),
// needed in order to use django_webpack_loader
new BundleTracker({
filename: './swh/web/static/webpack-stats.json'
}),
// for generating the robots.txt file
new RobotstxtPlugin({
policy: [{
userAgent: '*',
disallow: '/api/'
}]
}),
// for extracting all stylesheets in separate css files
new MiniCssExtractPlugin({
filename: 'css/[name].css',
chunkFilename: 'css/[name].css'
}),
// for removing some warnings in js console about not found source maps
new RemoveSourceMapUrlPlugin(),
// define some global variables accessible from js code
new webpack.DefinePlugin({
__STATIC__: JSON.stringify(publicPath)
}),
// needed in order to use bootstrap 4.x
new webpack.ProvidePlugin({
Popper: ['popper.js', 'default'],
Alert: 'exports-loader?Alert!bootstrap/js/dist/alert',
Button: 'exports-loader?Button!bootstrap/js/dist/button',
Carousel: 'exports-loader?Carousel!bootstrap/js/dist/carousel',
Collapse: 'exports-loader?Collapse!bootstrap/js/dist/collapse',
Dropdown: 'exports-loader?Dropdown!bootstrap/js/dist/dropdown',
Modal: 'exports-loader?Modal!bootstrap/js/dist/modal',
Popover: 'exports-loader?Popover!bootstrap/js/dist/popover',
Scrollspy: 'exports-loader?Scrollspy!bootstrap/js/dist/scrollspy',
Tab: 'exports-loader?Tab!bootstrap/js/dist/tab',
Tooltip: 'exports-loader?Tooltip!bootstrap/js/dist/tooltip',
Util: 'exports-loader?Util!bootstrap/js/dist/util'
})
],
// webpack optimizations
optimization: {
// ensure the vendors bundle gets emitted in a single chunk
splitChunks: {
cacheGroups: {
vendors: {
test: 'vendors',
chunks: 'all',
name: 'vendors',
enforce: true
}
}
}
},
// disable webpack warnings about bundle sizes
performance: {
hints: false
}
};
diff --git a/swh/web/assets/config/webpack.config.production.js b/swh/web/assets/config/webpack.config.production.js
index 3129ecf05..32664ee17 100644
--- a/swh/web/assets/config/webpack.config.production.js
+++ b/swh/web/assets/config/webpack.config.production.js
@@ -1,35 +1,42 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// webpack configuration for compiling static assets in production mode
// import required webpack plugins
const UglifyJsPlugin = require('uglifyjs-webpack-plugin');
const OptimizeCSSAssetsPlugin = require('optimize-css-assets-webpack-plugin');
// import webpack development configuration
var webpackProdConfig = require('./webpack.config.development');
// override mode to production
webpackProdConfig.mode = 'production';
// configure minimizer for js and css assets
webpackProdConfig.optimization.minimizer = [
// use ugligyjs for minimizing js and generate source map files
new UglifyJsPlugin({
cache: true,
parallel: true,
sourceMap: true
}),
// use cssnano for minimizing css and generate source map files
new OptimizeCSSAssetsPlugin({
cssProcessorOptions: {
map: {
inline: false
},
minifyFontValues: false,
discardUnused: false,
zindex: false
}
})
];
// webpack production configuration
module.exports = webpackProdConfig;
diff --git a/swh/web/assets/src/bundles/browse/breadcrumbs.css b/swh/web/assets/src/bundles/browse/breadcrumbs.css
index ecfc477ad..6f1e635c4 100644
--- a/swh/web/assets/src/bundles/browse/breadcrumbs.css
+++ b/swh/web/assets/src/bundles/browse/breadcrumbs.css
@@ -1,11 +1,18 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.swh-browse-bread-crumbs {
font-size: inherit;
vertical-align: text-top;
margin-bottom: 1px;
}
.swh-browse-bread-crumbs li:nth-child(n+2)::before {
content: "";
display: inline-block;
margin: 0 2px;
}
diff --git a/swh/web/assets/src/bundles/browse/browse.css b/swh/web/assets/src/bundles/browse/browse.css
index ef006311a..bac22dbf0 100644
--- a/swh/web/assets/src/bundles/browse/browse.css
+++ b/swh/web/assets/src/bundles/browse/browse.css
@@ -1,102 +1,109 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.navbar-header a,
ul.dropdown-menu a,
ul.navbar-nav a {
border-bottom-style: none;
color: #323232;
font-weight: 700;
}
.swh-browse-nav li a {
border-radius: 4px;
}
.nav-link.active {
background-color: #e7e7e7;
}
.scrollable-menu {
max-height: 180px;
overflow-x: hidden;
}
.swh-corner-ribbon {
width: 200px;
background: #e43;
position: absolute;
text-align: center;
line-height: 50px;
letter-spacing: 1px;
color: #f0f0f0;
box-shadow: 0 0 3px rgba(0, 0, 0, 0.3);
top: 25px;
right: -50px;
left: auto;
transform: rotate(45deg);
z-index: 2000;
}
.swh-loading {
display: none;
}
.swh-loading.show {
display: inline-block;
position: fixed;
background: white;
border: 1px solid black;
top: 50%;
left: 50%;
margin: -50px 0 0 -50px;
text-align: center;
z-index: 100;
}
.swh-metadata-table-row {
border-top: 1px solid #ddd !important;
}
.swh-metadata-table-key {
min-width: 200px;
max-width: 200px;
width: 200px;
}
.swh-metadata-table-value pre {
white-space: pre-wrap;
}
.swh-table-even-odd th {
border-top: none;
}
.swh-table-even-odd tr:nth-child(even) {
background-color: #f5f5f5;
}
.swh-table-even-odd tr:nth-child(odd) {
background-color: #fff;
}
.swh-table-cell-text-overflow {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.swh-directory-table {
margin-bottom: 0;
}
.swh-directory-table td {
border-top: 1px solid #ddd !important;
}
.swh-title-color {
color: #e20026;
}
.swh-log-entry-message {
min-width: 460px;
max-width: 460px;
width: 460px;
}
diff --git a/swh/web/assets/src/bundles/browse/content.css b/swh/web/assets/src/bundles/browse/content.css
index 7eb854d4c..46ee4e0ea 100644
--- a/swh/web/assets/src/bundles/browse/content.css
+++ b/swh/web/assets/src/bundles/browse/content.css
@@ -1,12 +1,19 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.swh-content {
background-image: none;
border: none;
background-color: white;
padding: 0;
}
.swh-content pre,
.swh-content pre code {
margin: 0;
padding: 0;
}
diff --git a/swh/web/assets/src/bundles/browse/index.js b/swh/web/assets/src/bundles/browse/index.js
index 65d382347..e03d15f7c 100644
--- a/swh/web/assets/src/bundles/browse/index.js
+++ b/swh/web/assets/src/bundles/browse/index.js
@@ -1,10 +1,17 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// main bundle for the swh-web/browse application
import './browse.css';
import './breadcrumbs.css';
import './content.css';
import './snapshot-navigation.css';
export * from './snapshot-navigation';
export * from './origin-search';
export * from './main-navigation';
diff --git a/swh/web/assets/src/bundles/browse/main-navigation.js b/swh/web/assets/src/bundles/browse/main-navigation.js
index a0961993f..14420c5f3 100644
--- a/swh/web/assets/src/bundles/browse/main-navigation.js
+++ b/swh/web/assets/src/bundles/browse/main-navigation.js
@@ -1,61 +1,68 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
export let browseTabsHash = ['#browse', '#search', '#help', '#vault'];
export function removeHash() {
history.replaceState('', document.title, window.location.pathname + window.location.search);
}
export function showTab(hash) {
$('.navbar-nav.swh-browse-nav a[href="' + hash + '"]').tab('show');
window.scrollTo(0, 0);
}
export function showRequestedTab() {
let hash = window.location.hash;
if (hash && browseTabsHash.indexOf(hash) === -1) {
return;
}
if (hash) {
showTab(hash);
} else {
showTab('#browse');
}
}
export function initMainNavigation() {
$(document).ready(() => {
$('.dropdown-submenu a.dropdown-item').on('click', e => {
$(e.target).next('div').toggle();
if ($(e.target).next('div').css('display') !== 'none') {
$(e.target).focus();
} else {
$(e.target).blur();
}
e.stopPropagation();
e.preventDefault();
});
// Change hash for page reload
$('.navbar-nav.swh-browse-nav a').on('shown.bs.tab', e => {
if (e.target.hash.trim() !== '#browse') {
window.location.hash = e.target.hash;
} else {
let hash = window.location.hash;
if (browseTabsHash.indexOf(hash) !== -1) {
removeHash();
}
}
showRequestedTab();
});
// update displayed tab when the url fragment changes
$(window).on('hashchange', () => {
showRequestedTab();
});
// show requested tab when loading the page
showRequestedTab();
});
}
diff --git a/swh/web/assets/src/bundles/browse/origin-search.js b/swh/web/assets/src/bundles/browse/origin-search.js
index bfd8cd01a..b2f390510 100644
--- a/swh/web/assets/src/bundles/browse/origin-search.js
+++ b/swh/web/assets/src/bundles/browse/origin-search.js
@@ -1,152 +1,159 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
import {heapsPermute} from 'utils/heaps-permute';
import {handleFetchError} from 'utils/functions';
let originPatterns;
let perPage = 15;
let limit = perPage * 10;
let offset = 0;
let currentData = null;
let inSearch = false;
function fixTableRowsStyle() {
setTimeout(() => {
$('#origin-search-results tbody tr').removeAttr('style');
});
}
function populateOriginSearchResultsTable(data, offset) {
let localOffset = offset % limit;
if (data.length > 0) {
$('#swh-origin-search-results').show();
$('#swh-no-origins-found').hide();
$('#origin-search-results tbody tr').remove();
let table = $('#origin-search-results tbody');
for (let i = localOffset; i < localOffset + perPage && i < data.length; ++i) {
let elem = data[i];
let tableRow = '
';
tableRow += '
' + elem.type + '
';
let browseUrl = Urls.browse_origin(elem.type, elem.url);
tableRow += '
';
$(e.element).popover({
trigger: 'manual',
container: 'body',
html: true,
content: content
});
$(e.element).popover('show');
currentPopover = e.element;
}
}
});
$('#swh-visits-timeline').mouseenter(() => {
closePopover();
});
$('#swh-visits-list').mouseenter(() => {
closePopover();
});
$('#swh-visits-calendar.calendar table td').css('width', maxSize + 'px');
$('#swh-visits-calendar.calendar table td').css('height', maxSize + 'px');
$('#swh-visits-calendar.calendar table td').css('padding', '0px');
}
diff --git a/swh/web/assets/src/bundles/origin/visits-histogram.js b/swh/web/assets/src/bundles/origin/visits-histogram.js
index 8b9e3d2b0..03f75cb9a 100644
--- a/swh/web/assets/src/bundles/origin/visits-histogram.js
+++ b/swh/web/assets/src/bundles/origin/visits-histogram.js
@@ -1,330 +1,337 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// Creation of a stacked histogram with D3.js for SWH origin visits history
// Parameters description:
// - container: selector for the div that will contain the histogram
// - visitsData: raw swh origin visits data
// - currentYear: the visits year to display by default
// - yearClickCallback: callback when the user selects a year through the histogram
import * as d3 from 'd3';
export function createVisitsHistogram(container, visitsData, currentYear, yearClickCallback) {
// remove previously created hisogram and tooltip if any
d3.select(container).select('svg').remove();
d3.select('div.d3-tooltip').remove();
// histogram size and margins
let width = 1000;
let height = 300;
let margin = {top: 20, right: 80, bottom: 30, left: 50};
// create responsive svg
let svg = d3.select(container)
.attr('style',
'padding-bottom: ' + Math.ceil(height * 100 / width) + '%')
.append('svg')
.attr('viewBox', '0 0 ' + width + ' ' + height);
// create tooltip div
let tooltip = d3.select('body')
.append('div')
.attr('class', 'd3-tooltip')
.style('opacity', 0);
// update width and height without margins
width = width - margin.left - margin.right;
height = height - margin.top - margin.bottom;
// create main svg group element
let g = svg.append('g').attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
// create x scale
let x = d3.scaleTime().rangeRound([0, width]);
// create y scale
let y = d3.scaleLinear().range([height, 0]);
// create oridinal colorscale mapping visit status
let colors = d3.scaleOrdinal()
.domain(['full', 'partial', 'failed', 'ongoing'])
.range(['#008000', '#edc344', '#ff0000', '#0000ff']);
// first SWH crawls were made in 2015
let startYear = 2015;
// set latest display year as the current one
let now = new Date();
let endYear = now.getUTCFullYear() + 1;
let monthExtent = [new Date(Date.UTC(startYear, 0, 1)), new Date(Date.UTC(endYear, 0, 1))];
// create months bins based on setup extent
let monthBins = d3.timeMonths(d3.timeMonth.offset(monthExtent[0], -1), monthExtent[1]);
// create years bins based on setup extent
let yearBins = d3.timeYears(monthExtent[0], monthExtent[1]);
// set x scale domain
x.domain(d3.extent(monthBins));
// use D3 histogram layout to create a function that will bin the visits by month
let binByMonth = d3.histogram()
.value(d => d.date)
.domain(x.domain())
.thresholds(monthBins);
// use D3 nest function to group the visits by status
let visitsByStatus = d3.nest()
.key(d => d['status'])
.sortKeys(d3.ascending)
.entries(visitsData);
// prepare data in order to be able to stack visit statuses by month
let statuses = [];
let histData = [];
for (let i = 0; i < monthBins.length; ++i) {
histData[i] = {};
}
visitsByStatus.forEach(entry => {
statuses.push(entry.key);
let monthsData = binByMonth(entry.values);
for (let i = 0; i < monthsData.length; ++i) {
histData[i]['x0'] = monthsData[i]['x0'];
histData[i]['x1'] = monthsData[i]['x1'];
histData[i][entry.key] = monthsData[i];
}
});
// create function to stack visits statuses by month
let stacked = d3.stack()
.keys(statuses)
.value((d, key) => d[key].length);
// compute the maximum amount of visits by month
let yMax = d3.max(histData, d => {
let total = 0;
for (let i = 0; i < statuses.length; ++i) {
total += d[statuses[i]].length;
}
return total;
});
// set y scale domain
y.domain([0, yMax]);
// compute ticks values for the y axis
let step = 5;
let yTickValues = [];
for (let i = 0; i <= yMax / step; ++i) {
yTickValues.push(i * step);
}
if (yTickValues.length === 0) {
for (let i = 0; i <= yMax; ++i) {
yTickValues.push(i);
}
} else if (yMax % step !== 0) {
yTickValues.push(yMax);
}
// add histogram background grid
g.append('g')
.attr('class', 'grid')
.call(d3.axisLeft(y)
.tickValues(yTickValues)
.tickSize(-width)
.tickFormat(''));
// create one fill only rectangle by displayed year
// each rectangle will be made visible when hovering the mouse over a year range
// user will then be able to select a year by clicking in the rectangle
g.append('g')
.selectAll('rect')
.data(yearBins)
.enter().append('rect')
.attr('class', d => 'year' + d.getUTCFullYear())
.attr('fill', 'red')
.attr('fill-opacity', d => d.getUTCFullYear() === currentYear ? 0.3 : 0)
.attr('stroke', 'none')
.attr('x', d => {
let date = new Date(Date.UTC(d.getUTCFullYear(), 0, 1));
return x(date);
})
.attr('y', 0)
.attr('height', height)
.attr('width', d => {
let date = new Date(Date.UTC(d.getUTCFullYear(), 0, 1));
let yearWidth = x(d3.timeYear.offset(date, 1)) - x(date);
return yearWidth;
})
// mouse event callbacks used to show rectangle years
// when hovering the mouse over the histograms
.on('mouseover', d => {
svg.selectAll('rect.year' + d.getUTCFullYear())
.attr('fill-opacity', 0.5);
})
.on('mouseout', d => {
svg.selectAll('rect.year' + d.getUTCFullYear())
.attr('fill-opacity', 0);
svg.selectAll('rect.year' + currentYear)
.attr('fill-opacity', 0.3);
})
// callback to select a year after a mouse click
// in a rectangle year
.on('click', d => {
svg.selectAll('rect.year' + currentYear)
.attr('fill-opacity', 0);
svg.selectAll('rect.yearoutline' + currentYear)
.attr('stroke', 'none');
currentYear = d.getUTCFullYear();
svg.selectAll('rect.year' + currentYear)
.attr('fill-opacity', 0.5);
svg.selectAll('rect.yearoutline' + currentYear)
.attr('stroke', 'black');
yearClickCallback(currentYear);
});
// create the stacked histogram of visits
g.append('g')
.selectAll('g')
.data(stacked(histData))
.enter().append('g')
.attr('fill', d => colors(d.key))
.selectAll('rect')
.data(d => d)
.enter().append('rect')
.attr('class', d => 'month' + d.data.x1.getMonth())
.attr('x', d => x(d.data.x0))
.attr('y', d => y(d[1]))
.attr('height', d => y(d[0]) - y(d[1]))
.attr('width', d => x(d.data.x1) - x(d.data.x0) - 1)
// mouse event callbacks used to show rectangle years
// but also to show tooltips when hovering the mouse
// over the histogram bars
.on('mouseover', d => {
svg.selectAll('rect.year' + d.data.x1.getUTCFullYear())
.attr('fill-opacity', 0.5);
tooltip.transition()
.duration(200)
.style('opacity', 1);
let ds = d.data.x1.toISOString().substr(0, 7).split('-');
let tooltipText = '' + ds[1] + ' / ' + ds[0] + ': ';
for (let i = 0; i < statuses.length; ++i) {
let visitStatus = statuses[i];
let nbVisits = d.data[visitStatus].length;
if (nbVisits === 0) continue;
tooltipText += nbVisits + ' ' + visitStatus + ' visits';
if (i !== statuses.length - 1) tooltipText += ' ';
}
tooltip.html(tooltipText)
.style('left', d3.event.pageX + 15 + 'px')
.style('top', d3.event.pageY + 'px');
})
.on('mouseout', d => {
svg.selectAll('rect.year' + d.data.x1.getUTCFullYear())
.attr('fill-opacity', 0);
svg.selectAll('rect.year' + currentYear)
.attr('fill-opacity', 0.3);
tooltip.transition()
.duration(500)
.style('opacity', 0);
})
.on('mousemove', () => {
tooltip.style('left', d3.event.pageX + 15 + 'px')
.style('top', d3.event.pageY + 'px');
})
// callback to select a year after a mouse click
// inside a histogram bar
.on('click', d => {
svg.selectAll('rect.year' + currentYear)
.attr('fill-opacity', 0);
svg.selectAll('rect.yearoutline' + currentYear)
.attr('stroke', 'none');
currentYear = d.data.x1.getUTCFullYear();
svg.selectAll('rect.year' + currentYear)
.attr('fill-opacity', 0.5);
svg.selectAll('rect.yearoutline' + currentYear)
.attr('stroke', 'black');
yearClickCallback(currentYear);
});
// create one stroke only rectangle by displayed year
// that will be displayed on top of the histogram when the user has selected a year
g.append('g')
.selectAll('rect')
.data(yearBins)
.enter().append('rect')
.attr('class', d => 'yearoutline' + d.getUTCFullYear())
.attr('fill', 'none')
.attr('stroke', d => d.getUTCFullYear() === currentYear ? 'black' : 'none')
.attr('x', d => {
let date = new Date(Date.UTC(d.getUTCFullYear(), 0, 1));
return x(date);
})
.attr('y', 0)
.attr('height', height)
.attr('width', d => {
let date = new Date(Date.UTC(d.getUTCFullYear(), 0, 1));
let yearWidth = x(d3.timeYear.offset(date, 1)) - x(date);
return yearWidth;
});
// add x axis with a tick for every 1st day of each year
let xAxis = g.append('g')
.attr('class', 'axis')
.attr('transform', 'translate(0,' + height + ')')
.call(
d3.axisBottom(x)
.ticks(d3.timeYear.every(1))
.tickFormat(d => d.getUTCFullYear())
);
// shift tick labels in order to display them at the middle
// of each year range
xAxis.selectAll('text')
.attr('transform', d => {
let year = d.getUTCFullYear();
let date = new Date(Date.UTC(year, 0, 1));
let yearWidth = x(d3.timeYear.offset(date, 1)) - x(date);
return 'translate(' + -yearWidth / 2 + ', 0)';
});
// add y axis for the number of visits
g.append('g')
.attr('class', 'axis')
.call(d3.axisLeft(y).tickValues(yTickValues));
// add legend for visit statuses
let legendGroup = g.append('g')
.attr('font-family', 'sans-serif')
.attr('font-size', 10)
.attr('text-anchor', 'end');
legendGroup.append('text')
.attr('x', width + margin.right - 5)
.attr('y', 9.5)
.attr('dy', '0.32em')
.text('visit status:');
let legend = legendGroup.selectAll('g')
.data(statuses.slice().reverse())
.enter().append('g')
.attr('transform', (d, i) => 'translate(0,' + (i + 1) * 20 + ')');
legend.append('rect')
.attr('x', width + 2 * margin.right / 3)
.attr('width', 19)
.attr('height', 19)
.attr('fill', colors);
legend.append('text')
.attr('x', width + 2 * margin.right / 3 - 5)
.attr('y', 9.5)
.attr('dy', '0.32em')
.text(d => d);
// add text label for the y axis
g.append('text')
.attr('transform', 'rotate(-90)')
.attr('y', -margin.left)
.attr('x', -(height / 2))
.attr('dy', '1em')
.style('text-anchor', 'middle')
.text('Number of visits');
}
diff --git a/swh/web/assets/src/bundles/origin/visits-reporting.css b/swh/web/assets/src/bundles/origin/visits-reporting.css
index adba1cf2b..90fe49f39 100644
--- a/swh/web/assets/src/bundles/origin/visits-reporting.css
+++ b/swh/web/assets/src/bundles/origin/visits-reporting.css
@@ -1,113 +1,120 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.swh-visit-full {
color: green;
position: relative;
}
.swh-visit-full::before {
content: '\f00c';
font-family: 'FontAwesome';
left: -20px;
position: absolute;
top: -2px;
}
.swh-visit-partial {
color: #edc344;
position: relative;
}
.swh-visit-partial::before {
content: '\f071';
font-family: 'FontAwesome';
left: -20px;
position: absolute;
top: -2px;
}
.swh-visit-failed {
color: #f00;
position: relative;
}
.swh-visit-failed::before {
content: '\f06a';
font-family: 'FontAwesome';
left: -20px;
position: absolute;
top: -2px;
}
.swh-visit-ongoing {
color: #00f;
position: relative;
}
.swh-visit-ongoing::before {
content: '\f021';
font-family: 'FontAwesome';
left: -20px;
position: absolute;
top: -2px;
}
#swh-visits-calendar.calendar table td {
width: 28px;
height: 28px;
padding: 0;
}
.d3-wrapper {
position: relative;
height: 0;
width: 100%;
padding: 0;
/* padding-bottom will be overwritten by JavaScript later */
padding-bottom: 100%;
}
.d3-wrapper > svg {
position: absolute;
height: 100%;
width: 100%;
left: 0;
top: 0;
}
svg .grid line {
stroke: lightgrey;
stroke-opacity: 0.7;
shape-rendering: crispEdges;
}
svg .grid path {
stroke-width: 0;
}
div.d3-tooltip {
position: absolute;
text-align: center;
width: auto;
height: auto;
padding: 2px;
font: 12px sans-serif;
background: white;
border: 1px solid black;
border-radius: 4px;
pointer-events: none;
}
.swh-visits-list-column {
float: left;
padding: 10px;
}
.swh-visits-list-row {
padding-left: 50px;
}
.swh-visits-list-row::after {
content: "";
display: table;
clear: both;
}
diff --git a/swh/web/assets/src/bundles/origin/visits-reporting.js b/swh/web/assets/src/bundles/origin/visits-reporting.js
index f5bc4a37d..35885f40b 100644
--- a/swh/web/assets/src/bundles/origin/visits-reporting.js
+++ b/swh/web/assets/src/bundles/origin/visits-reporting.js
@@ -1,121 +1,128 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
import {createVisitsHistogram} from './visits-histogram';
import {updateCalendar} from './visits-calendar';
import './visits-reporting.css';
// will hold all visits
let allVisits;
// will hold filtered visits to display
let filteredVisits;
// will hold currently displayed year
let currentYear;
// function to gather full visits
function filterFullVisits(differentSnapshots) {
let filteredVisits = [];
for (let i = 0; i < allVisits.length; ++i) {
if (allVisits[i].status !== 'full') continue;
if (!differentSnapshots) {
filteredVisits.push(allVisits[i]);
} else if (filteredVisits.length === 0) {
filteredVisits.push(allVisits[i]);
} else {
let lastVisit = filteredVisits[filteredVisits.length - 1];
if (allVisits[i].snapshot !== lastVisit.snapshot) {
filteredVisits.push(allVisits[i]);
}
}
}
return filteredVisits;
}
// function to update the visits list view based on the selected year
function updateVisitsList(year) {
$('#swh-visits-list').children().remove();
let visitsByYear = [];
for (let i = 0; i < filteredVisits.length; ++i) {
if (filteredVisits[i].date.getUTCFullYear() === year) {
visitsByYear.push(filteredVisits[i]);
}
}
let visitsCpt = 0;
let nbVisitsByRow = 4;
let visitsListHtml = '
';
for (let i = 0; i < visitsByYear.length; ++i) {
if (visitsCpt > 0 && visitsCpt % nbVisitsByRow === 0) {
visitsListHtml += '
';
$('#swh-visits-list').append($(visitsListHtml));
}
// callback when the user selects a year through the visits histogram
function yearClicked(year) {
currentYear = year;
updateCalendar(year, filteredVisits);
updateVisitsList(year);
}
// function to update the visits views (histogram, calendar, list)
function updateDisplayedVisits() {
if (filteredVisits.length === 0) {
return;
}
if (!currentYear) {
currentYear = filteredVisits[filteredVisits.length - 1].date.getUTCFullYear();
}
createVisitsHistogram('.d3-wrapper', filteredVisits, currentYear, yearClicked);
updateCalendar(currentYear, filteredVisits);
updateVisitsList(currentYear);
}
// callback when the user only wants to see full visits pointing
// to different snapshots (default)
export function showFullVisitsDifferentSnapshots(event) {
filteredVisits = filterFullVisits(true);
updateDisplayedVisits();
}
// callback when the user only wants to see full visits
export function showFullVisits(event) {
filteredVisits = filterFullVisits(false);
updateDisplayedVisits();
}
// callback when the user wants to see all visits (including partial, ongoing and failed ones)
export function showAllVisits(event) {
filteredVisits = allVisits;
updateDisplayedVisits();
}
export function initVisitsReporting(visits) {
allVisits = visits;
// process input visits
let firstFullVisit;
allVisits.forEach((v, i) => {
// Turn Unix epoch into Javascript Date object
v.date = new Date(Math.floor(v.date * 1000));
let visitLink = '' + v.fmt_date + '';
if (v.status === 'full') {
if (!firstFullVisit) {
firstFullVisit = v;
$('#swh-first-full-visit').append($(visitLink));
} else {
$('#swh-last-full-visit')[0].innerHTML = visitLink;
}
}
if (i === allVisits.length - 1) {
$('#swh-last-visit').append($(visitLink));
}
});
// display full visits pointing to different snapshots by default
showFullVisitsDifferentSnapshots();
}
diff --git a/swh/web/assets/src/bundles/revision/diff-utils.js b/swh/web/assets/src/bundles/revision/diff-utils.js
index a191265a0..e5631c62b 100644
--- a/swh/web/assets/src/bundles/revision/diff-utils.js
+++ b/swh/web/assets/src/bundles/revision/diff-utils.js
@@ -1,508 +1,515 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
import 'waypoints/lib/jquery.waypoints';
import {staticAsset} from 'utils/functions';
// path to static spinner asset
let swhSpinnerSrc = staticAsset('img/swh-spinner.gif');
// number of changed files in the revision
let changes = null;
let nbChangedFiles = 0;
// to track the number of already computed files diffs
let nbDiffsComputed = 0;
// the no newline at end of file marker from Github
let noNewLineMarker = ``;
// to track the total number of added lines in files diffs
let nbAdditions = 0;
// to track the total number of deleted lines in files diffs
let nbDeletions = 0;
// to track the already computed diffs by id
let computedDiffs = {};
// map a diff id to its computation url
let diffsUrls = {};
// to check if a DOM element is in the viewport
function isInViewport(elt) {
let elementTop = $(elt).offset().top;
let elementBottom = elementTop + $(elt).outerHeight();
let viewportTop = $(window).scrollTop();
let viewportBottom = viewportTop + $(window).height();
return elementBottom > viewportTop && elementTop < viewportBottom;
}
// to format the diffs line numbers
function formatDiffLineNumbers(fromLine, toLine, maxNumberChars) {
let ret = '';
if (fromLine != null) {
for (let i = 0; i < (maxNumberChars - fromLine.length); ++i) {
ret += ' ';
}
ret += fromLine;
}
if (fromLine != null && toLine != null) {
ret += ' ';
}
if (toLine != null) {
for (let i = 0; i < (maxNumberChars - toLine.length); ++i) {
ret += ' ';
}
ret += toLine;
}
return ret;
}
// to compute diff and process it for display
export function computeDiff(diffUrl, diffId) {
// force diff computation ?
let force = diffUrl.indexOf('force=true') !== -1;
// it no forced computation and diff already computed, do nothing
if (!force && computedDiffs.hasOwnProperty(diffId)) {
return;
}
// mark diff computation as already requested
computedDiffs[diffId] = true;
$(`#${diffId}-loading`).css('visibility', 'visible');
// set spinner visible while requesting diff
$(`#${diffId}-loading`).css('display', 'block');
$(`#${diffId}-highlightjs`).css('display', 'none');
// request diff computation and process it
fetch(diffUrl, {credentials: 'same-origin'})
.then(response => response.json())
.then(data => {
// increment number of computed diffs
++nbDiffsComputed;
// toggle the 'Compute all diffs' button if all diffs have been computed
if (nbDiffsComputed === changes.length) {
$('#swh-compute-all-diffs').addClass('active');
}
// Large diff (> threshold) are not automatically computed,
// add a button to force its computation
if (data.diff_str.indexOf('Large diff') === 0) {
$(`#${diffId}`)[0].innerHTML = data.diff_str +
` `;
setDiffVisible(diffId);
} else if (data.diff_str.indexOf('@@') !== 0) {
$(`#${diffId}`).text(data.diff_str);
setDiffVisible(diffId);
} else {
// prepare code highlighting
$(`.${diffId}`).removeClass('nohighlight-swh');
$(`.${diffId}`).addClass(data.language);
// set unified diff text
$(`#${diffId}`).text(data.diff_str);
// code highlighting for unified diff
$(`#${diffId}`).each((i, block) => {
hljs.highlightBlock(block);
hljs.lineNumbersBlock(block);
});
// hljs.lineNumbersBlock is asynchronous so we have to postpone our
// next treatments by adding it at the end of the current js events queue
setTimeout(() => {
// process unified diff lines in order to generate side-by-side diffs text
// but also compute line numbers for unified and side-by-side difss
let linesInfoRegExp = new RegExp(/^@@ -(\d+),(\d+) \+(\d+),(\d+) @@$/gm);
let baseFromLine = '';
let baseToLine = '';
let fromToLines = [];
let fromLines = [];
let toLines = [];
let maxNumberChars = 0;
let diffFromStr = '';
let diffToStr = '';
let linesOffset = 0;
$(`#${diffId} .hljs-ln-numbers`).each((i, lnElt) => {
let lnText = lnElt.nextSibling.innerText;
let linesInfo = linesInfoRegExp.exec(lnText);
let fromLine = '';
let toLine = '';
// parsed lines info from the diff output
if (linesInfo) {
baseFromLine = parseInt(linesInfo[1]) - 1;
baseToLine = parseInt(linesInfo[3]) - 1;
linesOffset = 0;
diffFromStr += (lnText + '\n');
diffToStr += (lnText + '\n');
fromLines.push('');
toLines.push('');
// line removed in the from file
} else if (lnText.length > 0 && lnText[0] === '-') {
baseFromLine = baseFromLine + 1;
fromLine = baseFromLine.toString();
fromLines.push(fromLine);
++nbDeletions;
diffFromStr += (lnText + '\n');
++linesOffset;
// line added in the from file
} else if (lnText.length > 0 && lnText[0] === '+') {
baseToLine = baseToLine + 1;
toLine = baseToLine.toString();
toLines.push(toLine);
++nbAdditions;
diffToStr += (lnText + '\n');
--linesOffset;
// line present in both files
} else {
baseFromLine = baseFromLine + 1;
baseToLine = baseToLine + 1;
fromLine = baseFromLine.toString();
toLine = baseToLine.toString();
for (let j = 0; j < Math.abs(linesOffset); ++j) {
if (linesOffset > 0) {
diffToStr += '\n';
toLines.push('');
} else {
diffFromStr += '\n';
fromLines.push('');
}
}
linesOffset = 0;
diffFromStr += (lnText + '\n');
diffToStr += (lnText + '\n');
toLines.push(toLine);
fromLines.push(fromLine);
}
if (!baseFromLine) {
fromLine = '';
}
if (!baseToLine) {
toLine = '';
}
fromToLines[i] = [fromLine, toLine];
maxNumberChars = Math.max(maxNumberChars, fromLine.length);
maxNumberChars = Math.max(maxNumberChars, toLine.length);
});
// set side-by-side diffs text
$(`#${diffId}-from`).text(diffFromStr);
$(`#${diffId}-to`).text(diffToStr);
// code highlighting for side-by-side diffs
$(`#${diffId}-from, #${diffId}-to`).each((i, block) => {
hljs.highlightBlock(block);
hljs.lineNumbersBlock(block);
});
// hljs.lineNumbersBlock is asynchronous so we have to postpone our
// next treatments by adding it at the end of the current js events queue
setTimeout(() => {
// diff highlighting for added/removed lines on top of code highlighting
$(`.${diffId} .hljs-ln-numbers`).each((i, lnElt) => {
let lnText = lnElt.nextSibling.innerText;
let linesInfo = linesInfoRegExp.exec(lnText);
if (linesInfo) {
$(lnElt).parent().addClass('swh-diff-lines-info');
let linesInfoText = $(lnElt).parent().find('.hljs-ln-code .hljs-ln-line').text();
$(lnElt).parent().find('.hljs-ln-code .hljs-ln-line').children().remove();
$(lnElt).parent().find('.hljs-ln-code .hljs-ln-line').text('');
$(lnElt).parent().find('.hljs-ln-code .hljs-ln-line').append(`${linesInfoText}`);
} else if (lnText.length > 0 && lnText[0] === '-') {
$(lnElt).parent().addClass('swh-diff-removed-line');
} else if (lnText.length > 0 && lnText[0] === '+') {
$(lnElt).parent().addClass('swh-diff-added-line');
}
});
// set line numbers for unified diff
$(`#${diffId} .hljs-ln-numbers`).each((i, lnElt) => {
$(lnElt).children().attr('data-line-number',
formatDiffLineNumbers(fromToLines[i][0], fromToLines[i][1],
maxNumberChars));
});
// set line numbers for the from side-by-side diff
$(`#${diffId}-from .hljs-ln-numbers`).each((i, lnElt) => {
$(lnElt).children().attr('data-line-number',
formatDiffLineNumbers(fromLines[i], null,
maxNumberChars));
});
// set line numbers for the to side-by-side diff
$(`#${diffId}-to .hljs-ln-numbers`).each((i, lnElt) => {
$(lnElt).children().attr('data-line-number',
formatDiffLineNumbers(null, toLines[i],
maxNumberChars));
});
// last processings:
// - remove the '+' and '-' at the beginning of the diff lines
// from code highlighting
// - add the "no new line at end of file marker" if needed
$(`.${diffId} .hljs-ln-line`).each((i, lnElt) => {
if (lnElt.firstChild) {
if (lnElt.firstChild.nodeName !== '#text') {
let lineText = lnElt.firstChild.innerHTML;
if (lineText[0] === '-' || lineText[0] === '+') {
lnElt.firstChild.innerHTML = lineText.substr(1);
let newTextNode = document.createTextNode(lineText[0]);
$(lnElt).prepend(newTextNode);
}
}
$(lnElt).contents().filter((i, elt) => {
return elt.nodeType === 3; // Node.TEXT_NODE
}).each((i, textNode) => {
let swhNoNewLineMarker = '[swh-no-nl-marker]';
if (textNode.textContent.indexOf(swhNoNewLineMarker) !== -1) {
textNode.textContent = textNode.textContent.replace(swhNoNewLineMarker, '');
$(lnElt).append($(noNewLineMarker));
}
});
}
});
// hide the diff mode switch button in case of not generated diffs
if (data.diff_str.indexOf('Diffs are not generated for non textual content') !== 0) {
$(`#panel_${diffId} .diff-styles`).css('visibility', 'visible');
}
setDiffVisible(diffId);
});
});
}
});
}
function setDiffVisible(diffId) {
// set the unified diff visible by default
$(`#${diffId}-loading`).css('display', 'none');
$(`#${diffId}-highlightjs`).css('display', 'block');
// update displayed counters
$('#swh-revision-lines-added').text(`${nbAdditions} additions`);
$('#swh-revision-lines-deleted').text(`${nbDeletions} deletions`);
$('#swh-nb-diffs-computed').text(nbDiffsComputed);
// refresh the waypoints triggering diffs computation as
// the DOM layout has been updated
Waypoint.refreshAll();
}
// to compute all visible diffs in the viewport
function computeVisibleDiffs() {
$('.swh-file-diff-panel').each((i, elt) => {
if (isInViewport(elt)) {
let diffId = elt.id.replace('panel_', '');
computeDiff(diffsUrls[diffId], diffId);
}
});
}
function genDiffPanel(diffData) {
let diffPanelTitle = diffData.path;
if (diffData.type === 'rename') {
diffPanelTitle = `${diffData.from_path} → ${diffData.to_path}`;
}
let diffPanelHtml =
`
`;
return diffPanelHtml;
}
// setup waypoints to request diffs computation on the fly while scrolling
function setupWaypoints() {
for (let i = 0; i < changes.length; ++i) {
let diffData = changes[i];
// create a waypoint that will trigger diff computation when
// the top of the diff panel hits the bottom of the viewport
$(`#panel_${diffData.id}`).waypoint({
handler: function() {
if (isInViewport(this.element)) {
let diffId = this.element.id.replace('panel_', '');
computeDiff(diffsUrls[diffId], diffId);
this.destroy();
}
},
offset: '100%'
});
// create a waypoint that will trigger diff computation when
// the bottom of the diff panel hits the top of the viewport
$(`#panel_${diffData.id}`).waypoint({
handler: function() {
if (isInViewport(this.element)) {
let diffId = this.element.id.replace('panel_', '');
computeDiff(diffsUrls[diffId], diffId);
this.destroy();
}
},
offset: function() {
return -$(this.element).height();
}
});
}
Waypoint.refreshAll();
}
// callback to switch from side-by-side diff to unified one
export function showUnifiedDiff(event, diffId) {
$(`#${diffId}-splitted-diff`).css('display', 'none');
$(`#${diffId}-unified-diff`).css('display', 'block');
}
// callback to switch from unified diff to side-by-side one
export function showSplittedDiff(event, diffId) {
$(`#${diffId}-unified-diff`).css('display', 'none');
$(`#${diffId}-splitted-diff`).css('display', 'block');
}
// callback when the user clicks on the 'Compute all diffs' button
export function computeAllDiffs(event) {
$(event.currentTarget).addClass('active');
for (let diffId in diffsUrls) {
if (diffsUrls.hasOwnProperty(diffId)) {
computeDiff(diffsUrls[diffId], diffId);
}
}
event.stopPropagation();
}
export async function initRevisionDiff(revisionMessageBody, diffRevisionUrl) {
await import(/* webpackChunkName: "highlightjs" */ 'utils/highlightjs');
// callback when the 'Changes' tab is activated
$(document).on('shown.bs.tab', 'a[data-toggle="tab"]', e => {
if (e.currentTarget.text.trim() === 'Changes') {
$('#readme-panel').css('display', 'none');
if (changes) {
return;
}
// request computation of revision file changes list
// when navigating to the 'Changes' tab and add diff panels
// to the DOM when receiving the result
fetch(diffRevisionUrl, {credentials: 'same-origin'})
.then(response => response.json())
.then(data => {
changes = data.changes;
nbChangedFiles = data.total_nb_changes;
let changedFilesText = `${nbChangedFiles} changed file`;
if (nbChangedFiles !== 1) {
changedFilesText += 's';
}
$('#swh-revision-changed-files').text(changedFilesText);
$('#swh-total-nb-diffs').text(changes.length);
$('#swh-revision-changes-list pre')[0].innerHTML = data.changes_msg;
$('#swh-revision-changes-loading').css('display', 'none');
$('#swh-revision-changes-list pre').css('display', 'block');
$('#swh-compute-all-diffs').css('visibility', 'visible');
$('#swh-revision-changes-list').removeClass('in');
if (nbChangedFiles > changes.length) {
$('#swh-too-large-revision-diff').css('display', 'block');
$('#swh-nb-loaded-diffs').text(changes.length);
}
for (let i = 0; i < changes.length; ++i) {
let diffData = changes[i];
diffsUrls[diffData.id] = diffData.diff_url;
$('#swh-revision-diffs').append(genDiffPanel(diffData));
}
setupWaypoints();
computeVisibleDiffs();
});
} else if (e.currentTarget.text.trim() === 'Files') {
$('#readme-panel').css('display', 'block');
}
});
$(document).ready(() => {
if (revisionMessageBody.length > 0) {
$('#swh-revision-message').addClass('in');
} else {
$('#swh-collapse-revision-message').attr('data-toggle', '');
}
let $root = $('html, body');
// callback when the user requests to scroll on a specific diff or back to top
$('#swh-revision-changes-list a[href^="#"], #back-to-top a[href^="#"]').click(e => {
let href = $.attr(e.currentTarget, 'href');
// disable waypoints while scrolling as we do not want to
// launch computation of diffs the user is not interested in
// (file changes list can be large)
Waypoint.disableAll();
$root.animate(
{
scrollTop: $(href).offset().top
},
{
duration: 500,
complete: () => {
window.location.hash = href;
// enable waypoints back after scrolling
Waypoint.enableAll();
// compute diffs visible in the viewport
computeVisibleDiffs();
}
});
return false;
});
});
}
diff --git a/swh/web/assets/src/bundles/revision/index.js b/swh/web/assets/src/bundles/revision/index.js
index 32430df66..7f6c1ba7d 100644
--- a/swh/web/assets/src/bundles/revision/index.js
+++ b/swh/web/assets/src/bundles/revision/index.js
@@ -1,4 +1,11 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// bundle for browse/revision view
import './revision.css';
export * from './diff-utils';
diff --git a/swh/web/assets/src/bundles/revision/revision.css b/swh/web/assets/src/bundles/revision/revision.css
index 2b41a3ff9..ee729ea6d 100644
--- a/swh/web/assets/src/bundles/revision/revision.css
+++ b/swh/web/assets/src/bundles/revision/revision.css
@@ -1,43 +1,50 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.swh-diff-lines-info {
background-color: rgba(0, 0, 255, 0.1) !important;
}
.swh-diff-added-line {
background-color: rgba(0, 255, 0, 0.1) !important;
}
.swh-diff-removed-line {
background-color: rgba(255, 0, 0, 0.1) !important;
}
span.no-nl-marker {
position: relative;
color: #cb2431;
vertical-align: middle;
}
span.no-nl-marker svg {
vertical-align: text-bottom;
}
span.no-nl-marker svg path {
fill: currentColor;
}
.swh-revision-log-entry-id {
min-width: 110px;
max-width: 110px;
width: 110px;
}
.swh-revision-log-entry-author {
min-width: 160px;
max-width: 160px;
width: 160px;
}
.swh-revision-log-entry-date {
min-width: 230px;
max-width: 230px;
width: 230px;
}
diff --git a/swh/web/assets/src/bundles/vault/index.js b/swh/web/assets/src/bundles/vault/index.js
index 60e977c64..ef18a1256 100644
--- a/swh/web/assets/src/bundles/vault/index.js
+++ b/swh/web/assets/src/bundles/vault/index.js
@@ -1,5 +1,12 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// bundle for the vault related views
import './vault.css';
export * from './vault-ui';
export * from './vault-create-tasks';
diff --git a/swh/web/assets/src/bundles/vault/vault-create-tasks.js b/swh/web/assets/src/bundles/vault/vault-create-tasks.js
index aef446dc9..0045793c6 100644
--- a/swh/web/assets/src/bundles/vault/vault-create-tasks.js
+++ b/swh/web/assets/src/bundles/vault/vault-create-tasks.js
@@ -1,83 +1,90 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
import {handleFetchError} from 'utils/functions';
function addVaultCookingTask(cookingTask) {
let vaultCookingTasks = JSON.parse(localStorage.getItem('swh-vault-cooking-tasks'));
if (!vaultCookingTasks) {
vaultCookingTasks = [];
}
if (vaultCookingTasks.find(val => {
return val.object_type === cookingTask.object_type &&
val.object_id === cookingTask.object_id;
}) === undefined) {
let cookingUrl;
if (cookingTask.object_type === 'directory') {
cookingUrl = Urls.vault_cook_directory(cookingTask.object_id);
} else {
cookingUrl = Urls.vault_cook_revision_gitfast(cookingTask.object_id);
}
if (cookingTask.email) {
cookingUrl += '?email=' + cookingTask.email;
}
fetch(cookingUrl, {credentials: 'same-origin', method: 'POST'})
.then(handleFetchError)
.then(() => {
vaultCookingTasks.push(cookingTask);
localStorage.setItem('swh-vault-cooking-tasks', JSON.stringify(vaultCookingTasks));
$('#vault-cook-directory-modal').modal('hide');
$('#vault-cook-revision-modal').modal('hide');
swh.browse.showTab('#vault');
})
.catch(() => {
$('#vault-cook-directory-modal').modal('hide');
$('#vault-cook-revision-modal').modal('hide');
});
} else {
swh.browse.showTab('#vault');
}
}
function validateEmail(email) {
let re = /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
return re.test(String(email).toLowerCase());
}
export function cookDirectoryArchive(directoryId) {
let email = $('#swh-vault-directory-email').val().trim();
if (!email || validateEmail(email)) {
let cookingTask = {'object_type': 'directory',
'object_id': directoryId,
'email': email,
'status': 'new'};
addVaultCookingTask(cookingTask);
} else {
$('#invalid-email-modal').modal('show');
}
}
export function cookRevisionArchive(revisionId) {
let email = $('#swh-vault-revision-email').val().trim();
if (!email || validateEmail(email)) {
let cookingTask = {
'object_type': 'revision',
'object_id': revisionId,
'email': email,
'status': 'new'
};
addVaultCookingTask(cookingTask);
} else {
$('#invalid-email-modal').modal('show');
}
}
export function initTaskCreationUi() {
// reparent the modals to the top navigation div in order to be able
// to display them
$(document).ready(function() {
$('.swh-browse-top-navigation').append($('#vault-cook-directory-modal'));
$('.swh-browse-top-navigation').append($('#vault-cook-revision-modal'));
$('.swh-browse-top-navigation').append($('#invalid-email-modal'));
});
}
diff --git a/swh/web/assets/src/bundles/vault/vault-ui.js b/swh/web/assets/src/bundles/vault/vault-ui.js
index bcb3750df..8dc7203cb 100644
--- a/swh/web/assets/src/bundles/vault/vault-ui.js
+++ b/swh/web/assets/src/bundles/vault/vault-ui.js
@@ -1,175 +1,182 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
import {handleFetchErrors} from 'utils/functions';
let progress = `
;`;
let pollingInterval = 5000;
let checkVaultId;
function updateProgressBar(progressBar, cookingTask) {
if (cookingTask.status === 'new') {
progressBar.css('background-color', 'rgba(128, 128, 128, 0.5)');
} else if (cookingTask.status === 'pending') {
progressBar.css('background-color', 'rgba(0, 0, 255, 0.5)');
} else if (cookingTask.status === 'done') {
progressBar.css('background-color', '#5cb85c');
} else if (cookingTask.status === 'failed') {
progressBar.css('background-color', 'rgba(255, 0, 0, 0.5)');
progressBar.css('background-image', 'none');
}
progressBar.text(cookingTask.progress_message || cookingTask.status);
if (cookingTask.status === 'new' || cookingTask.status === 'pending') {
progressBar.addClass('progress-bar-animated');
} else {
progressBar.removeClass('progress-bar-striped');
}
}
function checkVaultCookingTasks() {
let vaultCookingTasks = JSON.parse(localStorage.getItem('swh-vault-cooking-tasks'));
if (!vaultCookingTasks || vaultCookingTasks.length === 0) {
$('.swh-vault-table tbody tr').remove();
checkVaultId = setTimeout(checkVaultCookingTasks, pollingInterval);
return;
}
let cookingTaskRequests = [];
let tasks = {};
let currentObjectIds = [];
for (let i = 0; i < vaultCookingTasks.length; ++i) {
let cookingTask = vaultCookingTasks[i];
currentObjectIds.push(cookingTask.object_id);
tasks[cookingTask.object_id] = cookingTask;
let cookingUrl;
if (cookingTask.object_type === 'directory') {
cookingUrl = Urls.vault_cook_directory(cookingTask.object_id);
} else {
cookingUrl = Urls.vault_cook_revision_gitfast(cookingTask.object_id);
}
if (cookingTask.status !== 'done' && cookingTask.status !== 'failed') {
cookingTaskRequests.push(fetch(cookingUrl, {credentials: 'same-origin'}));
}
}
$('.swh-vault-table tbody tr').each((i, row) => {
let objectId = $(row).find('.vault-object-id').data('object-id');
if ($.inArray(objectId, currentObjectIds) === -1) {
$(row).remove();
}
});
Promise.all(cookingTaskRequests)
.then(handleFetchErrors)
.then(responses => Promise.all(responses.map(r => r.json())))
.then(cookingTasks => {
let table = $('#vault-cooking-tasks tbody');
for (let i = 0; i < cookingTasks.length; ++i) {
let cookingTask = tasks[cookingTasks[i].obj_id];
cookingTask.status = cookingTasks[i].status;
cookingTask.fetch_url = cookingTasks[i].fetch_url;
cookingTask.progress_message = cookingTasks[i].progress_message;
}
for (let i = 0; i < vaultCookingTasks.length; ++i) {
let cookingTask = vaultCookingTasks[i];
let rowTask = $('#vault-task-' + cookingTask.object_id);
if (!rowTask.length) {
let browseUrl;
if (cookingTask.object_type === 'directory') {
browseUrl = Urls.browse_directory(cookingTask.object_id);
} else {
browseUrl = Urls.browse_revision(cookingTask.object_id);
}
let progressBar = $.parseHTML(progress)[0];
let progressBarContent = $(progressBar).find('.progress-bar');
updateProgressBar(progressBarContent, cookingTask);
let tableRow;
if (cookingTask.object_type === 'directory') {
tableRow = `
`;
} else {
tableRow = `
`;
}
tableRow += '
';
if (cookingTask.object_type === 'directory') {
tableRow += '
`;
let downloadLink = 'Waiting for download link to be available';
if (cookingTask.status === 'done') {
downloadLink = `Download';
} else if (cookingTask.status === 'failed') {
downloadLink = '';
}
tableRow += `
${downloadLink}
`;
tableRow += '
';
table.prepend(tableRow);
} else {
let progressBar = rowTask.find('.progress-bar');
updateProgressBar(progressBar, cookingTask);
let downloadLink = rowTask.find('.vault-dl-link');
if (cookingTask.status === 'done') {
downloadLink[0].innerHTML = `Download';
} else if (cookingTask.status === 'failed') {
downloadLink[0].innerHTML = '';
}
}
}
localStorage.setItem('swh-vault-cooking-tasks', JSON.stringify(vaultCookingTasks));
checkVaultId = setTimeout(checkVaultCookingTasks, pollingInterval);
})
.catch(() => {});
}
export function initUi() {
$('#vault-tasks-toggle-selection').change(event => {
$('.vault-task-toggle-selection').prop('checked', event.currentTarget.checked);
});
$('#vault-remove-tasks').click(() => {
clearTimeout(checkVaultId);
let tasksToRemove = [];
$('.swh-vault-table tbody tr').each((i, row) => {
let taskSelected = $(row).find('.vault-task-toggle-selection').prop('checked');
if (taskSelected) {
let objectId = $(row).find('.vault-object-id').data('object-id');
tasksToRemove.push(objectId);
$(row).remove();
}
});
let vaultCookingTasks = JSON.parse(localStorage.getItem('swh-vault-cooking-tasks'));
vaultCookingTasks = $.grep(vaultCookingTasks, task => {
return $.inArray(task.object_id, tasksToRemove) === -1;
});
localStorage.setItem('swh-vault-cooking-tasks', JSON.stringify(vaultCookingTasks));
$('#vault-tasks-toggle-selection').prop('checked', false);
checkVaultId = setTimeout(checkVaultCookingTasks, pollingInterval);
});
checkVaultId = setTimeout(checkVaultCookingTasks, pollingInterval);
$(document).on('shown.bs.tab', 'a[data-toggle="tab"]', e => {
if (e.currentTarget.text.trim() === 'Vault') {
clearTimeout(checkVaultId);
checkVaultCookingTasks();
}
});
window.onfocus = () => {
clearTimeout(checkVaultId);
checkVaultCookingTasks();
};
}
diff --git a/swh/web/assets/src/bundles/vault/vault.css b/swh/web/assets/src/bundles/vault/vault.css
index 92f30effc..c4b486863 100644
--- a/swh/web/assets/src/bundles/vault/vault.css
+++ b/swh/web/assets/src/bundles/vault/vault.css
@@ -1,10 +1,17 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.swh-vault-table {
border-bottom: none !important;
margin-bottom: 0 !important;
}
.swh-vault-table td {
vertical-align: middle !important;
border-top: 1px solid #ddd !important;
}
diff --git a/swh/web/assets/src/bundles/vendors/datatables.css b/swh/web/assets/src/bundles/vendors/datatables.css
index 14d7d2e83..e689e5e65 100644
--- a/swh/web/assets/src/bundles/vendors/datatables.css
+++ b/swh/web/assets/src/bundles/vendors/datatables.css
@@ -1,25 +1,32 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.dataTables_filter {
margin-top: 15px;
}
.dataTables_filter input {
width: 70%;
float: right;
}
.dataTables_filter label {
font-weight: bold !important;
white-space: inherit !important;
}
.dataTables_wrapper {
position: static;
}
.swh-table.dataTable {
border-collapse: collapse !important;
}
.swh-table.dataTable th {
border-top: none;
}
diff --git a/swh/web/assets/src/bundles/vendors/index.js b/swh/web/assets/src/bundles/vendors/index.js
index 47fbc5a1f..749a6771d 100644
--- a/swh/web/assets/src/bundles/vendors/index.js
+++ b/swh/web/assets/src/bundles/vendors/index.js
@@ -1,21 +1,28 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// vendors bundles centralizing assets used in all swh-web applications
// polyfills in order to use advanded js features (like Promise or fetch)
// in older browsers
import 'babel-polyfill';
import 'whatwg-fetch';
// jquery and bootstrap
import 'jquery';
import 'bootstrap-loader/lib/bootstrap.loader?configFilePath=../../../swh/web/assets/config/.bootstraprc!bootstrap-loader/no-op.js';
// jquery datatables
import 'datatables.net';
import 'datatables.net-bs4/css/dataTables.bootstrap4.css';
import './datatables.css';
// web fonts
import 'typeface-alegreya';
import 'typeface-alegreya-sans';
import 'font-awesome/css/font-awesome.css';
import 'octicons/build/font/octicons.css';
diff --git a/swh/web/assets/src/bundles/webapp/breadcrumbs.css b/swh/web/assets/src/bundles/webapp/breadcrumbs.css
index 3083985af..3c5cc50e3 100644
--- a/swh/web/assets/src/bundles/webapp/breadcrumbs.css
+++ b/swh/web/assets/src/bundles/webapp/breadcrumbs.css
@@ -1,24 +1,31 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.bread-crumbs {
display: inline-block;
overflow: hidden;
color: rgba(0, 0, 0, 0.55);
}
.bread-crumbs ul {
list-style-type: none;
}
.bread-crumbs li {
float: left;
list-style-type: none;
}
.bread-crumbs a {
color: rgba(0, 0, 0, 0.75);
border-bottom-style: none;
}
.bread-crumbs a:hover {
color: rgba(0, 0, 0, 0.85);
text-decoration: underline;
}
diff --git a/swh/web/assets/src/bundles/webapp/code-highlighting.js b/swh/web/assets/src/bundles/webapp/code-highlighting.js
index 850f9e1a5..872a52739 100644
--- a/swh/web/assets/src/bundles/webapp/code-highlighting.js
+++ b/swh/web/assets/src/bundles/webapp/code-highlighting.js
@@ -1,111 +1,118 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
export async function highlightCode(showLineNumbers = true) {
await import(/* webpackChunkName: "highlightjs" */ 'utils/highlightjs');
// empty hljs language definition
function noHighlight(hljs) {
return {};
}
// just a trick to get line numbers working when no highlight
// has to be performed
hljs.registerLanguage('nohighlight-swh', noHighlight);
// keep track of the first highlighted line
let firstHighlightedLine = null;
// highlighting color
let lineHighlightColor = 'rgb(193, 255, 193)';
// function to highlight a line
function highlightLine(i) {
let lineTd = $(`.swh-content div[data-line-number="${i}"]`).parent().parent();
lineTd.css('background-color', lineHighlightColor);
return lineTd;
}
function removeHash() {
history.replaceState('', document.title, window.location.pathname + window.location.search);
}
// function to reset highlighting
function resetHighlightedLines() {
firstHighlightedLine = null;
$('.swh-content tr').css('background-color', 'inherit');
}
function scrollToLine(lineDomElt) {
if ($(lineDomElt).closest('.swh-content').length > 0) {
$('html, body').animate({
scrollTop: $(lineDomElt).offset().top - 70
}, 500);
}
}
// function to highlight lines based on a url fragment
// in the form '#Lx' or '#Lx-Ly'
function parseUrlFragmentForLinesToHighlight() {
let lines = [];
let linesRegexp = new RegExp(/L(\d+)/g);
let line = linesRegexp.exec(window.location.hash);
while (line) {
lines.push(parseInt(line[1]));
line = linesRegexp.exec(window.location.hash);
}
resetHighlightedLines();
if (lines.length === 1) {
firstHighlightedLine = parseInt(lines[0]);
scrollToLine(highlightLine(lines[0]));
} else if (lines[0] < lines[lines.length - 1]) {
firstHighlightedLine = parseInt(lines[0]);
scrollToLine(highlightLine(lines[0]));
for (let i = lines[0] + 1; i <= lines[lines.length - 1]; ++i) {
highlightLine(i);
}
}
}
$(document).ready(() => {
// highlight code and add line numbers
$('code').each((i, block) => {
hljs.highlightBlock(block);
if (showLineNumbers) {
hljs.lineNumbersBlock(block);
}
});
if (!showLineNumbers) {
return;
}
// click handler to dynamically highlight line(s)
// when the user clicks on a line number (lines range
// can also be highlighted while holding the shift key)
$('body').click(evt => {
if (evt.target.classList.contains('hljs-ln-n')) {
let line = parseInt($(evt.target).data('line-number'));
if (evt.shiftKey && firstHighlightedLine && line > firstHighlightedLine) {
let firstLine = firstHighlightedLine;
resetHighlightedLines();
for (let i = firstLine; i <= line; ++i) {
highlightLine(i);
}
firstHighlightedLine = firstLine;
window.location.hash = `#L${firstLine}-L${line}`;
} else {
resetHighlightedLines();
highlightLine(line);
window.location.hash = `#L${line}`;
scrollToLine(evt.target);
}
} else {
resetHighlightedLines();
removeHash();
}
});
// update lines highlighting when the url fragment changes
$(window).on('hashchange', () => parseUrlFragmentForLinesToHighlight());
$('.navbar-nav.swh-browse-nav a[href="#browse"]').tab('show');
});
}
diff --git a/swh/web/assets/src/bundles/webapp/index.js b/swh/web/assets/src/bundles/webapp/index.js
index f7401639d..fbc386eeb 100644
--- a/swh/web/assets/src/bundles/webapp/index.js
+++ b/swh/web/assets/src/bundles/webapp/index.js
@@ -1,13 +1,20 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// webapp entrypoint bundle centralizing global custom stylesheets
// and utility js modules used in all swh-web applications
// explicitely import the vendors bundle
import '../vendors';
// global swh-web custom stylesheets
import './webapp.css';
import './breadcrumbs.css';
// utility js modules
export * from './code-highlighting';
export * from './markdown-rendering';
diff --git a/swh/web/assets/src/bundles/webapp/markdown-rendering.js b/swh/web/assets/src/bundles/webapp/markdown-rendering.js
index e9ff3cb1d..145dd1064 100644
--- a/swh/web/assets/src/bundles/webapp/markdown-rendering.js
+++ b/swh/web/assets/src/bundles/webapp/markdown-rendering.js
@@ -1,14 +1,21 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
export async function renderMarkdown(domElt, markdownDocUrl) {
let showdown = await import(/* webpackChunkName: "showdown" */ 'utils/showdown');
$(document).ready(() => {
let converter = new showdown.Converter({tables: true});
fetch(markdownDocUrl, {credentials: 'same-origin'})
.then(response => response.text())
.then(data => {
$(domElt).html(converter.makeHtml(data));
});
});
}
diff --git a/swh/web/assets/src/bundles/webapp/webapp.css b/swh/web/assets/src/bundles/webapp/webapp.css
index 7d7f7d464..c0f14ac9f 100644
--- a/swh/web/assets/src/bundles/webapp/webapp.css
+++ b/swh/web/assets/src/bundles/webapp/webapp.css
@@ -1,235 +1,242 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
html {
height: 100%;
overflow-x: hidden;
}
body {
min-height: 100%;
margin: 0;
position: relative;
padding-bottom: 120px;
}
a {
border-bottom-style: none;
outline: none;
}
code {
background-color: #f9f2f4;
}
pre code {
background-color: transparent;
}
footer {
background-color: #262626;
color: #fff;
font-size: 0.8rem;
position: absolute;
bottom: 0;
width: 100%;
padding-top: 20px;
padding-bottom: 20px;
}
footer a,
footer a:visited {
color: #fecd1b;
}
footer a:hover {
text-decoration: underline;
}
pre {
background-color: #f5f5f5;
border: 1px solid #ccc;
border-radius: 4px;
padding: 9.5px;
font-size: 0.8rem;
}
.btn.active {
background-color: #e7e7e7;
}
.card {
margin-bottom: 5px !important;
overflow-x: auto;
}
.navbar-brand {
padding: 5px;
margin-right: 0;
}
.table {
margin-bottom: 0;
}
.swh-web-app-link a {
text-decoration: none;
outline: none;
border: none;
}
.swh-web-app-link:hover {
background-color: #efeff2;
}
.table > thead > tr > th {
border-top: none;
border-bottom: 1px solid #e20026;
}
.table > tbody > tr > td {
border-style: none;
}
.sitename .first-word,
.sitename .second-word {
color: rgba(0, 0, 0, 0.75);
font-weight: normal;
font-size: 1.2rem;
}
.sitename .first-word {
font-family: 'Alegreya Sans', sans-serif;
}
.sitename .second-word {
font-family: 'Alegreya', serif;
}
.swh-api-doc-route-upcoming > td,
.swh-api-doc-route-upcoming > td > a {
font-size: 90%;
}
.swh-api-doc-route-deprecated > td,
.swh-api-doc-route-deprecated > td > a {
color: red;
}
.swh-api-doc p {
margin-bottom: 0;
}
.swh-api-doc dt {
text-align: right;
}
.swh-counter {
font-size: 150%;
}
.swh-http-error {
margin: 0 auto;
text-align: center;
}
.swh-http-error-head {
color: #2d353c;
font-size: 30px;
}
.swh-http-error-code {
bottom: 60%;
color: #2d353c;
font-size: 96px;
line-height: 80px;
margin-bottom: 10px !important;
}
.swh-http-error-desc {
font-size: 12px;
color: #647788;
text-align: center;
}
.swh-http-error-desc pre {
display: inline-block;
text-align: left;
max-width: 800px;
white-space: pre-wrap;
}
.popover {
max-width: 100%;
}
.modal {
text-align: center;
padding: 0 !important;
}
.modal::before {
content: '';
display: inline-block;
height: 100%;
vertical-align: middle;
margin-right: -4px;
}
.modal-dialog {
display: inline-block;
text-align: left;
vertical-align: middle;
}
.dropdown-submenu {
position: relative;
}
.dropdown-submenu .dropdown-menu {
top: 0;
left: -100%;
margin-top: -5px;
margin-left: -2px;
}
.dropdown-item:hover,
.dropdown-item:focus {
background-color: rgba(0, 0, 0, 0.1);
}
a.dropdown-left::before {
content: "\f0d9";
font-family: 'FontAwesome';
display: block;
width: 20px;
height: 20px;
float: left;
margin-left: 0;
}
#swh-navbar-collapse {
border-top-style: none;
border-left-style: none;
border-right-style: none;
border-bottom: 5px solid;
border-image: linear-gradient(to right, rgb(226, 0, 38) 0%, rgb(254, 205, 27) 100%) 1 1 1 1;
width: 100%;
padding: 5px;
margin-bottom: 20px;
}
#back-to-top {
display: initial;
position: fixed;
bottom: 30px;
right: 30px;
z-index: 10;
}
#back-to-top a img {
display: block;
width: 32px;
height: 32px;
background-size: 32px 32px;
text-indent: -999px;
overflow: hidden;
}
diff --git a/swh/web/assets/src/utils/functions.js b/swh/web/assets/src/utils/functions.js
index e4d056d75..79d92a70a 100644
--- a/swh/web/assets/src/utils/functions.js
+++ b/swh/web/assets/src/utils/functions.js
@@ -1,21 +1,28 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// utility functions
export function handleFetchError(response) {
if (!response.ok) {
throw Error(response.statusText);
}
return response;
}
export function handleFetchErrors(responses) {
for (let i = 0; i < responses.length; ++i) {
if (!responses[i].ok) {
throw Error(responses[i].statusText);
}
}
return responses;
}
export function staticAsset(asset) {
return `${__STATIC__}${asset}`;
}
diff --git a/swh/web/assets/src/utils/heaps-permute.js b/swh/web/assets/src/utils/heaps-permute.js
index 4b1bcdac3..ef103160b 100644
--- a/swh/web/assets/src/utils/heaps-permute.js
+++ b/swh/web/assets/src/utils/heaps-permute.js
@@ -1,25 +1,32 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// http://dsernst.com/2014/12/14/heaps-permutation-algorithm-in-javascript/
function swap(array, pos1, pos2) {
let temp = array[pos1];
array[pos1] = array[pos2];
array[pos2] = temp;
}
export function heapsPermute(array, output, n) {
n = n || array.length; // set n default to array.length
if (n === 1) {
output(array);
} else {
for (let i = 1; i <= n; i += 1) {
heapsPermute(array, output, n - 1);
let j;
if (n % 2) {
j = 1;
} else {
j = i;
}
swap(array, j - 1, n - 1); // -1 to account for javascript zero-indexing
}
}
}
diff --git a/swh/web/assets/src/utils/highlightjs.css b/swh/web/assets/src/utils/highlightjs.css
index 31d2a7989..c669d918b 100644
--- a/swh/web/assets/src/utils/highlightjs.css
+++ b/swh/web/assets/src/utils/highlightjs.css
@@ -1,26 +1,33 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.highlightjs pre {
background-color: transparent;
border-radius: 0;
border-color: transparent;
}
.hljs {
background-color: transparent;
white-space: pre;
}
/* for block of numbers */
.hljs-ln-numbers {
user-select: none;
text-align: center;
color: #aaa;
border-right: 1px solid #ccc;
vertical-align: top;
padding-right: 5px !important;
}
/* for block of code */
.hljs-ln-code {
padding-left: 10px !important;
width: 100%;
}
diff --git a/swh/web/assets/src/utils/highlightjs.js b/swh/web/assets/src/utils/highlightjs.js
index f36925c19..ec6eae1a0 100644
--- a/swh/web/assets/src/utils/highlightjs.js
+++ b/swh/web/assets/src/utils/highlightjs.js
@@ -1,6 +1,13 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// highlightjs chunk that will be lazily loaded
import 'highlightjs';
import 'highlightjs-line-numbers.js';
import 'highlightjs/styles/github.css';
import './highlightjs.css';
diff --git a/swh/web/assets/src/utils/showdown.css b/swh/web/assets/src/utils/showdown.css
index 84291809f..4833042ed 100644
--- a/swh/web/assets/src/utils/showdown.css
+++ b/swh/web/assets/src/utils/showdown.css
@@ -1,19 +1,26 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
.swh-showdown a {
outline: none;
border: none;
}
.swh-showdown table {
border-collapse: collapse;
}
.swh-showdown table,
.swh-showdown table th,
.swh-showdown table td {
padding: 6px 13px;
border: 1px solid #dfe2e5;
}
.swh-showdown table tr:nth-child(even) {
background-color: #f2f2f2;
}
diff --git a/swh/web/assets/src/utils/showdown.js b/swh/web/assets/src/utils/showdown.js
index e9898db34..a05769336 100644
--- a/swh/web/assets/src/utils/showdown.js
+++ b/swh/web/assets/src/utils/showdown.js
@@ -1,4 +1,11 @@
+/**
+ * Copyright (C) 2018 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
// showdown chunk that will be lazily loaded
import './showdown.css';
export * from 'showdown';
diff --git a/swh/web/browse/browseurls.py b/swh/web/browse/browseurls.py
index 49fc77ed3..c5968158b 100644
--- a/swh/web/browse/browseurls.py
+++ b/swh/web/browse/browseurls.py
@@ -1,38 +1,38 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.web.common.urlsindex import UrlsIndex
class BrowseUrls(UrlsIndex):
"""
Class to manage SWH web browse application urls.
"""
scope = 'browse'
class browse_route(object): # noqa: N801
"""
Decorator to ease the registration of a SWH web browse endpoint
Args:
url_patterns: list of url patterns used by Django to identify the browse routes
view_name: the name of the Django view associated to the routes used to
reverse the url
""" # noqa
def __init__(self, *url_patterns, view_name=None):
super().__init__()
self.url_patterns = []
for url_pattern in url_patterns:
self.url_patterns.append('^' + url_pattern + '$')
self.view_name = view_name
def __call__(self, f):
# register the route and its view in the browse endpoints index
for url_pattern in self.url_patterns:
BrowseUrls.add_url_pattern(url_pattern, f, self.view_name)
return f
diff --git a/swh/web/browse/urls.py b/swh/web/browse/urls.py
index 0e7273ba6..7935ef4d9 100644
--- a/swh/web/browse/urls.py
+++ b/swh/web/browse/urls.py
@@ -1,39 +1,39 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.conf.urls import url
from django.shortcuts import render
import swh.web.browse.views.directory # noqa
import swh.web.browse.views.content # noqa
import swh.web.browse.views.identifiers # noqa
import swh.web.browse.views.origin # noqa
import swh.web.browse.views.person # noqa
import swh.web.browse.views.release # noqa
import swh.web.browse.views.revision # noqa
import swh.web.browse.views.snapshot # noqa
from swh.web.browse.browseurls import BrowseUrls
def default_browse_view(request):
"""Default django view used as an entry point
for the swh browse ui web application.
The url that point to it is /browse/.
Args:
request: input django http request
"""
return render(request, 'person.html',
{'heading': 'Browse the Software Heritage archive',
'empty_browse': True})
urlpatterns = [
url(r'^$', default_browse_view, name='browse-homepage')
]
urlpatterns += BrowseUrls.get_url_patterns()
diff --git a/swh/web/browse/utils.py b/swh/web/browse/utils.py
index c20eb7703..891fdfa50 100644
--- a/swh/web/browse/utils.py
+++ b/swh/web/browse/utils.py
@@ -1,919 +1,919 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import base64
import magic
import math
import stat
from django.core.cache import cache
from django.utils.safestring import mark_safe
from importlib import reload
from swh.web.common import highlightjs, service
from swh.web.common.exc import NotFoundExc
from swh.web.common.utils import (
reverse, format_utc_iso_date, parse_timestamp
)
from swh.web.config import get_config
def get_directory_entries(sha1_git):
"""Function that retrieves the content of a SWH directory
from the SWH archive.
The directories entries are first sorted in lexicographical order.
Sub-directories and regular files are then extracted.
Args:
sha1_git: sha1_git identifier of the directory
Returns:
A tuple whose first member corresponds to the sub-directories list
and second member the regular files list
Raises:
NotFoundExc if the directory is not found
"""
cache_entry_id = 'directory_entries_%s' % sha1_git
cache_entry = cache.get(cache_entry_id)
if cache_entry:
return cache_entry
entries = list(service.lookup_directory(sha1_git))
entries = sorted(entries, key=lambda e: e['name'])
for entry in entries:
entry['perms'] = stat.filemode(entry['perms'])
dirs = [e for e in entries if e['type'] == 'dir']
files = [e for e in entries if e['type'] == 'file']
cache.set(cache_entry_id, (dirs, files))
return dirs, files
def get_mimetype_and_encoding_for_content(content):
"""Function that returns the mime type and the encoding associated to
a content buffer using the magic module under the hood.
Args:
content (bytes): a content buffer
Returns:
A tuple (mimetype, encoding), for instance ('text/plain', 'us-ascii'),
associated to the provided content.
"""
while True:
try:
magic_result = magic.detect_from_content(content)
mime_type = magic_result.mime_type
encoding = magic_result.encoding
break
except Exception as exc:
# workaround an issue with the magic module who can fail
# if detect_from_content is called multiple times in
# a short amount of time
reload(magic)
return mime_type, encoding
# maximum authorized content size in bytes for HTML display
# with code highlighting
content_display_max_size = get_config()['content_display_max_size']
def request_content(query_string, max_size=content_display_max_size):
"""Function that retrieves a SWH content from the SWH archive.
Raw bytes content is first retrieved, then the content mime type.
If the mime type is not stored in the archive, it will be computed
using Python magic module.
Args:
query_string: a string of the form "[ALGO_HASH:]HASH" where
optional ALGO_HASH can be either *sha1*, *sha1_git*, *sha256*,
or *blake2s256* (default to *sha1*) and HASH the hexadecimal
representation of the hash value
max_size: the maximum size for a content to retrieve (default to 1MB,
no size limit if None)
Returns:
A tuple whose first member corresponds to the content raw bytes
and second member the content mime type
Raises:
NotFoundExc if the content is not found
"""
content_data = service.lookup_content(query_string)
filetype = service.lookup_content_filetype(query_string)
language = service.lookup_content_language(query_string)
license = service.lookup_content_license(query_string)
mimetype = 'unknown'
encoding = 'unknown'
if filetype:
mimetype = filetype['mimetype']
encoding = filetype['encoding']
if not max_size or content_data['length'] < max_size:
content_raw = service.lookup_content_raw(query_string)
content_data['raw_data'] = content_raw['data']
if not filetype:
mimetype, encoding = \
get_mimetype_and_encoding_for_content(content_data['raw_data'])
# encode textual content to utf-8 if needed
if mimetype.startswith('text/'):
# probably a malformed UTF-8 content, reencode it
# by replacing invalid chars with a substitution one
if encoding == 'unknown-8bit':
content_data['raw_data'] = \
content_data['raw_data'].decode('utf-8', 'replace')\
.encode('utf-8')
elif 'ascii' not in encoding and encoding not in ['utf-8', 'binary']: # noqa
content_data['raw_data'] = \
content_data['raw_data'].decode(encoding, 'replace')\
.encode('utf-8')
else:
content_data['raw_data'] = None
content_data['mimetype'] = mimetype
content_data['encoding'] = encoding
if language:
content_data['language'] = language['lang']
else:
content_data['language'] = 'not detected'
if license:
content_data['licenses'] = ', '.join(license['licenses'])
else:
content_data['licenses'] = 'not detected'
return content_data
_browsers_supported_image_mimes = set(['image/gif', 'image/png',
'image/jpeg', 'image/bmp',
'image/webp'])
def prepare_content_for_display(content_data, mime_type, path):
"""Function that prepares a content for HTML display.
The function tries to associate a programming language to a
content in order to perform syntax highlighting client-side
using highlightjs. The language is determined using either
the content filename or its mime type.
If the mime type corresponds to an image format supported
by web browsers, the content will be encoded in base64
for displaying the image.
Args:
content_data (bytes): raw bytes of the content
mime_type (string): mime type of the content
path (string): path of the content including filename
Returns:
A dict containing the content bytes (possibly different from the one
provided as parameter if it is an image) under the key 'content_data
and the corresponding highlightjs language class under the
key 'language'.
"""
language = highlightjs.get_hljs_language_from_filename(path)
if not language:
language = highlightjs.get_hljs_language_from_mime_type(mime_type)
if not language:
language = 'nohighlight-swh'
elif mime_type.startswith('application/'):
mime_type = mime_type.replace('application/', 'text/')
if mime_type.startswith('image/'):
if mime_type in _browsers_supported_image_mimes:
content_data = base64.b64encode(content_data)
else:
content_data = None
return {'content_data': content_data,
'language': language}
def get_origin_visits(origin_info):
"""Function that returns the list of visits for a swh origin.
That list is put in cache in order to speedup the navigation
in the swh web browse ui.
Args:
origin_id (int): the id of the swh origin to fetch visits from
Returns:
A list of dict describing the origin visits::
[{'date': ,
'origin': ,
'status': <'full' | 'partial'>,
'visit':
},
...
]
Raises:
NotFoundExc if the origin is not found
"""
cache_entry_id = 'origin_%s_visits' % origin_info['id']
cache_entry = cache.get(cache_entry_id)
if cache_entry:
return cache_entry
origin_visits = []
per_page = service.MAX_LIMIT
last_visit = None
while 1:
visits = list(service.lookup_origin_visits(origin_info['id'],
last_visit=last_visit,
per_page=per_page))
origin_visits += visits
if len(visits) < per_page:
break
else:
if not last_visit:
last_visit = per_page
else:
last_visit += per_page
def _visit_sort_key(visit):
ts = parse_timestamp(visit['date']).timestamp()
return ts + (float(visit['visit']) / 10e3)
for v in origin_visits:
if 'metadata' in v:
del v['metadata']
origin_visits = [dict(t) for t in set([tuple(d.items())
for d in origin_visits])]
origin_visits = sorted(origin_visits, key=lambda v: _visit_sort_key(v))
cache.set(cache_entry_id, origin_visits)
return origin_visits
def get_origin_visit(origin_info, visit_ts=None, visit_id=None):
"""Function that returns information about a SWH visit for
a given origin.
The visit is retrieved from a provided timestamp.
The closest visit from that timestamp is selected.
Args:
origin_info (dict): a dict filled with origin information
(id, url, type)
visit_ts (int or str): an ISO date string or Unix timestamp to parse
Returns:
A dict containing the visit info as described below::
{'origin': 2,
'date': '2017-10-08T11:54:25.582463+00:00',
'metadata': {},
'visit': 25,
'status': 'full'}
"""
visits = get_origin_visits(origin_info)
if not visits:
raise NotFoundExc('No SWH visit associated to origin with'
' type %s and url %s!' % (origin_info['type'],
origin_info['url']))
if visit_id:
visit = [v for v in visits if v['visit'] == int(visit_id)]
if len(visit) == 0:
raise NotFoundExc(
'Visit with id %s for origin with type %s'
' and url %s not found!' % (visit_id, origin_info['type'],
origin_info['url']))
return visit[0]
if not visit_ts:
# returns the latest full visit when no timestamp is provided
for v in reversed(visits):
if v['status'] == 'full':
return v
return visits[-1]
parsed_visit_ts = math.floor(parse_timestamp(visit_ts).timestamp())
visit_idx = None
for i, visit in enumerate(visits):
ts = math.floor(parse_timestamp(visit['date']).timestamp())
if i == 0 and parsed_visit_ts <= ts:
return visit
elif i == len(visits) - 1:
if parsed_visit_ts >= ts:
return visit
else:
next_ts = math.floor(
parse_timestamp(visits[i+1]['date']).timestamp())
if parsed_visit_ts >= ts and parsed_visit_ts < next_ts:
if (parsed_visit_ts - ts) < (next_ts - parsed_visit_ts):
visit_idx = i
break
else:
visit_idx = i+1
break
if visit_idx:
visit = visits[visit_idx]
while visit_idx < len(visits) - 1 and \
visit['date'] == visits[visit_idx+1]['date']:
visit_idx = visit_idx + 1
visit = visits[visit_idx]
return visit
else:
raise NotFoundExc(
'Visit with timestamp %s for origin with type %s and url %s not found!' % # noqa
(visit_ts, origin_info['type'], origin_info['url']))
def get_snapshot_content(snapshot_id):
"""Returns the lists of branches and releases
associated to a swh snapshot.
That list is put in cache in order to speedup the navigation
in the swh-web/browse ui.
Args:
snapshot_id (str): hexadecimal representation of the snapshot
identifier
Returns:
A tuple with two members. The first one is a list of dict describing
the snapshot branches. The second one is a list of dict describing the
snapshot releases.
Raises:
NotFoundExc if the snapshot does not exist
"""
cache_entry_id = 'swh_snapshot_%s' % snapshot_id
cache_entry = cache.get(cache_entry_id)
if cache_entry:
return cache_entry['branches'], cache_entry['releases']
branches = []
releases = []
if snapshot_id:
revision_ids = []
releases_ids = []
snapshot = service.lookup_snapshot(snapshot_id)
snapshot_branches = snapshot['branches']
for key in sorted(snapshot_branches.keys()):
if not snapshot_branches[key]:
continue
if snapshot_branches[key]['target_type'] == 'revision':
branches.append({'name': key,
'revision': snapshot_branches[key]['target']})
revision_ids.append(snapshot_branches[key]['target'])
elif snapshot_branches[key]['target_type'] == 'release':
releases_ids.append(snapshot_branches[key]['target'])
releases_info = service.lookup_release_multiple(releases_ids)
for release in releases_info:
releases.append({'name': release['name'],
'date': format_utc_iso_date(release['date']),
'id': release['id'],
'message': release['message'],
'target_type': release['target_type'],
'target': release['target']})
revision_ids.append(release['target'])
revisions = service.lookup_revision_multiple(revision_ids)
branches_to_remove = []
for idx, revision in enumerate(revisions):
if idx < len(branches):
if revision:
branches[idx]['directory'] = revision['directory']
branches[idx]['date'] = format_utc_iso_date(revision['date']) # noqa
branches[idx]['message'] = revision['message']
else:
branches_to_remove.append(branches[idx])
else:
rel_idx = idx - len(branches)
if revision:
releases[rel_idx]['directory'] = revision['directory']
for b in branches_to_remove:
branches.remove(b)
cache.set(cache_entry_id, {'branches': branches, 'releases': releases})
return branches, releases
def get_origin_visit_snapshot(origin_info, visit_ts=None, visit_id=None):
"""Returns the lists of branches and releases
associated to a swh origin for a given visit.
The visit is expressed by a timestamp. In the latter case,
the closest visit from the provided timestamp will be used.
If no visit parameter is provided, it returns the list of branches
found for the latest visit.
That list is put in cache in order to speedup the navigation
in the swh-web/browse ui.
Args:
origin_info (dict): a dict filled with origin information
(id, url, type)
visit_ts (int or str): an ISO date string or Unix timestamp to parse
visit_id (int): optional visit id for desambiguation in case
several visits have the same timestamp
Returns:
A tuple with two members. The first one is a list of dict describing
the origin branches for the given visit.
The second one is a list of dict describing the origin releases
for the given visit.
Raises:
NotFoundExc if the origin or its visit are not found
"""
visit_info = get_origin_visit(origin_info, visit_ts, visit_id)
return get_snapshot_content(visit_info['snapshot'])
def gen_link(url, link_text, link_attrs={}):
"""
Utility function for generating an HTML link to insert
in Django templates.
Args:
url (str): an url
link_text (str): the text for the produced link
link_attrs (dict): optional attributes (e.g. class)
to add to the link
Returns:
An HTML link in the form 'link_text'
"""
attrs = ' '
for k, v in link_attrs.items():
attrs += '%s="%s" ' % (k, v)
link = '%s' % (attrs, url, link_text)
return mark_safe(link)
def gen_person_link(person_id, person_name, link_attrs={}):
"""
Utility function for generating a link to a SWH person HTML view
to insert in Django templates.
Args:
person_id (int): a SWH person id
person_name (str): the associated person name
link_attrs (dict): optional attributes (e.g. class)
to add to the link
Returns:
An HTML link in the form 'person_name'
"""
person_url = reverse('browse-person', kwargs={'person_id': person_id})
return gen_link(person_url, person_name, link_attrs)
def gen_revision_link(revision_id, shorten_id=False, snapshot_context=None,
link_text=None, link_attrs={}):
"""
Utility function for generating a link to a SWH revision HTML view
to insert in Django templates.
Args:
revision_id (str): a SWH revision id
shorten_id (boolean): wheter to shorten the revision id to 7
characters for the link text
snapshot_context (dict): if provided, generate snapshot-dependent
browsing link
link_attrs (dict): optional attributes (e.g. class)
to add to the link
Returns:
An HTML link in the form 'revision_id'
"""
query_params = None
if snapshot_context and snapshot_context['origin_info']:
origin_info = snapshot_context['origin_info']
query_params = {'origin_type': origin_info['type'],
'origin_url': origin_info['url']}
if 'timestamp' in snapshot_context['url_args']:
query_params['timestamp'] = \
snapshot_context['url_args']['timestamp']
if 'visit_id' in snapshot_context['query_params']:
query_params['visit_id'] = \
snapshot_context['query_params']['visit_id']
elif snapshot_context:
query_params = {'snapshot_id': snapshot_context['snapshot_id']}
revision_url = reverse('browse-revision',
kwargs={'sha1_git': revision_id},
query_params=query_params)
if shorten_id:
return gen_link(revision_url, revision_id[:7], link_attrs)
else:
if not link_text:
link_text = revision_id
return gen_link(revision_url, link_text, link_attrs)
def gen_origin_link(origin_info, link_attrs={}):
"""
Utility function for generating a link to a SWH origin HTML view
to insert in Django templates.
Args:
origin_info (dict): a dicted filled with origin information
(id, type, url)
link_attrs (dict): optional attributes (e.g. class)
to add to the link
Returns:
An HTML link in the form 'Origin: origin_url'
""" # noqa
origin_browse_url = reverse('browse-origin',
kwargs={'origin_type': origin_info['type'],
'origin_url': origin_info['url']})
return gen_link(origin_browse_url,
'Origin: ' + origin_info['url'], link_attrs)
def gen_directory_link(sha1_git, link_text=None, link_attrs={}):
"""
Utility function for generating a link to a SWH directory HTML view
to insert in Django templates.
Args:
sha1_git (str): directory identifier
link_text (str): optional text for the generated link
(the generated url will be used by default)
link_attrs (dict): optional attributes (e.g. class)
to add to the link
Returns:
An HTML link in the form 'link_text'
"""
directory_url = reverse('browse-directory',
kwargs={'sha1_git': sha1_git})
if not link_text:
link_text = directory_url
return gen_link(directory_url, link_text, link_attrs)
def gen_snapshot_link(snapshot_id, link_text=None, link_attrs={}):
"""
Utility function for generating a link to a SWH snapshot HTML view
to insert in Django templates.
Args:
snapshot_id (str): snapshot identifier
link_text (str): optional text for the generated link
(the generated url will be used by default)
link_attrs (dict): optional attributes (e.g. class)
to add to the link
Returns:
An HTML link in the form 'link_text'
"""
snapshot_url = reverse('browse-snapshot',
kwargs={'snapshot_id': snapshot_id})
if not link_text:
link_text = snapshot_url
return gen_link(snapshot_url, link_text, link_attrs)
def gen_snapshot_directory_link(snapshot_context, revision_id=None,
link_text=None, link_attrs={}):
"""
Utility function for generating a link to a SWH directory HTML view
in the context of a snapshot to insert in Django templates.
Args:
snapshot_context (dict): the snapshot information
revision_id (str): optional revision identifier in order
to use the associated directory
link_text (str): optional text to use for the generated link
link_attrs (dict): optional attributes (e.g. class)
to add to the link
Returns:
An HTML link in the form
'origin_directory_view_url'
"""
query_params = {'revision': revision_id}
if snapshot_context['origin_info']:
origin_info = snapshot_context['origin_info']
url_args = {'origin_type': origin_info['type'],
'origin_url': origin_info['url']}
if 'timestamp' in snapshot_context['url_args']:
url_args['timestamp'] = \
snapshot_context['url_args']['timestamp']
if 'visit_id' in snapshot_context['query_params']:
query_params['visit_id'] = \
snapshot_context['query_params']['visit_id']
directory_url = reverse('browse-origin-directory',
kwargs=url_args,
query_params=query_params)
else:
url_args = {'snapshot_id': snapshot_context['snapshot_id']}
directory_url = reverse('browse-snapshot-directory',
kwargs=url_args,
query_params=query_params)
if not link_text:
link_text = directory_url
return gen_link(directory_url, link_text, link_attrs)
def gen_content_link(sha1_git, link_text=None, link_attrs={}):
"""
Utility function for generating a link to a SWH content HTML view
to insert in Django templates.
Args:
sha1_git (str): content identifier
link_text (str): optional text for the generated link
(the generated url will be used by default)
link_attrs (dict): optional attributes (e.g. class)
to add to the link
Returns:
An HTML link in the form 'link_text'
"""
content_url = reverse('browse-content',
kwargs={'query_string': 'sha1_git:' + sha1_git})
if not link_text:
link_text = content_url
return gen_link(content_url, link_text, link_attrs)
def get_revision_log_url(revision_id, snapshot_context=None):
"""
Utility function for getting the URL for a SWH revision log HTML view
(possibly in the context of an origin).
Args:
revision_id (str): revision identifier the history heads to
snapshot_context (dict): if provided, generate snapshot-dependent
browsing link
Returns:
The SWH revision log view URL
"""
query_params = {'revision': revision_id}
if snapshot_context and snapshot_context['origin_info']:
origin_info = snapshot_context['origin_info']
url_args = {'origin_type': origin_info['type'],
'origin_url': origin_info['url']}
if 'timestamp' in snapshot_context['url_args']:
url_args['timestamp'] = \
snapshot_context['url_args']['timestamp']
if 'visit_id' in snapshot_context['query_params']:
query_params['visit_id'] = \
snapshot_context['query_params']['visit_id']
revision_log_url = reverse('browse-origin-log',
kwargs=url_args,
query_params=query_params)
elif snapshot_context:
url_args = {'snapshot_id': snapshot_context['snapshot_id']}
revision_log_url = reverse('browse-snapshot-log',
kwargs=url_args,
query_params=query_params)
else:
revision_log_url = reverse('browse-revision-log',
kwargs={'sha1_git': revision_id})
return revision_log_url
def gen_revision_log_link(revision_id, snapshot_context=None, link_text=None,
link_attrs={}):
"""
Utility function for generating a link to a SWH revision log HTML view
(possibly in the context of an origin) to insert in Django templates.
Args:
revision_id (str): revision identifier the history heads to
snapshot_context (dict): if provided, generate snapshot-dependent
browsing link
link_text (str): optional text to use for the generated link
link_attrs (dict): optional attributes (e.g. class)
to add to the link
Returns:
An HTML link in the form
'link_text'
"""
revision_log_url = get_revision_log_url(revision_id, snapshot_context)
if not link_text:
link_text = revision_log_url
return gen_link(revision_log_url, link_text, link_attrs)
def _format_log_entries(revision_log, per_page, snapshot_context=None):
revision_log_data = []
for i, log in enumerate(revision_log):
if i == per_page:
break
revision_log_data.append(
{'author': gen_person_link(log['author']['id'],
log['author']['name']),
'revision': gen_revision_link(log['id'], True, snapshot_context),
'message': log['message'],
'date': format_utc_iso_date(log['date']),
'directory': log['directory']})
return revision_log_data
def prepare_revision_log_for_display(revision_log, per_page, revs_breadcrumb,
snapshot_context=None):
"""
Utility functions that process raw revision log data for HTML display.
Its purpose is to:
* add links to relevant SWH browse views
* format date in human readable format
* truncate the message log
It also computes the data needed to generate the links for navigating back
and forth in the history log.
Args:
revision_log (list): raw revision log as returned by the SWH web api
per_page (int): number of log entries per page
revs_breadcrumb (str): breadcrumbs of revisions navigated so far,
in the form 'rev1[/rev2/../revN]'. Each revision corresponds to
the first one displayed in the HTML view for history log.
snapshot_context (dict): if provided, generate snapshot-dependent
browsing link
"""
current_rev = revision_log[0]['id']
next_rev = None
prev_rev = None
next_revs_breadcrumb = None
prev_revs_breadcrumb = None
if len(revision_log) == per_page + 1:
prev_rev = revision_log[-1]['id']
prev_rev_bc = current_rev
if snapshot_context:
prev_rev_bc = prev_rev
if revs_breadcrumb:
revs = revs_breadcrumb.split('/')
next_rev = revs[-1]
if len(revs) > 1:
next_revs_breadcrumb = '/'.join(revs[:-1])
if len(revision_log) == per_page + 1:
prev_revs_breadcrumb = revs_breadcrumb + '/' + prev_rev_bc
else:
prev_revs_breadcrumb = prev_rev_bc
return {'revision_log_data': _format_log_entries(revision_log, per_page,
snapshot_context),
'prev_rev': prev_rev,
'prev_revs_breadcrumb': prev_revs_breadcrumb,
'next_rev': next_rev,
'next_revs_breadcrumb': next_revs_breadcrumb}
def get_snapshot_context(snapshot_id=None, origin_type=None, origin_url=None,
timestamp=None, visit_id=None):
"""
Utility function to compute relevant information when navigating
the SWH archive in a snapshot context. The snapshot is either
referenced by its id or it will be retrieved from an origin visit.
Args:
snapshot_id (str): hexadecimal representation of a snapshot identifier,
all other parameters will be ignored if it is provided
origin_type (str): the origin type (git, svn, deposit, ...)
origin_url (str): the origin_url (e.g. https://github.com/(user)/(repo)/)
timestamp (str): a datetime string for retrieving the closest
SWH visit of the origin
visit_id (int): optional visit id for disambiguation in case
of several visits with the same timestamp
Returns:
A dict with the following entries:
* origin_info: dict containing origin information
* visit_info: dict containing SWH visit information
* branches: the list of branches for the origin found
during the visit
* releases: the list of releases for the origin found
during the visit
* origin_browse_url: the url to browse the origin
* origin_branches_url: the url to browse the origin branches
* origin_releases_url': the url to browse the origin releases
* origin_visit_url: the url to browse the snapshot of the origin
found during the visit
* url_args: dict containg url arguments to use when browsing in
the context of the origin and its visit
""" # noqa
origin_info = None
visit_info = None
url_args = None
query_params = {}
branches = []
releases = []
browse_url = None
visit_url = None
branches_url = None
releases_url = None
swh_type = 'snapshot'
if origin_type and origin_url:
swh_type = 'origin'
origin_info = service.lookup_origin({'type': origin_type,
'url': origin_url})
visit_info = get_origin_visit(origin_info, timestamp, visit_id)
visit_info['fmt_date'] = format_utc_iso_date(visit_info['date'])
snapshot_id = visit_info['snapshot']
# provided timestamp is not necessarily equals to the one
# of the retrieved visit, so get the exact one in order
# use it in the urls generated below
if timestamp:
timestamp = visit_info['date']
branches, releases = \
get_origin_visit_snapshot(origin_info, timestamp, visit_id)
url_args = {'origin_type': origin_info['type'],
'origin_url': origin_info['url']}
query_params = {'visit_id': visit_id}
browse_url = reverse('browse-origin',
kwargs=url_args)
if timestamp:
url_args['timestamp'] = format_utc_iso_date(timestamp,
'%Y-%m-%dT%H:%M:%S')
visit_url = reverse('browse-origin-directory',
kwargs=url_args,
query_params=query_params)
visit_info['url'] = visit_url
branches_url = reverse('browse-origin-branches',
kwargs=url_args,
query_params=query_params)
releases_url = reverse('browse-origin-releases',
kwargs=url_args,
query_params=query_params)
elif snapshot_id:
branches, releases = get_snapshot_content(snapshot_id)
url_args = {'snapshot_id': snapshot_id}
browse_url = reverse('browse-snapshot',
kwargs=url_args)
branches_url = reverse('browse-snapshot-branches',
kwargs=url_args)
releases_url = reverse('browse-snapshot-releases',
kwargs=url_args)
releases = list(reversed(releases))
return {
'swh_type': swh_type,
'snapshot_id': snapshot_id,
'origin_info': origin_info,
'visit_info': visit_info,
'branches': branches,
'releases': releases,
'branch': None,
'release': None,
'browse_url': browse_url,
'branches_url': branches_url,
'releases_url': releases_url,
'url_args': url_args,
'query_params': query_params
}
diff --git a/swh/web/browse/views/content.py b/swh/web/browse/views/content.py
index 2e691d3fc..36c547b55 100644
--- a/swh/web/browse/views/content.py
+++ b/swh/web/browse/views/content.py
@@ -1,244 +1,244 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import difflib
import json
from distutils.util import strtobool
from django.http import HttpResponse
from django.utils.safestring import mark_safe
from django.shortcuts import render
from django.template.defaultfilters import filesizeformat
from swh.model.hashutil import hash_to_hex
from swh.web.common import query
from swh.web.common.utils import reverse, gen_path_info
from swh.web.common.exc import handle_view_exception
from swh.web.browse.utils import (
request_content, prepare_content_for_display,
content_display_max_size
)
from swh.web.browse.browseurls import browse_route
@browse_route(r'content/(?P.+)/raw/',
view_name='browse-content-raw')
def content_raw(request, query_string):
"""Django view that produces a raw display of a SWH content identified
by its hash value.
The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/raw/`
""" # noqa
try:
algo, checksum = query.parse_hash(query_string)
checksum = hash_to_hex(checksum)
content_data = request_content(query_string, max_size=None)
except Exception as exc:
return handle_view_exception(request, exc)
filename = request.GET.get('filename', None)
if not filename:
filename = '%s_%s' % (algo, checksum)
if content_data['mimetype'].startswith('text/') or \
content_data['mimetype'] == 'inode/x-empty':
response = HttpResponse(content_data['raw_data'],
content_type="text/plain")
response['Content-disposition'] = 'filename=%s' % filename
else:
response = HttpResponse(content_data['raw_data'],
content_type='application/octet-stream')
response['Content-disposition'] = 'attachment; filename=%s' % filename
return response
_auto_diff_size_limit = 20000
@browse_route(r'content/(?P.*)/diff/(?P.*)', # noqa
view_name='diff-contents')
def _contents_diff(request, from_query_string, to_query_string):
"""
Browse endpoint used to compute unified diffs between two contents.
Diffs are generated only if the two contents are textual.
By default, diffs whose size are greater than 20 kB will
not be generated. To force the generation of large diffs,
the 'force' boolean query parameter must be used.
Args:
request: input django http request
from_query_string: a string of the form "[ALGO_HASH:]HASH" where
optional ALGO_HASH can be either *sha1*, *sha1_git*, *sha256*,
or *blake2s256* (default to *sha1*) and HASH the hexadecimal
representation of the hash value identifying the first content
to_query_string: same as above for identifying the second content
Returns:
A JSON object containing the unified diff.
"""
diff_data = {}
content_from = None
content_to = None
content_from_size = 0
content_to_size = 0
content_from_lines = []
content_to_lines = []
force = request.GET.get('force', 'false')
path = request.GET.get('path', None)
language = 'nohighlight-swh'
force = bool(strtobool(force))
if from_query_string == to_query_string:
diff_str = 'File renamed without changes'
else:
text_diff = True
if from_query_string:
content_from = request_content(from_query_string, max_size=None)
content_from_display_data = prepare_content_for_display(
content_from['raw_data'], content_from['mimetype'], path)
language = content_from_display_data['language']
content_from_size = content_from['length']
if not (content_from['mimetype'].startswith('text/') or
content_from['mimetype'] == 'inode/x-empty'):
text_diff = False
if text_diff and to_query_string:
content_to = request_content(to_query_string, max_size=None)
content_to_display_data = prepare_content_for_display(
content_to['raw_data'], content_to['mimetype'], path)
language = content_to_display_data['language']
content_to_size = content_to['length']
if not (content_to['mimetype'].startswith('text/') or
content_to['mimetype'] == 'inode/x-empty'):
text_diff = False
diff_size = abs(content_to_size - content_from_size)
if not text_diff:
diff_str = 'Diffs are not generated for non textual content'
language = 'nohighlight-swh'
elif not force and diff_size > _auto_diff_size_limit:
diff_str = 'Large diffs are not automatically computed'
language = 'nohighlight-swh'
else:
if content_from:
content_from_lines = content_from['raw_data'].decode('utf-8')\
.splitlines(True)
if content_from_lines and content_from_lines[-1][-1] != '\n':
content_from_lines[-1] += '[swh-no-nl-marker]\n'
if content_to:
content_to_lines = content_to['raw_data'].decode('utf-8')\
.splitlines(True)
if content_to_lines and content_to_lines[-1][-1] != '\n':
content_to_lines[-1] += '[swh-no-nl-marker]\n'
diff_lines = difflib.unified_diff(content_from_lines,
content_to_lines)
diff_str = ''.join(list(diff_lines)[2:])
diff_data['diff_str'] = diff_str
diff_data['language'] = language
diff_data_json = json.dumps(diff_data, separators=(',', ': '))
return HttpResponse(diff_data_json, content_type='application/json')
@browse_route(r'content/(?P.+)/',
view_name='browse-content')
def content_display(request, query_string):
"""Django view that produces an HTML display of a SWH content identified
by its hash value.
The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/`
""" # noqa
try:
algo, checksum = query.parse_hash(query_string)
checksum = hash_to_hex(checksum)
content_data = request_content(query_string)
except Exception as exc:
return handle_view_exception(request, exc)
path = request.GET.get('path', None)
content = None
language = None
if content_data['raw_data'] is not None:
content_display_data = prepare_content_for_display(
content_data['raw_data'], content_data['mimetype'], path)
content = content_display_data['content_data']
language = content_display_data['language']
root_dir = None
filename = None
path_info = None
breadcrumbs = []
if path:
split_path = path.split('/')
root_dir = split_path[0]
filename = split_path[-1]
path = path.replace(root_dir + '/', '')
path = path[:-len(filename)]
path_info = gen_path_info(path)
breadcrumbs.append({'name': root_dir[:7],
'url': reverse('browse-directory',
kwargs={'sha1_git': root_dir})})
for pi in path_info:
breadcrumbs.append({'name': pi['name'],
'url': reverse('browse-directory',
kwargs={'sha1_git': root_dir,
'path': pi['path']})})
breadcrumbs.append({'name': filename,
'url': None})
query_params = None
if filename:
query_params = {'filename': filename}
content_raw_url = reverse('browse-content-raw',
kwargs={'query_string': query_string},
query_params=query_params)
content_metadata = {
'sha1 checksum': content_data['checksums']['sha1'],
'sha1_git checksum': content_data['checksums']['sha1_git'],
'sha256 checksum': content_data['checksums']['sha256'],
'blake2s256 checksum': content_data['checksums']['blake2s256'],
'mime type': content_data['mimetype'],
'encoding': content_data['encoding'],
'size': filesizeformat(content_data['length']),
'language': content_data['language'],
'licenses': content_data['licenses']
}
return render(request, 'content.html',
{'empty_browse': False,
'heading': 'Content information',
'top_panel_visible': True,
'top_panel_collapsible': True,
'top_panel_text': 'Content metadata',
'swh_object_metadata': content_metadata,
'main_panel_visible': True,
'content': content,
'content_size': content_data['length'],
'max_content_size': content_display_max_size,
'mimetype': content_data['mimetype'],
'language': language,
'breadcrumbs': breadcrumbs,
'top_right_link': content_raw_url,
'top_right_link_text': mark_safe(
''
'Raw File'),
'snapshot_context': None,
'vault_cooking': None,
'show_actions_menu': False
})
diff --git a/swh/web/browse/views/directory.py b/swh/web/browse/views/directory.py
index 4335e207f..48340e481 100644
--- a/swh/web/browse/views/directory.py
+++ b/swh/web/browse/views/directory.py
@@ -1,106 +1,106 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.shortcuts import render
from django.template.defaultfilters import filesizeformat
from swh.web.common import service
from swh.web.common.utils import reverse, gen_path_info
from swh.web.common.exc import handle_view_exception
from swh.web.browse.utils import get_directory_entries
from swh.web.browse.browseurls import browse_route
@browse_route(r'directory/(?P[0-9a-f]+)/',
r'directory/(?P[0-9a-f]+)/(?P.+)/',
view_name='browse-directory')
def directory_browse(request, sha1_git, path=None):
"""Django view for browsing the content of a SWH directory identified
by its sha1_git value.
The url that points to it is :http:get:`/browse/directory/(sha1_git)/[(path)/]`
""" # noqa
root_sha1_git = sha1_git
try:
if path:
dir_info = service.lookup_directory_with_path(sha1_git, path)
sha1_git = dir_info['target']
dirs, files = get_directory_entries(sha1_git)
except Exception as exc:
return handle_view_exception(request, exc)
path_info = gen_path_info(path)
breadcrumbs = []
breadcrumbs.append({'name': root_sha1_git[:7],
'url': reverse('browse-directory',
kwargs={'sha1_git': root_sha1_git})})
for pi in path_info:
breadcrumbs.append({'name': pi['name'],
'url': reverse('browse-directory',
kwargs={'sha1_git': root_sha1_git,
'path': pi['path']})})
path = '' if path is None else (path + '/')
for d in dirs:
d['url'] = reverse('browse-directory',
kwargs={'sha1_git': root_sha1_git,
'path': path + d['name']})
sum_file_sizes = 0
readme_name = None
readme_url = None
for f in files:
query_string = 'sha1_git:' + f['target']
f['url'] = reverse('browse-content',
kwargs={'query_string': query_string},
query_params={'path': root_sha1_git + '/' +
path + f['name']})
sum_file_sizes += f['length']
f['length'] = filesizeformat(f['length'])
if f['name'].lower().startswith('readme'):
readme_name = f['name']
readme_sha1 = f['checksums']['sha1']
readme_url = reverse('browse-content-raw',
kwargs={'query_string': readme_sha1})
sum_file_sizes = filesizeformat(sum_file_sizes)
dir_metadata = {'id': sha1_git,
'number of regular files': len(files),
'number of subdirectories': len(dirs),
'sum of regular file sizes': sum_file_sizes}
vault_cooking = {
'directory_context': True,
'directory_id': sha1_git,
'revision_context': False,
'revision_id': None
}
return render(request, 'directory.html',
{'empty_browse': False,
'heading': 'Directory information',
'top_panel_visible': True,
'top_panel_collapsible': True,
'top_panel_text': 'Directory metadata',
'swh_object_metadata': dir_metadata,
'main_panel_visible': True,
'dirs': dirs,
'files': files,
'breadcrumbs': breadcrumbs,
'top_right_link': None,
'top_right_link_text': None,
'readme_name': readme_name,
'readme_url': readme_url,
'snapshot_context': None,
'vault_cooking': vault_cooking,
'show_actions_menu': True})
diff --git a/swh/web/browse/views/identifiers.py b/swh/web/browse/views/identifiers.py
index 31f3a08c7..82543566e 100644
--- a/swh/web/browse/views/identifiers.py
+++ b/swh/web/browse/views/identifiers.py
@@ -1,56 +1,56 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.shortcuts import redirect
from swh.model.identifiers import parse_persistent_identifier
from swh.web.browse.browseurls import browse_route
from swh.web.common.utils import reverse
from swh.web.common.exc import BadInputExc, handle_view_exception
@browse_route(r'(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+)/',
view_name='browse-swh-id')
def swh_id_browse(request, swh_id):
"""
Django view enabling to browse the SWH archive using
:ref:`persistent-identifiers`.
The url that points to it is :http:get:`/browse/(swh_id)/`.
"""
try:
swh_id_parsed = parse_persistent_identifier(swh_id)
object_type = swh_id_parsed['object_type']
object_id = swh_id_parsed['object_id']
view_url = None
if object_type == 'cnt':
query_string = 'sha1_git:' + object_id
view_url = reverse('browse-content',
kwargs={'query_string': query_string},
query_params=request.GET)
elif object_type == 'dir':
view_url = reverse('browse-directory',
kwargs={'sha1_git': object_id},
query_params=request.GET)
elif object_type == 'rel':
view_url = reverse('browse-release',
kwargs={'sha1_git': object_id},
query_params=request.GET)
elif object_type == 'rev':
view_url = reverse('browse-revision',
kwargs={'sha1_git': object_id},
query_params=request.GET)
elif object_type == 'snp':
view_url = reverse('browse-snapshot',
kwargs={'snapshot_id': object_id},
query_params=request.GET)
else:
msg = '\'%s\' is not a valid SWH persistent identifier!' % swh_id
raise BadInputExc(msg)
except Exception as exc:
return handle_view_exception(request, exc)
return redirect(view_url)
diff --git a/swh/web/browse/views/origin.py b/swh/web/browse/views/origin.py
index 6e8b4fe37..646944faa 100644
--- a/swh/web/browse/views/origin.py
+++ b/swh/web/browse/views/origin.py
@@ -1,211 +1,211 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import json
from distutils.util import strtobool
from django.http import HttpResponse
from django.shortcuts import render
from swh.web.common import service
from swh.web.common.utils import (
reverse, format_utc_iso_date, parse_timestamp
)
from swh.web.common.exc import handle_view_exception
from swh.web.browse.utils import get_origin_visits
from swh.web.browse.browseurls import browse_route
from .utils.snapshot_context import (
browse_snapshot_directory, browse_snapshot_content,
browse_snapshot_log, browse_snapshot_branches,
browse_snapshot_releases
)
@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/directory/', # noqa
r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/directory/(?P.+)/', # noqa
r'origin/(?P[a-z]+)/url/(?P.+)/directory/', # noqa
r'origin/(?P[a-z]+)/url/(?P.+)/directory/(?P.+)/', # noqa
view_name='browse-origin-directory')
def origin_directory_browse(request, origin_type, origin_url,
timestamp=None, path=None):
"""Django view for browsing the content of a SWH directory associated
to an origin for a given visit.
The url scheme that points to it is the following:
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/directory/[(path)/]`
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/visit/(timestamp)/directory/[(path)/]`
""" # noqa
return browse_snapshot_directory(
request, origin_type=origin_type, origin_url=origin_url,
timestamp=timestamp, path=path)
@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/content/(?P.+)/', # noqa
r'origin/(?P[a-z]+)/url/(?P.+)/content/(?P.+)/', # noqa
view_name='browse-origin-content')
def origin_content_browse(request, origin_type, origin_url, path,
timestamp=None):
"""Django view that produces an HTML display of a SWH content
associated to an origin for a given visit.
The url scheme that points to it is the following:
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/content/(path)/`
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/visit/(timestamp)/content/(path)/`
""" # noqa
return browse_snapshot_content(request, origin_type=origin_type,
origin_url=origin_url, timestamp=timestamp,
path=path)
PER_PAGE = 20
@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/log/', # noqa
r'origin/(?P[a-z]+)/url/(?P.+)/log/',
view_name='browse-origin-log')
def origin_log_browse(request, origin_type, origin_url, timestamp=None):
"""Django view that produces an HTML display of revisions history (aka
the commit log) associated to a SWH origin.
The url scheme that points to it is the following:
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/log/`
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/visit/(timestamp)/log/`
""" # noqa
return browse_snapshot_log(request, origin_type=origin_type,
origin_url=origin_url, timestamp=timestamp)
@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/branches/', # noqa
r'origin/(?P[a-z]+)/url/(?P.+)/branches/', # noqa
view_name='browse-origin-branches')
def origin_branches_browse(request, origin_type, origin_url, timestamp=None):
"""Django view that produces an HTML display of the list of branches
associated to an origin for a given visit.
The url scheme that points to it is the following:
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/branches/`
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/visit/(timestamp)/branches/`
""" # noqa
return browse_snapshot_branches(request, origin_type=origin_type,
origin_url=origin_url, timestamp=timestamp)
@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/visit/(?P.+)/releases/', # noqa
r'origin/(?P[a-z]+)/url/(?P.+)/releases/', # noqa
view_name='browse-origin-releases')
def origin_releases_browse(request, origin_type, origin_url, timestamp=None):
"""Django view that produces an HTML display of the list of releases
associated to an origin for a given visit.
The url scheme that points to it is the following:
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/releases/`
* :http:get:`/browse/origin/(origin_type)/url/(origin_url)/visit/(timestamp)/releases/`
""" # noqa
return browse_snapshot_releases(request, origin_type=origin_type,
origin_url=origin_url, timestamp=timestamp)
@browse_route(r'origin/(?P[a-z]+)/url/(?P.+)/',
view_name='browse-origin')
def origin_browse(request, origin_type=None, origin_url=None):
"""Django view that produces an HTML display of a swh origin identified
by its id or its url.
The url that points to it is :http:get:`/browse/origin/(origin_type)/url/(origin_url)/`.
""" # noqa
try:
origin_info = service.lookup_origin({
'type': origin_type,
'url': origin_url
})
origin_visits = get_origin_visits(origin_info)
except Exception as exc:
return handle_view_exception(request, exc)
origin_info['last swh visit browse url'] = \
reverse('browse-origin-directory',
kwargs={'origin_type': origin_type,
'origin_url': origin_url})
for i, visit in enumerate(origin_visits):
url_date = format_utc_iso_date(visit['date'], '%Y-%m-%dT%H:%M:%SZ')
visit['fmt_date'] = format_utc_iso_date(visit['date'])
query_params = {}
if i < len(origin_visits) - 1:
if visit['date'] == origin_visits[i+1]['date']:
query_params = {'visit_id': visit['visit']}
if i > 0:
if visit['date'] == origin_visits[i-1]['date']:
query_params = {'visit_id': visit['visit']}
snapshot = visit['snapshot'] if visit['snapshot'] else ''
visit['browse_url'] = reverse('browse-origin-directory',
kwargs={'origin_type': origin_type,
'origin_url': origin_url,
'timestamp': url_date},
query_params=query_params)
if not snapshot:
visit['snapshot'] = ''
visit['date'] = parse_timestamp(visit['date']).timestamp()
return render(request, 'origin.html',
{'empty_browse': False,
'heading': 'Origin information',
'top_panel_visible': False,
'top_panel_collapsible': False,
'top_panel_text': 'Origin visits',
'swh_object_metadata': origin_info,
'main_panel_visible': True,
'origin_visits': origin_visits,
'origin_info': origin_info,
'browse_url_base': '/browse/origin/%s/url/%s/' %
(origin_type, origin_url),
'vault_cooking': None,
'show_actions_menu': False})
@browse_route(r'origin/search/(?P.+)/',
view_name='browse-origin-search')
def _origin_search(request, url_pattern):
"""Internal browse endpoint to search for origins whose urls contain
a provided string pattern or match a provided regular expression.
The search is performed in a case insensitive way.
"""
offset = int(request.GET.get('offset', '0'))
limit = int(request.GET.get('limit', '50'))
regexp = request.GET.get('regexp', 'false')
results = service.search_origin(url_pattern, offset, limit,
bool(strtobool(regexp)))
results = json.dumps(list(results), sort_keys=True, indent=4,
separators=(',', ': '))
return HttpResponse(results, content_type='application/json')
@browse_route(r'origin/(?P[0-9]+)/latest_snapshot/',
view_name='browse-origin-latest-snapshot')
def _origin_latest_snapshot(request, origin_id):
"""
Internal browse endpoint used to check if an origin has already
been visited by Software Heritage and has at least one full visit.
"""
result = service.lookup_latest_origin_snapshot(origin_id,
allowed_statuses=['full'])
result = json.dumps(result, sort_keys=True, indent=4,
separators=(',', ': '))
return HttpResponse(result, content_type='application/json')
diff --git a/swh/web/browse/views/person.py b/swh/web/browse/views/person.py
index d379bda6f..eec6989fd 100644
--- a/swh/web/browse/views/person.py
+++ b/swh/web/browse/views/person.py
@@ -1,36 +1,36 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.shortcuts import render
from swh.web.common import service
from swh.web.common.exc import handle_view_exception
from swh.web.browse.browseurls import browse_route
@browse_route(r'person/(?P[0-9]+)/',
view_name='browse-person')
def person_browse(request, person_id):
"""
Django view that produces an HTML display of a swh person
identified by its id.
The url that points to it is :http:get:`/browse/person/(person_id)/`.
"""
try:
person = service.lookup_person(person_id)
except Exception as exc:
return handle_view_exception(request, exc)
return render(request, 'person.html',
{'empty_browse': False,
'heading': 'Person information',
'top_panel_visible': True,
'top_panel_collapsible': False,
'top_panel_text': 'Person metadata',
'swh_object_metadata': person,
'main_panel_visible': False,
'vault_cooking': None,
'show_actions_menu': False})
diff --git a/swh/web/browse/views/release.py b/swh/web/browse/views/release.py
index 79d0ca52e..51bbf8838 100644
--- a/swh/web/browse/views/release.py
+++ b/swh/web/browse/views/release.py
@@ -1,94 +1,94 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.shortcuts import render
from django.utils.safestring import mark_safe
from swh.web.common import service
from swh.web.common.utils import reverse, format_utc_iso_date
from swh.web.common.exc import handle_view_exception
from swh.web.browse.browseurls import browse_route
from swh.web.browse.utils import (
gen_person_link, gen_revision_link,
get_snapshot_context, gen_link
)
@browse_route(r'release/(?P[0-9a-f]+)/',
view_name='browse-release')
def release_browse(request, sha1_git):
"""
Django view that produces an HTML display of a SWH release
identified by its id.
The url that points to it is :http:get:`/browse/release/(sha1_git)/`.
"""
try:
release = service.lookup_release(sha1_git)
snapshot_context = None
snapshot_id = request.GET.get('snapshot_id', None)
origin_type = request.GET.get('origin_type', None)
origin_url = request.GET.get('origin_url', None)
timestamp = request.GET.get('timestamp', None)
visit_id = request.GET.get('visit_id', None)
if origin_type and origin_url:
snapshot_context = get_snapshot_context(snapshot_id, origin_type,
origin_url, timestamp,
visit_id)
except Exception as exc:
return handle_view_exception(request, exc)
release_data = {}
release_data['author'] = gen_person_link(
release['author']['id'], release['author']['name'])
release_data['date'] = format_utc_iso_date(release['date'])
release_data['id'] = sha1_git
release_data['name'] = release['name']
release_data['synthetic'] = release['synthetic']
release_data['target type'] = release['target_type']
if release['target_type'] == 'revision':
release_data['target'] = \
gen_revision_link(release['target'],
snapshot_context=snapshot_context)
elif release['target_type'] == 'content':
content_url = \
reverse('browse-content',
kwargs={'query_string': 'sha1_git:' + release['target']})
release_data['target'] = gen_link(content_url, release['target'])
elif release['target_type'] == 'directory':
directory_url = \
reverse('browse-directory',
kwargs={'sha1_git': release['target']})
release_data['target'] = gen_link(directory_url, release['target'])
elif release['target_type'] == 'release':
release_url = \
reverse('browse-release',
kwargs={'sha1_git': release['target']})
release_data['target'] = gen_link(release_url, release['target'])
release_note_lines = release['message'].split('\n')
release_target_link = 'Target: '
if release['target_type'] == 'revision':
release_target_link += '' # noqa
else:
release_target_link += release['target_type']
release_target_link += ' ' + release_data['target']
return render(request, 'release.html',
{'empty_browse': False,
'heading': 'Release information',
'top_panel_visible': True,
'top_panel_collapsible': True,
'top_panel_text': 'Release metadata',
'swh_object_metadata': release_data,
'main_panel_visible': True,
'release_name': release['name'],
'release_note_header': release_note_lines[0],
'release_note_body': '\n'.join(release_note_lines[1:]),
'release_target_link': mark_safe(release_target_link),
'snapshot_context': snapshot_context})
diff --git a/swh/web/browse/views/revision.py b/swh/web/browse/views/revision.py
index 2ff1923cb..bdb818a24 100644
--- a/swh/web/browse/views/revision.py
+++ b/swh/web/browse/views/revision.py
@@ -1,462 +1,462 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import hashlib
import json
from django.http import HttpResponse
from django.shortcuts import render
from django.template.defaultfilters import filesizeformat
from django.utils.safestring import mark_safe
from swh.web.common import service
from swh.web.common.utils import reverse, format_utc_iso_date, gen_path_info
from swh.web.common.exc import handle_view_exception
from swh.web.browse.browseurls import browse_route
from swh.web.browse.utils import (
gen_link, gen_person_link, gen_revision_link,
prepare_revision_log_for_display,
get_snapshot_context, gen_snapshot_directory_link,
get_revision_log_url, get_directory_entries,
gen_directory_link, request_content, prepare_content_for_display,
content_display_max_size, gen_snapshot_link
)
def _gen_content_url(revision, query_string, path, snapshot_context):
if snapshot_context:
url_args = snapshot_context['url_args']
url_args['path'] = path
query_params = snapshot_context['query_params']
query_params['revision'] = revision['id']
content_url = reverse('browse-origin-content',
kwargs=url_args,
query_params=query_params)
else:
content_path = '%s/%s' % (revision['directory'], path)
content_url = reverse('browse-content',
kwargs={'query_string': query_string},
query_params={'path': content_path})
return content_url
def _gen_diff_link(idx, diff_anchor, link_text):
if idx < _max_displayed_file_diffs:
return gen_link(diff_anchor, link_text)
else:
return link_text
# TODO: put in conf
_max_displayed_file_diffs = 1000
def _gen_revision_changes_list(revision, changes, snapshot_context):
"""
Returns a HTML string describing the file changes
introduced in a revision.
As this string will be displayed in the browse revision view,
links to adequate file diffs are also generated.
Args:
revision (str): hexadecimal representation of a revision identifier
changes (list): list of file changes in the revision
snapshot_context (dict): optional origin context used to reverse
the content urls
Returns:
A string to insert in a revision HTML view.
"""
changes_msg = []
for i, change in enumerate(changes):
hasher = hashlib.sha1()
from_query_string = ''
to_query_string = ''
diff_id = 'diff-'
if change['from']:
from_query_string = 'sha1_git:' + change['from']['target']
diff_id += change['from']['target'] + '-' + change['from_path']
diff_id += '-'
if change['to']:
to_query_string = 'sha1_git:' + change['to']['target']
diff_id += change['to']['target'] + change['to_path']
change['path'] = change['to_path'] or change['from_path']
url_args = {'from_query_string': from_query_string,
'to_query_string': to_query_string}
query_params = {'path': change['path']}
change['diff_url'] = reverse('diff-contents',
kwargs=url_args,
query_params=query_params)
hasher.update(diff_id.encode('utf-8'))
diff_id = hasher.hexdigest()
change['id'] = diff_id
panel_diff_link = '#panel_' + diff_id
if change['type'] == 'modify':
change['content_url'] = \
_gen_content_url(revision, to_query_string,
change['to_path'], snapshot_context)
changes_msg.append('modified: %s' %
_gen_diff_link(i, panel_diff_link,
change['to_path']))
elif change['type'] == 'insert':
change['content_url'] = \
_gen_content_url(revision, to_query_string,
change['to_path'], snapshot_context)
changes_msg.append('new file: %s' %
_gen_diff_link(i, panel_diff_link,
change['to_path']))
elif change['type'] == 'delete':
parent = service.lookup_revision(revision['parents'][0])
change['content_url'] = \
_gen_content_url(parent,
from_query_string,
change['from_path'], snapshot_context)
changes_msg.append('deleted: %s' %
_gen_diff_link(i, panel_diff_link,
change['from_path']))
elif change['type'] == 'rename':
change['content_url'] = \
_gen_content_url(revision, to_query_string,
change['to_path'], snapshot_context)
link_text = change['from_path'] + ' → ' + change['to_path']
changes_msg.append('renamed: %s' %
_gen_diff_link(i, panel_diff_link, link_text))
if not changes:
changes_msg.append('No changes')
return mark_safe('\n'.join(changes_msg))
@browse_route(r'revision/(?P[0-9a-f]+)/diff/',
view_name='diff-revision')
def _revision_diff(request, sha1_git):
"""
Browse internal endpoint to compute revision diff
"""
try:
revision = service.lookup_revision(sha1_git)
snapshot_context = None
origin_type = request.GET.get('origin_type', None)
origin_url = request.GET.get('origin_url', None)
timestamp = request.GET.get('timestamp', None)
visit_id = request.GET.get('visit_id', None)
if origin_type and origin_url:
snapshot_context = get_snapshot_context(None, origin_type,
origin_url,
timestamp, visit_id)
except Exception as exc:
return handle_view_exception(request, exc)
changes = service.diff_revision(sha1_git)
changes_msg = _gen_revision_changes_list(revision, changes,
snapshot_context)
diff_data = {
'total_nb_changes': len(changes),
'changes': changes[:_max_displayed_file_diffs],
'changes_msg': changes_msg
}
diff_data_json = json.dumps(diff_data, separators=(',', ': '))
return HttpResponse(diff_data_json, content_type='application/json')
@browse_route(r'revision/(?P[0-9a-f]+)/',
view_name='browse-revision')
def revision_browse(request, sha1_git):
"""
Django view that produces an HTML display of a SWH revision
identified by its id.
The url that points to it is :http:get:`/browse/revision/(sha1_git)/`.
"""
try:
revision = service.lookup_revision(sha1_git)
origin_info = None
snapshot_context = None
origin_type = request.GET.get('origin_type', None)
origin_url = request.GET.get('origin_url', None)
timestamp = request.GET.get('timestamp', None)
visit_id = request.GET.get('visit_id', None)
snapshot_id = request.GET.get('snapshot_id', None)
path = request.GET.get('path', None)
dir_id = None
dirs, files = None, None
content_data = None
if origin_type and origin_url:
snapshot_context = get_snapshot_context(None, origin_type,
origin_url,
timestamp, visit_id)
origin_info = snapshot_context['origin_info']
snapshot_id = snapshot_context['snapshot_id']
elif snapshot_id:
snapshot_context = get_snapshot_context(snapshot_id)
if path:
path_info = \
service.lookup_directory_with_path(revision['directory'], path)
if path_info['type'] == 'dir':
dir_id = path_info['target']
else:
query_string = 'sha1_git:' + path_info['target']
content_data = request_content(query_string)
else:
dir_id = revision['directory']
if dir_id:
path = '' if path is None else (path + '/')
dirs, files = get_directory_entries(dir_id)
except Exception as exc:
return handle_view_exception(request, exc)
revision_data = {}
revision_data['author'] = gen_person_link(
revision['author']['id'], revision['author']['name'])
revision_data['committer'] = gen_person_link(
revision['committer']['id'], revision['committer']['name'])
revision_data['committer date'] = format_utc_iso_date(
revision['committer_date'])
revision_data['date'] = format_utc_iso_date(revision['date'])
if snapshot_context:
revision_data['snapshot id'] = snapshot_id
revision_data['directory'] = \
gen_snapshot_directory_link(snapshot_context, sha1_git,
link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm', # noqa
'role': 'button'})
else:
revision_data['directory'] = \
gen_directory_link(revision['directory'], link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
revision_data['id'] = sha1_git
revision_data['merge'] = revision['merge']
revision_data['metadata'] = json.dumps(revision['metadata'],
sort_keys=True,
indent=4, separators=(',', ': '))
if origin_info:
revision_data['context-independent revision'] = \
gen_revision_link(sha1_git, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
revision_data['origin id'] = origin_info['id']
revision_data['origin type'] = origin_info['type']
revision_data['origin url'] = gen_link(origin_info['url'],
origin_info['url'])
browse_snapshot_link = \
gen_snapshot_link(snapshot_id, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
revision_data['snapshot'] = browse_snapshot_link
parents = ''
for p in revision['parents']:
parent_link = gen_revision_link(p, snapshot_context=snapshot_context)
parents += parent_link + ' '
revision_data['parents'] = mark_safe(parents)
revision_data['synthetic'] = revision['synthetic']
revision_data['type'] = revision['type']
message_lines = revision['message'].split('\n')
parents_links = '%s parent%s ' % \
(len(revision['parents']),
'' if len(revision['parents']) == 1 else 's')
parents_links += ' '
for p in revision['parents']:
parent_link = gen_revision_link(p, shorten_id=True,
snapshot_context=snapshot_context)
parents_links += parent_link
if p != revision['parents'][-1]:
parents_links += ' + '
path_info = gen_path_info(path)
query_params = {'snapshot_id': snapshot_id,
'origin_type': origin_type,
'origin_url': origin_url,
'timestamp': timestamp,
'visit_id': visit_id}
breadcrumbs = []
breadcrumbs.append({'name': revision['directory'][:7],
'url': reverse('browse-revision',
kwargs={'sha1_git': sha1_git},
query_params=query_params)})
for pi in path_info:
query_params['path'] = pi['path']
breadcrumbs.append({'name': pi['name'],
'url': reverse('browse-revision',
kwargs={'sha1_git': sha1_git},
query_params=query_params)})
vault_cooking = {
'directory_context': False,
'directory_id': None,
'revision_context': True,
'revision_id': sha1_git
}
content = None
content_size = None
mimetype = None
language = None
readme_name = None
readme_url = None
if content_data:
breadcrumbs[-1]['url'] = None
content_size = content_data['length']
mimetype = content_data['mimetype']
if content_data['raw_data']:
content_display_data = prepare_content_for_display(
content_data['raw_data'], content_data['mimetype'], path)
content = content_display_data['content_data']
language = content_display_data['language']
query_params = {}
if path:
query_params['filename'] = path_info[-1]['name']
top_right_link = reverse('browse-content-raw',
kwargs={'query_string': query_string},
query_params=query_params)
top_right_link_text = mark_safe(
''
'Raw File')
else:
for d in dirs:
query_params['path'] = path + d['name']
d['url'] = reverse('browse-revision',
kwargs={'sha1_git': sha1_git},
query_params=query_params)
for f in files:
query_params['path'] = path + f['name']
f['url'] = reverse('browse-revision',
kwargs={'sha1_git': sha1_git},
query_params=query_params)
f['length'] = filesizeformat(f['length'])
if f['name'].lower().startswith('readme'):
readme_name = f['name']
readme_sha1 = f['checksums']['sha1']
readme_url = reverse('browse-content-raw',
kwargs={'query_string': readme_sha1})
top_right_link = get_revision_log_url(sha1_git, snapshot_context)
top_right_link_text = mark_safe(
''
'History')
vault_cooking['directory_context'] = True
vault_cooking['directory_id'] = dir_id
diff_revision_url = reverse('diff-revision', kwargs={'sha1_git': sha1_git},
query_params={'origin_type': origin_type,
'origin_url': origin_url,
'timestamp': timestamp,
'visit_id': visit_id})
return render(request, 'revision.html',
{'empty_browse': False,
'heading': 'Revision information',
'top_panel_visible': True,
'top_panel_collapsible': True,
'top_panel_text': 'Revision metadata',
'swh_object_metadata': revision_data,
'message_header': message_lines[0],
'message_body': '\n'.join(message_lines[1:]),
'parents_links': mark_safe(parents_links),
'main_panel_visible': True,
'snapshot_context': snapshot_context,
'dirs': dirs,
'files': files,
'content': content,
'content_size': content_size,
'max_content_size': content_display_max_size,
'mimetype': mimetype,
'language': language,
'readme_name': readme_name,
'readme_url': readme_url,
'breadcrumbs': breadcrumbs,
'top_right_link': top_right_link,
'top_right_link_text': top_right_link_text,
'vault_cooking': vault_cooking,
'diff_revision_url': diff_revision_url,
'show_actions_menu': True})
NB_LOG_ENTRIES = 20
@browse_route(r'revision/(?P[0-9a-f]+)/log/',
view_name='browse-revision-log')
def revision_log_browse(request, sha1_git):
"""
Django view that produces an HTML display of the history
log for a SWH revision identified by its id.
The url that points to it is :http:get:`/browse/revision/(sha1_git)/log/`.
""" # noqa
try:
per_page = int(request.GET.get('per_page', NB_LOG_ENTRIES))
revision_log = service.lookup_revision_log(sha1_git,
limit=per_page+1)
revision_log = list(revision_log)
except Exception as exc:
return handle_view_exception(request, exc)
revs_breadcrumb = request.GET.get('revs_breadcrumb', None)
revision_log_display_data = prepare_revision_log_for_display(
revision_log, per_page, revs_breadcrumb)
prev_rev = revision_log_display_data['prev_rev']
prev_revs_breadcrumb = revision_log_display_data['prev_revs_breadcrumb']
prev_log_url = None
if prev_rev:
prev_log_url = \
reverse('browse-revision-log',
kwargs={'sha1_git': prev_rev},
query_params={'revs_breadcrumb': prev_revs_breadcrumb,
'per_page': per_page})
next_rev = revision_log_display_data['next_rev']
next_revs_breadcrumb = revision_log_display_data['next_revs_breadcrumb']
next_log_url = None
if next_rev:
next_log_url = \
reverse('browse-revision-log',
kwargs={'sha1_git': next_rev},
query_params={'revs_breadcrumb': next_revs_breadcrumb,
'per_page': per_page})
revision_log_data = revision_log_display_data['revision_log_data']
for log in revision_log_data:
log['directory'] = gen_directory_link(
log['directory'],
link_text=''
'Browse files',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
return render(request, 'revision-log.html',
{'empty_browse': False,
'heading': 'Revision history information',
'top_panel_visible': False,
'top_panel_collapsible': False,
'top_panel_text': 'Revision history',
'swh_object_metadata': None,
'main_panel_visible': True,
'revision_log': revision_log_data,
'next_log_url': next_log_url,
'prev_log_url': prev_log_url,
'breadcrumbs': None,
'top_right_link': None,
'top_right_link_text': None,
'snapshot_context': None,
'vault_cooking': None,
'show_actions_menu': False})
diff --git a/swh/web/browse/views/snapshot.py b/swh/web/browse/views/snapshot.py
index fed533341..3d6d4acf2 100644
--- a/swh/web/browse/views/snapshot.py
+++ b/swh/web/browse/views/snapshot.py
@@ -1,86 +1,86 @@
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.shortcuts import redirect
from swh.web.browse.browseurls import browse_route
from swh.web.common.utils import reverse
from .utils.snapshot_context import (
browse_snapshot_directory, browse_snapshot_content,
browse_snapshot_log, browse_snapshot_branches,
browse_snapshot_releases
)
@browse_route(r'snapshot/(?P[0-9a-f]+)/',
view_name='browse-snapshot')
def snapshot_browse(request, snapshot_id):
"""Django view for browsing the content of a SWH snapshot.
The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/`
"""
browse_snapshot_url = reverse('browse-snapshot-directory',
kwargs={'snapshot_id': snapshot_id},
query_params=request.GET)
return redirect(browse_snapshot_url)
@browse_route(r'snapshot/(?P[0-9a-f]+)/directory/',
r'snapshot/(?P[0-9a-f]+)/directory/(?P.+)/',
view_name='browse-snapshot-directory')
def snapshot_directory_browse(request, snapshot_id, path=None):
"""Django view for browsing the content of a SWH directory collected
in a SWH snapshot.
The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/directory/[(path)/]`
""" # noqa
return browse_snapshot_directory(request, snapshot_id=snapshot_id,
path=path)
@browse_route(r'snapshot/(?P[0-9a-f]+)/content/(?P.+)/',
view_name='browse-snapshot-content')
def snapshot_content_browse(request, snapshot_id, path):
"""Django view that produces an HTML display of a SWH content
collected in a SWH snapshot.
The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/content/(path)/`
""" # noqa
return browse_snapshot_content(request, snapshot_id=snapshot_id, path=path)
@browse_route(r'snapshot/(?P[0-9a-f]+)/log/',
view_name='browse-snapshot-log')
def snapshot_log_browse(request, snapshot_id):
"""Django view that produces an HTML display of revisions history (aka
the commit log) collected in a SWH snapshot.
The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/log/`
""" # noqa
return browse_snapshot_log(request, snapshot_id=snapshot_id)
@browse_route(r'snapshot/(?P[0-9a-f]+)/branches/',
view_name='browse-snapshot-branches')
def snapshot_branches_browse(request, snapshot_id):
"""Django view that produces an HTML display of the list of releases
collected in a SWH snapshot.
The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/branches/`
""" # noqa
return browse_snapshot_branches(request, snapshot_id=snapshot_id)
@browse_route(r'snapshot/(?P[0-9a-f]+)/releases/',
view_name='browse-snapshot-releases')
def snapshot_releases_browse(request, snapshot_id):
"""Django view that produces an HTML display of the list of releases
collected in a SWH snapshot.
The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/releases/`
""" # noqa
return browse_snapshot_releases(request, snapshot_id=snapshot_id)
diff --git a/swh/web/browse/views/utils/snapshot_context.py b/swh/web/browse/views/utils/snapshot_context.py
index 07b93b1ad..e5108a7b3 100644
--- a/swh/web/browse/views/utils/snapshot_context.py
+++ b/swh/web/browse/views/utils/snapshot_context.py
@@ -1,762 +1,762 @@
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
# Utility module implementing Django views for browsing the SWH archive
# in a snapshot context.
# Its purpose is to factorize code for the views reachable from the
# /origin/.* and /snapshot/.* endpoints.
from django.shortcuts import render
from django.utils.safestring import mark_safe
from django.template.defaultfilters import filesizeformat
from swh.web.browse.utils import (
get_snapshot_context, get_directory_entries, gen_directory_link,
gen_revision_link, request_content, gen_content_link,
prepare_content_for_display, content_display_max_size,
prepare_revision_log_for_display, gen_snapshot_directory_link,
gen_revision_log_link, gen_link
)
from swh.web.common import service
from swh.web.common.exc import (
handle_view_exception, NotFoundExc
)
from swh.web.common.utils import (
reverse, gen_path_info, format_utc_iso_date
)
def _get_branch(branches, branch_name):
"""
Utility function to get a specific branch from a branches list.
Its purpose is to get the default HEAD branch as some SWH origin
(e.g those with svn type) does not have it. In that latter case, check
if there is a master branch instead and returns it.
"""
filtered_branches = \
[b for b in branches if b['name'].endswith(branch_name)]
if len(filtered_branches) > 0:
return filtered_branches[0]
elif branch_name == 'HEAD':
filtered_branches = \
[b for b in branches if b['name'].endswith('master')]
if len(filtered_branches) > 0:
return filtered_branches[0]
elif len(branches) > 0:
return branches[0]
return None
def _get_release(releases, release_name):
"""
Utility function to get a specific release from a releases list.
Returns None if the release can not be found in the list.
"""
filtered_releases = \
[r for r in releases if r['name'] == release_name]
if len(filtered_releases) > 0:
return filtered_releases[0]
else:
return None
def _branch_not_found(branch_type, branch, branches, snapshot_id=None,
origin_info=None, timestamp=None, visit_id=None):
"""
Utility function to raise an exception when a specified branch/release
can not be found.
"""
if branch_type == 'branch':
branch_type = 'Branch'
branch_type_plural = 'branches'
else:
branch_type = 'Release'
branch_type_plural = 'releases'
if snapshot_id and len(branches) == 0:
msg = 'Snapshot with id %s has an empty list' \
' of %s!' % (snapshot_id, branch_type_plural)
elif snapshot_id:
msg = '%s %s for snapshot with id %s' \
' not found!' % (branch_type, branch, snapshot_id)
elif visit_id and len(branches) == 0:
msg = 'Origin with type %s and url %s' \
' for visit with id %s has an empty list' \
' of %s!' % (origin_info['type'], origin_info['url'], visit_id,
branch_type_plural)
elif visit_id:
msg = '%s %s associated to visit with' \
' id %s for origin with type %s and url %s' \
' not found!' % (branch_type, branch, visit_id,
origin_info['type'], origin_info['url'])
elif len(branches) == 0:
msg = 'Origin with type %s and url %s' \
' for visit with timestamp %s has an empty list' \
' of %s!' % (origin_info['type'], origin_info['url'],
timestamp, branch_type_plural)
else:
msg = '%s %s associated to visit with' \
' timestamp %s for origin with type %s' \
' and url %s not found!' % (branch_type, branch, timestamp,
origin_info['type'],
origin_info['url'])
raise NotFoundExc(msg)
def _process_snapshot_request(request, snapshot_id=None, origin_type=None,
origin_url=None, timestamp=None, path=None,
browse_context='directory'):
"""
Utility function to perform common input request processing
for snapshot context views.
"""
visit_id = request.GET.get('visit_id', None)
snapshot_context = get_snapshot_context(snapshot_id, origin_type,
origin_url, timestamp, visit_id)
swh_type = snapshot_context['swh_type']
origin_info = snapshot_context['origin_info']
branches = snapshot_context['branches']
releases = snapshot_context['releases']
url_args = snapshot_context['url_args']
query_params = snapshot_context['query_params']
browse_view_name = 'browse-' + swh_type + '-' + browse_context
for b in branches:
branch_url_args = dict(url_args)
branch_query_params = dict(query_params)
branch_query_params['branch'] = b['name']
if path:
b['path'] = path
branch_url_args['path'] = path
b['url'] = reverse(browse_view_name,
kwargs=branch_url_args,
query_params=branch_query_params)
for r in releases:
release_url_args = dict(url_args)
release_query_params = dict(query_params)
release_query_params['release'] = r['name']
if path:
r['path'] = path
release_url_args['path'] = path
r['url'] = reverse(browse_view_name,
kwargs=release_url_args,
query_params=release_query_params)
root_sha1_git = None
revision_id = request.GET.get('revision', None)
release_name = request.GET.get('release', None)
branch_name = None
if revision_id:
revision = service.lookup_revision(revision_id)
root_sha1_git = revision['directory']
branches.append({'name': revision_id,
'revision': revision_id,
'directory': root_sha1_git,
'url': None})
branch_name = revision_id
query_params['revision'] = revision_id
elif release_name:
release = _get_release(releases, release_name)
if release:
root_sha1_git = release['directory']
revision_id = release['target']
query_params['release'] = release_name
else:
_branch_not_found("release", release_name, releases, snapshot_id,
origin_info, timestamp, visit_id)
else:
branch_name = request.GET.get('branch', None)
if branch_name:
query_params['branch'] = branch_name
branch = _get_branch(branches, branch_name or 'HEAD')
if branch:
branch_name = branch['name']
root_sha1_git = branch['directory']
revision_id = branch['revision']
else:
_branch_not_found("branch", branch_name, branches, snapshot_id,
origin_info, timestamp, visit_id)
snapshot_context['query_params'] = query_params
snapshot_context['root_sha1_git'] = root_sha1_git
snapshot_context['revision_id'] = revision_id
snapshot_context['branch'] = branch_name
snapshot_context['release'] = release_name
return snapshot_context
def browse_snapshot_directory(request, snapshot_id=None, origin_type=None,
origin_url=None, timestamp=None, path=None):
"""
Django view implementation for browsing a directory in a snapshot context.
"""
try:
snapshot_context = _process_snapshot_request(request, snapshot_id,
origin_type, origin_url,
timestamp, path,
browse_context='directory') # noqa
root_sha1_git = snapshot_context['root_sha1_git']
sha1_git = root_sha1_git
if path:
dir_info = service.lookup_directory_with_path(root_sha1_git, path)
sha1_git = dir_info['target']
dirs, files = get_directory_entries(sha1_git)
except Exception as exc:
return handle_view_exception(request, exc)
swh_type = snapshot_context['swh_type']
origin_info = snapshot_context['origin_info']
visit_info = snapshot_context['visit_info']
url_args = snapshot_context['url_args']
query_params = snapshot_context['query_params']
revision_id = snapshot_context['revision_id']
snapshot_id = snapshot_context['snapshot_id']
path_info = gen_path_info(path)
browse_view_name = 'browse-' + swh_type + '-directory'
breadcrumbs = []
breadcrumbs.append({'name': root_sha1_git[:7],
'url': reverse(browse_view_name,
kwargs=url_args,
query_params=query_params)})
for pi in path_info:
bc_url_args = dict(url_args)
bc_url_args['path'] = pi['path']
breadcrumbs.append({'name': pi['name'],
'url': reverse(browse_view_name,
kwargs=bc_url_args,
query_params=query_params)})
path = '' if path is None else (path + '/')
for d in dirs:
bc_url_args = dict(url_args)
bc_url_args['path'] = path + d['name']
d['url'] = reverse(browse_view_name,
kwargs=bc_url_args,
query_params=query_params)
sum_file_sizes = 0
readme_name = None
readme_url = None
browse_view_name = 'browse-' + swh_type + '-content'
for f in files:
bc_url_args = dict(url_args)
bc_url_args['path'] = path + f['name']
f['url'] = reverse(browse_view_name,
kwargs=bc_url_args,
query_params=query_params)
sum_file_sizes += f['length']
f['length'] = filesizeformat(f['length'])
if f['name'].lower().startswith('readme'):
readme_name = f['name']
readme_sha1 = f['checksums']['sha1']
readme_url = reverse('browse-content-raw',
kwargs={'query_string': readme_sha1})
browse_view_name = 'browse-' + swh_type + '-log'
history_url = reverse(browse_view_name,
kwargs=url_args,
query_params=query_params)
sum_file_sizes = filesizeformat(sum_file_sizes)
browse_dir_link = \
gen_directory_link(sha1_git, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
browse_rev_link = \
gen_revision_link(revision_id,
snapshot_context=snapshot_context,
link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
dir_metadata = {'id': sha1_git,
'context-independent directory': browse_dir_link,
'number of regular files': len(files),
'number of subdirectories': len(dirs),
'sum of regular file sizes': sum_file_sizes,
'path': '/' + path,
'revision id': revision_id,
'revision': browse_rev_link,
'snapshot id': snapshot_id}
if origin_info:
dir_metadata['origin id'] = origin_info['id']
dir_metadata['origin type'] = origin_info['type']
dir_metadata['origin url'] = origin_info['url']
dir_metadata['origin visit date'] = format_utc_iso_date(visit_info['date']), # noqa
dir_metadata['origin visit id'] = visit_info['visit']
snapshot_context_url = reverse('browse-snapshot-directory',
kwargs={'snapshot_id': snapshot_id},
query_params=request.GET)
browse_snapshot_link = \
gen_link(snapshot_context_url, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
dir_metadata['snapshot context'] = browse_snapshot_link
vault_cooking = {
'directory_context': True,
'directory_id': sha1_git,
'revision_context': True,
'revision_id': revision_id
}
return render(request, 'directory.html',
{'empty_browse': False,
'heading': 'Directory information',
'top_panel_visible': True,
'top_panel_collapsible': True,
'top_panel_text': 'Directory metadata',
'swh_object_metadata': dir_metadata,
'main_panel_visible': True,
'dirs': dirs,
'files': files,
'breadcrumbs': breadcrumbs,
'top_right_link': history_url,
'top_right_link_text': mark_safe(
''
'History'
),
'readme_name': readme_name,
'readme_url': readme_url,
'snapshot_context': snapshot_context,
'vault_cooking': vault_cooking,
'show_actions_menu': True})
def browse_snapshot_content(request, snapshot_id=None, origin_type=None,
origin_url=None, timestamp=None, path=None):
"""
Django view implementation for browsing a content in a snapshot context.
"""
try:
snapshot_context = _process_snapshot_request(request, snapshot_id,
origin_type, origin_url,
timestamp, path,
browse_context='content')
root_sha1_git = snapshot_context['root_sha1_git']
content_info = service.lookup_directory_with_path(root_sha1_git, path)
sha1_git = content_info['target']
query_string = 'sha1_git:' + sha1_git
content_data = request_content(query_string)
except Exception as exc:
return handle_view_exception(request, exc)
swh_type = snapshot_context['swh_type']
url_args = snapshot_context['url_args']
query_params = snapshot_context['query_params']
revision_id = snapshot_context['revision_id']
origin_info = snapshot_context['origin_info']
visit_info = snapshot_context['visit_info']
snapshot_id = snapshot_context['snapshot_id']
content = None
language = None
if content_data['raw_data'] is not None:
content_display_data = prepare_content_for_display(
content_data['raw_data'], content_data['mimetype'], path)
content = content_display_data['content_data']
language = content_display_data['language']
filename = None
path_info = None
browse_view_name = 'browse-' + swh_type + '-directory'
breadcrumbs = []
split_path = path.split('/')
filename = split_path[-1]
path_info = gen_path_info(path[:-len(filename)])
breadcrumbs.append({'name': root_sha1_git[:7],
'url': reverse(browse_view_name,
kwargs=url_args,
query_params=query_params)})
for pi in path_info:
bc_url_args = dict(url_args)
bc_url_args['path'] = pi['path']
breadcrumbs.append({'name': pi['name'],
'url': reverse(browse_view_name,
kwargs=bc_url_args,
query_params=query_params)})
breadcrumbs.append({'name': filename,
'url': None})
browse_content_link = \
gen_content_link(sha1_git, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
content_raw_url = reverse('browse-content-raw',
kwargs={'query_string': query_string},
query_params={'filename': filename})
browse_rev_link = \
gen_revision_link(revision_id,
snapshot_context=snapshot_context,
link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
content_metadata = {
'context-independent content': browse_content_link,
'sha1 checksum': content_data['checksums']['sha1'],
'sha1_git checksum': content_data['checksums']['sha1_git'],
'sha256 checksum': content_data['checksums']['sha256'],
'blake2s256 checksum': content_data['checksums']['blake2s256'],
'mime type': content_data['mimetype'],
'encoding': content_data['encoding'],
'size': filesizeformat(content_data['length']),
'language': content_data['language'],
'licenses': content_data['licenses'],
'path': '/' + path[:-len(filename)],
'filename': filename,
'revision id': revision_id,
'revision': browse_rev_link,
'snapshot id': snapshot_id
}
if origin_info:
content_metadata['origin id'] = origin_info['id'],
content_metadata['origin type'] = origin_info['type']
content_metadata['origin url'] = origin_info['url']
content_metadata['origin visit date'] = format_utc_iso_date(visit_info['date']) # noqa
content_metadata['origin visit id'] = visit_info['visit']
browse_snapshot_url = reverse('browse-snapshot-content',
kwargs={'snapshot_id': snapshot_id,
'path': path},
query_params=request.GET)
browse_snapshot_link = \
gen_link(browse_snapshot_url, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
content_metadata['snapshot context'] = browse_snapshot_link
return render(request, 'content.html',
{'empty_browse': False,
'heading': 'Content information',
'top_panel_visible': True,
'top_panel_collapsible': True,
'top_panel_text': 'Content metadata',
'swh_object_metadata': content_metadata,
'main_panel_visible': True,
'content': content,
'content_size': content_data['length'],
'max_content_size': content_display_max_size,
'mimetype': content_data['mimetype'],
'language': language,
'breadcrumbs': breadcrumbs,
'top_right_link': content_raw_url,
'top_right_link_text': mark_safe(
''
'Raw File'),
'snapshot_context': snapshot_context,
'vault_cooking': None,
'show_actions_menu': False})
PER_PAGE = 20
def browse_snapshot_log(request, snapshot_id=None, origin_type=None,
origin_url=None, timestamp=None):
"""
Django view implementation for browsing a revision history in a
snapshot context.
"""
try:
snapshot_context = _process_snapshot_request(request, snapshot_id,
origin_type, origin_url,
timestamp, browse_context='log') # noqa
revision_id = snapshot_context['revision_id']
current_rev = revision_id
per_page = int(request.GET.get('per_page', PER_PAGE))
revs_breadcrumb = request.GET.get('revs_breadcrumb', None)
if revs_breadcrumb:
current_rev = revs_breadcrumb.split('/')[-1]
revision_log = service.lookup_revision_log(current_rev,
limit=per_page+1)
revision_log = list(revision_log)
except Exception as exc:
return handle_view_exception(request, exc)
swh_type = snapshot_context['swh_type']
origin_info = snapshot_context['origin_info']
visit_info = snapshot_context['visit_info']
url_args = snapshot_context['url_args']
query_params = snapshot_context['query_params']
snapshot_id = snapshot_context['snapshot_id']
query_params['per_page'] = per_page
revision_log_display_data = prepare_revision_log_for_display(
revision_log, per_page, revs_breadcrumb, snapshot_context)
browse_view_name = 'browse-' + swh_type + '-log'
prev_rev = revision_log_display_data['prev_rev']
prev_revs_breadcrumb = revision_log_display_data['prev_revs_breadcrumb']
prev_log_url = None
query_params['revs_breadcrumb'] = prev_revs_breadcrumb
if prev_rev:
prev_log_url = \
reverse(browse_view_name,
kwargs=url_args,
query_params=query_params)
next_rev = revision_log_display_data['next_rev']
next_revs_breadcrumb = revision_log_display_data['next_revs_breadcrumb']
next_log_url = None
query_params['revs_breadcrumb'] = next_revs_breadcrumb
if next_rev:
next_log_url = \
reverse(browse_view_name,
kwargs=url_args,
query_params=query_params)
revision_log_data = revision_log_display_data['revision_log_data']
for i, log in enumerate(revision_log_data):
params = {
'revision': revision_log[i]['id'],
}
if 'visit_id' in query_params:
params['visit_id'] = query_params['visit_id']
log['directory'] = gen_snapshot_directory_link(
snapshot_context, revision_log[i]['id'],
link_text=''
'Browse files',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
browse_log_link = \
gen_revision_log_link(revision_id, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
revision_metadata = {
'context-independent revision history': browse_log_link,
'snapshot id': snapshot_id
}
if origin_info:
revision_metadata['origin id'] = origin_info['id']
revision_metadata['origin type'] = origin_info['type']
revision_metadata['origin url'] = origin_info['url']
revision_metadata['origin visit date'] = format_utc_iso_date(visit_info['date']) # noqa
revision_metadata['origin visit id'] = visit_info['visit']
browse_snapshot_url = reverse('browse-snapshot-log',
kwargs={'snapshot_id': snapshot_id},
query_params=request.GET)
browse_snapshot_link = \
gen_link(browse_snapshot_url, link_text='Browse',
link_attrs={'class': 'btn btn-default btn-sm',
'role': 'button'})
revision_metadata['snapshot context'] = browse_snapshot_link
return render(request, 'revision-log.html',
{'empty_browse': False,
'heading': 'Revision history information',
'top_panel_visible': True,
'top_panel_collapsible': True,
'top_panel_text': 'Revision history metadata',
'swh_object_metadata': revision_metadata,
'main_panel_visible': True,
'revision_log': revision_log_data,
'next_log_url': next_log_url,
'prev_log_url': prev_log_url,
'breadcrumbs': None,
'top_right_link': None,
'top_right_link_text': None,
'snapshot_context': snapshot_context,
'vault_cooking': None,
'show_actions_menu': False})
def browse_snapshot_branches(request, snapshot_id=None, origin_type=None,
origin_url=None, timestamp=None):
"""
Django view implementation for browsing a list of branches in a snapshot
context.
"""
try:
snapshot_context = _process_snapshot_request(request, snapshot_id,
origin_type, origin_url,
timestamp)
except Exception as exc:
return handle_view_exception(request, exc)
branches_offset = int(request.GET.get('branches_offset', 0))
swh_type = snapshot_context['swh_type']
origin_info = snapshot_context['origin_info']
url_args = snapshot_context['url_args']
query_params = snapshot_context['query_params']
browse_view_name = 'browse-' + swh_type + '-directory'
branches = snapshot_context['branches']
displayed_branches = \
branches[branches_offset:branches_offset+PER_PAGE]
for branch in displayed_branches:
if snapshot_id:
revision_url = reverse('browse-revision',
kwargs={'sha1_git': branch['revision']},
query_params={'snapshot_id': snapshot_id})
else:
revision_url = reverse('browse-revision',
kwargs={'sha1_git': branch['revision']},
query_params={'origin_type': origin_info['type'], # noqa
'origin_url': origin_info['url']}) # noqa
query_params['branch'] = branch['name']
directory_url = reverse(browse_view_name,
kwargs=url_args,
query_params=query_params)
del query_params['branch']
branch['revision_url'] = revision_url
branch['directory_url'] = directory_url
browse_view_name = 'browse-' + swh_type + '-branches'
prev_branches_url = None
next_branches_url = None
next_offset = branches_offset + PER_PAGE
prev_offset = branches_offset - PER_PAGE
if next_offset < len(branches):
query_params['branches_offset'] = next_offset
next_branches_url = reverse(browse_view_name,
kwargs=url_args, query_params=query_params)
query_params['branches_offset'] = None
if prev_offset >= 0:
if prev_offset != 0:
query_params['branches_offset'] = prev_offset
prev_branches_url = reverse(browse_view_name,
kwargs=url_args, query_params=query_params)
return render(request, 'branches.html',
{'empty_browse': False,
'heading': 'Origin branches list',
'top_panel_visible': False,
'top_panel_collapsible': False,
'top_panel_text': 'Branches list',
'swh_object_metadata': {},
'main_panel_visible': True,
'top_right_link': None,
'top_right_link_text': None,
'displayed_branches': displayed_branches,
'prev_branches_url': prev_branches_url,
'next_branches_url': next_branches_url,
'snapshot_context': snapshot_context})
def browse_snapshot_releases(request, snapshot_id=None, origin_type=None,
origin_url=None, timestamp=None):
"""
Django view implementation for browsing a list of releases in a snapshot
context.
"""
try:
snapshot_context = _process_snapshot_request(request, snapshot_id,
origin_type, origin_url,
timestamp)
except Exception as exc:
return handle_view_exception(request, exc)
releases_offset = int(request.GET.get('releases_offset', 0))
swh_type = snapshot_context['swh_type']
origin_info = snapshot_context['origin_info']
url_args = snapshot_context['url_args']
query_params = snapshot_context['query_params']
releases = snapshot_context['releases']
displayed_releases = \
releases[releases_offset:releases_offset+PER_PAGE]
for release in displayed_releases:
if snapshot_id:
release_url = reverse('browse-release',
kwargs={'sha1_git': release['id']},
query_params={'snapshot_id': snapshot_id})
else:
release_url = reverse('browse-release',
kwargs={'sha1_git': release['id']},
query_params={'origin_type': origin_info['type'], # noqa
'origin_url': origin_info['url']}) # noqa
query_params['release'] = release['name']
del query_params['release']
release['release_url'] = release_url
browse_view_name = 'browse-' + swh_type + '-releases'
prev_releases_url = None
next_releases_url = None
next_offset = releases_offset + PER_PAGE
prev_offset = releases_offset - PER_PAGE
if next_offset < len(releases):
query_params['releases_offset'] = next_offset
next_releases_url = reverse(browse_view_name,
kwargs=url_args, query_params=query_params)
query_params['releases_offset'] = None
if prev_offset >= 0:
if prev_offset != 0:
query_params['releases_offset'] = prev_offset
prev_releases_url = reverse(browse_view_name,
kwargs=url_args, query_params=query_params)
return render(request, 'releases.html',
{'empty_browse': False,
'heading': 'Origin releases list',
'top_panel_visible': False,
'top_panel_collapsible': False,
'top_panel_text': 'Releases list',
'swh_object_metadata': {},
'main_panel_visible': True,
'top_right_link': None,
'top_right_link_text': None,
'displayed_releases': displayed_releases,
'prev_releases_url': prev_releases_url,
'next_releases_url': next_releases_url,
'snapshot_context': snapshot_context,
'vault_cooking': None,
'show_actions_menu': False})
diff --git a/swh/web/common/swh_templatetags.py b/swh/web/common/swh_templatetags.py
index f26dee369..9dd28642f 100644
--- a/swh/web/common/swh_templatetags.py
+++ b/swh/web/common/swh_templatetags.py
@@ -1,103 +1,103 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import json
import re
from django import template
from django.core.serializers.json import DjangoJSONEncoder
from django.utils.safestring import mark_safe
from docutils.core import publish_parts
from docutils.writers.html4css1 import Writer, HTMLTranslator
from inspect import cleandoc
register = template.Library()
class NoHeaderHTMLTranslator(HTMLTranslator):
"""
Docutils translator subclass to customize the generation of HTML
from reST-formatted docstrings
"""
def __init__(self, document):
super().__init__(document)
self.body_prefix = []
self.body_suffix = []
def visit_bullet_list(self, node):
self.context.append((self.compact_simple, self.compact_p))
self.compact_p = None
self.compact_simple = self.is_compactable(node)
self.body.append(self.starttag(node, 'ul', CLASS='docstring'))
DOCSTRING_WRITER = Writer()
DOCSTRING_WRITER.translator_class = NoHeaderHTMLTranslator
@register.filter
def safe_docstring_display(docstring):
"""
Utility function to htmlize reST-formatted documentation in browsable
api.
"""
docstring = cleandoc(docstring)
return publish_parts(docstring, writer=DOCSTRING_WRITER)['html_body']
@register.filter
def urlize_links_and_mails(text):
"""Utility function for decorating api links in browsable api.
Args:
text: whose content matching links should be transformed into
contextual API or Browse html links.
Returns
The text transformed if any link is found.
The text as is otherwise.
"""
if 'href="' not in text:
text = re.sub(r'(/api/[^"<]*|/browse/[^"<]*|http.*$)',
r'\1',
text)
return re.sub(r'([^ <>"]+@[^ <>"]+)',
r'\1',
text)
else:
return text
@register.filter
def urlize_header_links(text):
"""Utility function for decorating headers links in browsable api.
Args
text: Text whose content contains Link header value
Returns:
The text transformed with html link if any link is found.
The text as is otherwise.
"""
return re.sub(r'<(/api/.*|/browse/.*)>', r'<\1>',
text)
@register.filter
def jsonify(obj):
"""Utility function for converting a django template variable
to JSON in order to use it in script tags.
Args
obj: Any django template context variable
Returns:
JSON representation of the variable.
"""
return mark_safe(json.dumps(obj, cls=DjangoJSONEncoder))
diff --git a/swh/web/common/throttling.py b/swh/web/common/throttling.py
index 76bb7b158..95d3b90fa 100644
--- a/swh/web/common/throttling.py
+++ b/swh/web/common/throttling.py
@@ -1,130 +1,130 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import ipaddress
from rest_framework.throttling import ScopedRateThrottle
from swh.web.config import get_config
class SwhWebRateThrottle(ScopedRateThrottle):
"""Custom request rate limiter for DRF enabling to exempt
specific networks specified in swh-web configuration.
Requests are grouped into scopes. It enables to apply different
requests rate limiting based on the scope name but also the
input HTTP request types.
To associate a scope to requests, one must add a 'throttle_scope'
attribute when using a class based view, or call the 'throttle_scope'
decorator when using a function based view. By default, requests
do not have an associated scope and are not rate limited.
Rate limiting can also be configured according to the type
of the input HTTP requests for fine grained tuning.
For instance, the following YAML configuration section sets a rate of:
- 1 per minute for POST requests
- 60 per minute for other request types
for the 'swh_api' scope while exempting those coming from the
127.0.0.0/8 ip network.
.. code-block:: yaml
throttling:
scopes:
swh_api:
limiter_rate:
default: 60/m
POST: 1/m
exempted_networks:
- 127.0.0.0/8
"""
scope = None
def __init__(self):
super().__init__()
self.exempted_networks = None
def get_exempted_networks(self, scope_name):
if not self.exempted_networks:
scopes = get_config()['throttling']['scopes']
scope = scopes.get(scope_name)
if scope:
networks = scope.get('exempted_networks')
if networks:
self.exempted_networks = [ipaddress.ip_network(network)
for network in networks]
return self.exempted_networks
def allow_request(self, request, view):
# class based view case
if not self.scope:
default_scope = getattr(view, self.scope_attr, None)
# check if there is a specific rate limiting associated
# to the request type
try:
request_scope = default_scope + '_' + request.method.lower()
setattr(view, self.scope_attr, request_scope)
request_allowed = \
super(SwhWebRateThrottle, self).allow_request(request, view) # noqa
setattr(view, self.scope_attr, default_scope)
# use default rate limiting otherwise
except Exception:
setattr(view, self.scope_attr, default_scope)
request_allowed = \
super(SwhWebRateThrottle, self).allow_request(request, view) # noqa
# function based view case
else:
default_scope = self.scope
# check if there is a specific rate limiting associated
# to the request type
try:
self.scope = default_scope + '_' + request.method.lower()
self.rate = self.get_rate()
# use default rate limiting otherwise
except Exception:
self.scope = default_scope
self.rate = self.get_rate()
self.num_requests, self.duration = self.parse_rate(self.rate)
request_allowed = \
super(ScopedRateThrottle, self).allow_request(request, view)
self.scope = default_scope
exempted_networks = self.get_exempted_networks(default_scope)
if exempted_networks:
remote_address = ipaddress.ip_address(self.get_ident(request))
return any(remote_address in network
for network in exempted_networks) or \
request_allowed
return request_allowed
def throttle_scope(scope):
"""Decorator that allows the throttle scope of a DRF
function based view to be set::
@api_view(['GET', ])
@throttle_scope('scope')
def view(request):
...
"""
def decorator(func):
SwhScopeRateThrottle = type(
'CustomScopeRateThrottle',
(SwhWebRateThrottle,),
{'scope': scope}
)
func.throttle_classes = (SwhScopeRateThrottle, )
return func
return decorator
diff --git a/swh/web/common/urlsindex.py b/swh/web/common/urlsindex.py
index dc0bde36e..6e44db7aa 100644
--- a/swh/web/common/urlsindex.py
+++ b/swh/web/common/urlsindex.py
@@ -1,48 +1,48 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.conf.urls import url
class UrlsIndex(object):
"""
Simple helper class for centralizing url patterns of a Django
web application.
Derived classes should override the 'scope' class attribute otherwise
all declared patterns will be grouped under the default one.
"""
_urlpatterns = {}
scope = 'default'
@classmethod
def add_url_pattern(cls, url_pattern, view, view_name):
"""
Class method that adds an url pattern to the current scope.
Args:
url_pattern: regex describing a Django url
view: function implementing the Django view
view_name: name of the view used to reverse the url
"""
if cls.scope not in cls._urlpatterns:
cls._urlpatterns[cls.scope] = []
if view_name:
cls._urlpatterns[cls.scope].append(url(url_pattern, view,
name=view_name))
else:
cls._urlpatterns[cls.scope].append(url(url_pattern, view))
@classmethod
def get_url_patterns(cls):
"""
Class method that returns the list of url pattern associated to
the current scope.
Returns:
The list of url patterns associated to the current scope
"""
return cls._urlpatterns[cls.scope]
diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py
index 8addd69a8..caea7b068 100644
--- a/swh/web/common/utils.py
+++ b/swh/web/common/utils.py
@@ -1,184 +1,184 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import re
from datetime import datetime, timezone
from dateutil import parser as date_parser
from dateutil import tz
from swh.web.common.exc import BadInputExc
from django.core import urlresolvers
from django.http import QueryDict
def reverse(viewname, args=None, kwargs=None, query_params=None,
current_app=None, urlconf=None):
"""An override of django reverse function supporting query parameters.
Args:
viewname: the name of the django view from which to compute a url
args: list of url arguments ordered according to their position it
kwargs: dictionary of url arguments indexed by their names
query_params: dictionary of query parameters to append to the
reversed url
current_app: the name of the django app tighted to the view
urlconf: url configuration module
Returns:
The url of the requested view with processed arguments and
query parameters
"""
url = urlresolvers.reverse(
viewname, urlconf=urlconf, args=args,
kwargs=kwargs, current_app=current_app)
if query_params:
query_params = {k: v for k, v in query_params.items() if v is not None}
if query_params and len(query_params) > 0:
query_dict = QueryDict('', mutable=True)
for k in sorted(query_params.keys()):
query_dict[k] = query_params[k]
url += ('?' + query_dict.urlencode(safe='/'))
return url
def fmap(f, data):
"""Map f to data at each level.
This must keep the origin data structure type:
- map -> map
- dict -> dict
- list -> list
- None -> None
Args:
f: function that expects one argument.
data: data to traverse to apply the f function.
list, map, dict or bare value.
Returns:
The same data-structure with modified values by the f function.
"""
if data is None:
return data
if isinstance(data, map):
return map(lambda y: fmap(f, y), (x for x in data))
if isinstance(data, list):
return [fmap(f, x) for x in data]
if isinstance(data, dict):
return {k: fmap(f, v) for (k, v) in data.items()}
return f(data)
def datetime_to_utc(date):
"""Returns datetime in UTC without timezone info
Args:
date (datetime.datetime): input datetime with timezone info
Returns:
datetime.datime: datetime in UTC without timezone info
"""
if date.tzinfo:
return date.astimezone(tz.gettz('UTC')).replace(tzinfo=timezone.utc)
else:
return date
def parse_timestamp(timestamp):
"""Given a time or timestamp (as string), parse the result as UTC datetime.
Returns:
a timezone-aware datetime representing the parsed value.
None if the parsing fails.
Samples:
- 2016-01-12
- 2016-01-12T09:19:12+0100
- Today is January 1, 2047 at 8:21:00AM
- 1452591542
"""
if not timestamp:
return None
try:
date = date_parser.parse(timestamp, ignoretz=False, fuzzy=True)
return datetime_to_utc(date)
except Exception:
try:
return datetime.utcfromtimestamp(float(timestamp)).replace(
tzinfo=timezone.utc)
except (ValueError, OverflowError) as e:
raise BadInputExc(e)
def shorten_path(path):
"""Shorten the given path: for each hash present, only return the first
8 characters followed by an ellipsis"""
sha256_re = r'([0-9a-f]{8})[0-9a-z]{56}'
sha1_re = r'([0-9a-f]{8})[0-9a-f]{32}'
ret = re.sub(sha256_re, r'\1...', path)
return re.sub(sha1_re, r'\1...', ret)
def format_utc_iso_date(iso_date, fmt='%d %B %Y, %H:%M UTC'):
"""Turns a string reprensation of an ISO 8601 date string
to UTC and format it into a more human readable one.
For instance, from the following input
string: '2017-05-04T13:27:13+02:00' the following one
is returned: '04 May 2017, 11:27 UTC'.
Custom format string may also be provided
as parameter
Args:
iso_date (str): a string representation of an ISO 8601 date
fmt (str): optional date formatting string
Returns:
A formatted string representation of the input iso date
"""
if not iso_date:
return iso_date
date = parse_timestamp(iso_date)
return date.strftime(fmt)
def gen_path_info(path):
"""Function to generate path data navigation for use
with a breadcrumb in the swh web ui.
For instance, from a path /folder1/folder2/folder3,
it returns the following list::
[{'name': 'folder1', 'path': 'folder1'},
{'name': 'folder2', 'path': 'folder1/folder2'},
{'name': 'folder3', 'path': 'folder1/folder2/folder3'}]
Args:
path: a filesystem path
Returns:
A list of path data for navigation as illustrated above.
"""
path_info = []
if path:
sub_paths = path.strip('/').split('/')
path_from_root = ''
for p in sub_paths:
path_from_root += '/' + p
path_info.append({'name': p,
'path': path_from_root.strip('/')})
return path_info
diff --git a/swh/web/config.py b/swh/web/config.py
index 5fdfdec00..59decdfec 100644
--- a/swh/web/config.py
+++ b/swh/web/config.py
@@ -1,93 +1,93 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.core import config
from swh.storage import get_storage
from swh.indexer.storage import get_indexer_storage
from swh.vault.api.client import RemoteVaultClient
DEFAULT_CONFIG = {
'allowed_hosts': ('list', []),
'storage': ('dict', {
'cls': 'remote',
'args': {
'url': 'http://127.0.0.1:5002/',
},
}),
'indexer_storage': ('dict', {
'cls': 'remote',
'args': {
'url': 'http://127.0.0.1:5007/',
}
}),
'vault': ('string', 'http://127.0.0.1:5005/'),
'log_dir': ('string', '/tmp/swh/log'),
'debug': ('bool', True),
'host': ('string', '127.0.0.1'),
'port': ('int', 5004),
'secret_key': ('string', 'development key'),
# do not display code highlighting for content > 1MB
'content_display_max_size': ('int', 1024 * 1024),
'throttling': ('dict', {
'cache_uri': None, # production: memcached as cache (127.0.0.1:11211)
# development: in-memory cache so None
'scopes': {
'swh_api': {
'limiter_rate': {
'default': '120/h'
},
'exempted_networks': ['127.0.0.0/8']
},
'swh_vault_cooking': {
'limiter_rate': {
'default': '120/h',
'GET': '60/m'
},
'exempted_networks': ['127.0.0.0/8']
}
}
})
}
swhweb_config = {}
def get_config(config_file='webapp/webapp'):
"""Read the configuration file `config_file`, update the app with
parameters (secret_key, conf) and return the parsed configuration as a
dict. If no configuration file is provided, return a default
configuration."""
if not swhweb_config:
cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
swhweb_config.update(cfg)
config.prepare_folders(swhweb_config, 'log_dir')
swhweb_config['storage'] = get_storage(**swhweb_config['storage'])
swhweb_config['vault'] = RemoteVaultClient(swhweb_config['vault'])
swhweb_config['indexer_storage'] = get_indexer_storage(
**swhweb_config['indexer_storage'])
return swhweb_config
def storage():
"""Return the current application's SWH storage.
"""
return get_config()['storage']
def vault():
"""Return the current application's SWH vault.
"""
return get_config()['vault']
def indexer_storage():
"""Return the current application's SWH indexer storage.
"""
return get_config()['indexer_storage']
diff --git a/swh/web/doc_config.py b/swh/web/doc_config.py
index 8e8d13747..7ccd576d4 100644
--- a/swh/web/doc_config.py
+++ b/swh/web/doc_config.py
@@ -1,34 +1,34 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from sphinxcontrib import httpdomain
_swh_web_base_url = 'https://archive.softwareheritage.org'
_swh_web_api_endpoint = 'api'
_swh_web_api_version = 1
_swh_web_api_url = '%s/%s/%s/' % (_swh_web_base_url,
_swh_web_api_endpoint,
_swh_web_api_version)
_swh_web_browse_endpoint = 'browse'
_swh_web_browse_url = '%s/%s/' % (_swh_web_base_url,
_swh_web_browse_endpoint)
def customize_sphinx_conf(sphinx_conf):
"""
Utility function used to customize the sphinx doc build for swh-web
globally (when building doc from swh-docs) or locally (when building doc
from swh-web).
Args:
sphinx_conf (module): a reference to the sphinx conf.py module
used to build the doc.
"""
# fix for sphinxcontrib.httpdomain 1.3
if 'Link' not in httpdomain.HEADER_REFS:
httpdomain.HEADER_REFS['Link'] = httpdomain.IETFRef(5988, '5')
sphinx_conf.extlinks['swh_web_api'] = (_swh_web_api_url + '%s', None)
sphinx_conf.extlinks['swh_web_browse'] = (_swh_web_browse_url + '%s', None)
diff --git a/swh/web/manage.py b/swh/web/manage.py
index 182bc12c4..56706b89e 100755
--- a/swh/web/manage.py
+++ b/swh/web/manage.py
@@ -1,45 +1,45 @@
#!/usr/bin/env python
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
import sys
from swh.web import config
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE",
"swh.web.settings.development")
# import root urls module for swh-web before running the django dev server
# in order to ensure it will be automatically reloaded when source files
# are modified (as django autoreload feature only works if the modules are
# in sys.modules)
try:
from swh.web import urls # noqa
except Exception:
pass
try:
from django.core.management.commands.runserver import (
Command as runserver
)
from django.core.management import execute_from_command_line
except ImportError:
# The above import may fail for some other reason. Ensure that the
# issue is really that Django is missing to avoid masking other
# exceptions on Python 2.
try:
import django # noqa
except ImportError:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
)
raise
swh_web_config = config.get_config()
runserver.default_port = swh_web_config['port']
runserver.default_addr = swh_web_config['host']
execute_from_command_line(sys.argv)
diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py
index 4145f7505..bccd64653 100644
--- a/swh/web/settings/common.py
+++ b/swh/web/settings/common.py
@@ -1,213 +1,213 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""
Django common settings for swh-web.
"""
import os
from swh.web.config import get_config
swh_web_config = get_config()
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = swh_web_config['secret_key']
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = swh_web_config['debug']
DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config['debug']
ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] + swh_web_config['allowed_hosts']
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'rest_framework',
'swh.web.api',
'swh.web.browse',
'webpack_loader',
'django_js_reverse'
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware'
]
# Compress all assets (static ones and dynamically generated html)
# served by django in a local development environement context.
# In a production environment, assets compression will be directly
# handled by web servers like apache or nginx.
if swh_web_config['debug']:
MIDDLEWARE.insert(0, 'django.middleware.gzip.GZipMiddleware')
ROOT_URLCONF = 'swh.web.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [os.path.join(PROJECT_DIR, "../templates")],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
'libraries': {
'swh_templatetags': 'swh.web.common.swh_templatetags',
},
},
},
]
WSGI_APPLICATION = 'swh.web.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(PROJECT_DIR, 'db.sqlite3'),
}
}
# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa
},
]
# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_URL = '/static/'
STATICFILES_DIRS = [
os.path.join(PROJECT_DIR, "../static")
]
INTERNAL_IPS = ['127.0.0.1']
throttle_rates = {}
http_requests = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH']
throttling = swh_web_config['throttling']
for limiter_scope, limiter_conf in throttling['scopes'].items():
if 'default' in limiter_conf['limiter_rate']:
throttle_rates[limiter_scope] = limiter_conf['limiter_rate']['default']
# for backward compatibility
else:
throttle_rates[limiter_scope] = limiter_conf['limiter_rate']
# register sub scopes specific for HTTP request types
for http_request in http_requests:
if http_request in limiter_conf['limiter_rate']:
throttle_rates[limiter_scope + '_' + http_request.lower()] = \
limiter_conf['limiter_rate'][http_request]
REST_FRAMEWORK = {
'DEFAULT_RENDERER_CLASSES': (
'rest_framework.renderers.JSONRenderer',
'swh.web.api.renderers.YAMLRenderer',
'rest_framework.renderers.TemplateHTMLRenderer'
),
'DEFAULT_THROTTLE_CLASSES': (
'swh.web.common.throttling.SwhWebRateThrottle',
),
'DEFAULT_THROTTLE_RATES': throttle_rates
}
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse',
},
'require_debug_true': {
'()': 'django.utils.log.RequireDebugTrue',
},
},
'handlers': {
'console': {
'level': 'DEBUG',
'filters': ['require_debug_true'],
'class': 'logging.StreamHandler',
},
'file': {
'level': 'INFO',
'filters': ['require_debug_false'],
'class': 'logging.FileHandler',
'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'),
},
},
'loggers': {
'django': {
'handlers': ['console', 'file'],
'level': 'DEBUG' if DEBUG else 'INFO',
'propagate': True,
}
},
}
WEBPACK_LOADER = {
'DEFAULT': {
'CACHE': not DEBUG,
'BUNDLE_DIR_NAME': './',
'STATS_FILE': os.path.join(PROJECT_DIR, '../static/webpack-stats.json'), # noqa
'POLL_INTERVAL': 0.1,
'TIMEOUT': None,
'IGNORE': ['.+\.hot-update.js', '.+\.map']
}
}
diff --git a/swh/web/settings/development.py b/swh/web/settings/development.py
index ae4156a03..feabdae2d 100644
--- a/swh/web/settings/development.py
+++ b/swh/web/settings/development.py
@@ -1,23 +1,23 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
# flake8: noqa
"""
Django development settings for swh-web.
"""
import os
# guard to avoid side effects on the django settings when building the
# Debian package for swh-web
if os.environ['DJANGO_SETTINGS_MODULE'] == 'swh.web.settings.development':
from .common import *
MIDDLEWARE += ['swh.web.common.middlewares.HtmlPrettifyMiddleware']
from django.core.cache import cache
cache.clear()
diff --git a/swh/web/settings/production.py b/swh/web/settings/production.py
index 6e02e23b5..a03afc6a1 100644
--- a/swh/web/settings/production.py
+++ b/swh/web/settings/production.py
@@ -1,50 +1,50 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
# flake8: noqa
"""
Django production settings for swh-web.
"""
import os
# guard to avoid side effects on the django settings when building the
# Debian package for swh-web
if os.environ['DJANGO_SETTINGS_MODULE'] == 'swh.web.settings.production':
from .common import *
from .common import swh_web_config
from .common import REST_FRAMEWORK
# activate per-site caching
if 'GZip' in MIDDLEWARE[0]:
MIDDLEWARE.insert(1, 'django.middleware.cache.UpdateCacheMiddleware')
else:
MIDDLEWARE.insert(0, 'django.middleware.cache.UpdateCacheMiddleware')
MIDDLEWARE += ['swh.web.common.middlewares.HtmlMinifyMiddleware',
'django.middleware.cache.FetchFromCacheMiddleware']
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': swh_web_config['throttling']['cache_uri'],
}
}
# Setup support for proxy headers
USE_X_FORWARDED_HOST = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
# We're going through seven (or, in that case, 2) proxies thanks to Varnish
REST_FRAMEWORK['NUM_PROXIES'] = 2
ALLOWED_HOSTS += [
'archive.softwareheritage.org',
'base.softwareheritage.org',
'archive.internal.softwareheritage.org',
]
diff --git a/swh/web/settings/tests.py b/swh/web/settings/tests.py
index 839900ce8..9edac6cdc 100644
--- a/swh/web/settings/tests.py
+++ b/swh/web/settings/tests.py
@@ -1,73 +1,73 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
# flake8: noqa
"""
Django tests settings for swh-web.
"""
import os
# guard to avoid side effects on the django settings when building the
# Debian package for swh-web
if os.environ['DJANGO_SETTINGS_MODULE'] == 'swh.web.settings.tests':
from swh.web.config import get_config
scope1_limiter_rate = 3
scope1_limiter_rate_post = 1
scope2_limiter_rate = 5
scope2_limiter_rate_post = 2
scope3_limiter_rate = 1
scope3_limiter_rate_post = 1
swh_web_config = get_config()
swh_web_config.update({
'debug': True,
'secret_key': 'test',
'throttling': {
'cache_uri': None,
'scopes': {
'swh_api': {
'limiter_rate': {
'default': '60/min'
},
'exempted_networks': ['127.0.0.0/8']
},
'swh_vault_cooking': {
'limiter_rate': {
'default': '120/h',
'GET': '60/m'
},
'exempted_networks': ['127.0.0.0/8']
},
'scope1': {
'limiter_rate': {
'default': '%s/min' % scope1_limiter_rate,
'POST': '%s/min' % scope1_limiter_rate_post,
}
},
'scope2': {
'limiter_rate': {
'default': '%s/min' % scope2_limiter_rate,
'POST': '%s/min' % scope2_limiter_rate_post
}
},
'scope3': {
'limiter_rate': {
'default': '%s/min' % scope3_limiter_rate,
'POST': '%s/min' % scope3_limiter_rate_post
},
'exempted_networks': ['127.0.0.0/8']
}
}
}
})
from .common import *
ALLOWED_HOSTS += ['testserver']
\ No newline at end of file
diff --git a/swh/web/templates/api-endpoints.html b/swh/web/templates/api-endpoints.html
index 69c6ff76a..399156a2c 100644
--- a/swh/web/templates/api-endpoints.html
+++ b/swh/web/templates/api-endpoints.html
@@ -1,77 +1,84 @@
{% extends "layout.html" %}
+{% comment %}
+Copyright (C) 2015-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% load swh_templatetags %}
{% block title %} Endpoints – Software Heritage API {% endblock %}
{% block content %}
Below you can find a list of the available endpoints for version 1 of the
Software Heritage API. For a more general introduction please refer to
the API overview.
Endpoints marked "available" are considered stable for the current version
of the API; endpoints marked "upcoming" are work in progress that will be
stabilized in the near future.
{% endblock %}
diff --git a/swh/web/templates/api.html b/swh/web/templates/api.html
index 6ba91e046..60d3d225d 100644
--- a/swh/web/templates/api.html
+++ b/swh/web/templates/api.html
@@ -1,16 +1,23 @@
{% extends "layout.html" %}
+{% comment %}
+Copyright (C) 2015-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% block title %} Overview – Software Heritage API {% endblock %}
{% block content %}
Software Heritage Web API
{% include 'includes/apidoc-header.html' %}
{% endblock %}
diff --git a/swh/web/templates/apidoc.html b/swh/web/templates/apidoc.html
index 569301237..6d8ac1aaa 100644
--- a/swh/web/templates/apidoc.html
+++ b/swh/web/templates/apidoc.html
@@ -1,154 +1,161 @@
{% extends "layout.html" %}
+{% comment %}
+Copyright (C) 2015-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% load swh_templatetags %}
{% load render_bundle from webpack_loader %}
{% block title %}{{ heading }} – Software Heritage API {% endblock %}
{% block content %}
{% if docstring %}
Description
{{ docstring | safe_docstring_display | safe }}
{% endif %}
{% if response_data %}
Request
{{ request.method }} {{ request.absolute_uri }}
Response
{% if status_code != 200 %}
Status Code
{{ status_code }}
{% endif %}
{% if headers_data %}
Headers
{% for header_name, header_value in headers_data.items %}
{% endblock %}
diff --git a/swh/web/templates/branches.html b/swh/web/templates/branches.html
index 3e1c53208..2eae25aca 100644
--- a/swh/web/templates/branches.html
+++ b/swh/web/templates/branches.html
@@ -1,57 +1,64 @@
{% extends "browse.html" %}
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% block swh-browse-before-panels %}
{% if snapshot_context %}
{% include "includes/snapshot-context.html" %}
{% endif %}
{% endblock %}
{% block swh-browse-main-panel-content %}
No Software Heritage object currently browsed.
To browse the content of the archive, you can either use
the Search interface or refer to the URI scheme described
in the Help page.
{% endblock %}
diff --git a/swh/web/templates/content.html b/swh/web/templates/content.html
index f6e1e2f9a..8671705d7 100644
--- a/swh/web/templates/content.html
+++ b/swh/web/templates/content.html
@@ -1,12 +1,19 @@
{% extends "browse.html" %}
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% block swh-browse-before-panels %}
{% if snapshot_context %}
{% include "includes/snapshot-context.html" %}
{% endif %}
{% endblock %}
{% block swh-browse-main-panel-content %}
{% include "includes/top-navigation.html" %}
{% include "includes/content-display.html" %}
{% endblock %}
diff --git a/swh/web/templates/directory.html b/swh/web/templates/directory.html
index 661a064e1..2e1ec4d6d 100644
--- a/swh/web/templates/directory.html
+++ b/swh/web/templates/directory.html
@@ -1,16 +1,23 @@
{% extends "browse.html" %}
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% block swh-browse-before-panels %}
{% if snapshot_context %}
{% include "includes/snapshot-context.html" %}
{% endif %}
{% endblock %}
{% block swh-browse-main-panel-content %}
{% include "includes/top-navigation.html" %}
{% include "includes/directory-display.html" %}
{% endblock %}
{% block swh-browse-panels-group-end %}
{% include "includes/readme-display.html" %}
{% endblock %}
diff --git a/swh/web/templates/error.html b/swh/web/templates/error.html
index 85eb0631b..b6797725e 100644
--- a/swh/web/templates/error.html
+++ b/swh/web/templates/error.html
@@ -1,32 +1,39 @@
{% extends "layout.html" %}
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% load static %}
{% block title %}Error {{ error_code }} – Software Heritage archive {% endblock %}
{% block content %}
{% endblock %}
\ No newline at end of file
diff --git a/swh/web/templates/homepage.html b/swh/web/templates/homepage.html
index e3472e07b..8a17fb798 100644
--- a/swh/web/templates/homepage.html
+++ b/swh/web/templates/homepage.html
@@ -1,109 +1,116 @@
{% extends "layout.html" %}
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% load static %}
{% block title %}The Software Heritage archive{% endblock %}
{% block content %}
Welcome to the Software Heritage archive
Overview
The long term goal of the Software Heritage initiative is to collect
all publicly available software in source code form together with its
development history, replicate it massively to ensure its preservation,
and share it with everyone who needs it.
The Software Heritage archive is growing over time as we crawl new source code from software
projects and development forges. We will incrementally release archive search
and browse functionalities — as of now you can check whether source code you care
about is already present in the archive or not.
Content
A significant amount of source code has already been ingested in the Software Heritage
archive. It currently includes:
{% endblock %}
diff --git a/swh/web/templates/includes/apidoc-header.html b/swh/web/templates/includes/apidoc-header.html
index 9f3c46a79..c45644ef6 100644
--- a/swh/web/templates/includes/apidoc-header.html
+++ b/swh/web/templates/includes/apidoc-header.html
@@ -1,180 +1,187 @@
+{% comment %}
+Copyright (C) 2015-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
You can jump directly to the
endpoint index, which lists all available API functionalities, or read on for more general information about the API.
Data model
The
Software Heritage project harvests publicly available source code by tracking software distribution channels such as
version control systems, tarball releases, and distribution packages.
All retrieved source code and related metadata are stored in the Software Heritage archive, that is conceptually a
Merkle DAG. All nodes in the graph are content-addressable, i.e., their node identifiers are computed by hashing their
content and, transitively, that of all nodes reachable from them; and no node or edge is ever removed from the graph: the
Software Heritage archive is an append-only data structure.
The following types of objects (i.e., graph nodes) can be found in the Software Heritage archive
(for more information see the
Software Heritage glossary):
Content: a specific version of a file stored in the archive, identified by its cryptographic hashes (currently:
SHA1, Git-like "salted" SHA1, SHA256). Note that content objects are nameless; their names are context-dependent
and stored as part of directory entries (see below).
Also known as: "blob"
Directory: a list of directory entries, where each entry can point to content objects ("file entries"),
revisions ("revision entries"), or transitively to other directories ("directory entries"). All entries
are associated to the local name of the entry (i.e., a relative path without any path separator) and permission metadata
(e.g., chmod value or equivalent).
Revision: a point in time snapshot of the content of a directory, together with associated development metadata
(e.g., author, timestamp, log message, etc).
Also known as: "commit".
Release: a revision that has been marked as noteworthy with a specific name (e.g., a version number), together
with associated development metadata (e.g., author, timestamp, etc).
Also known as: "tag"
Origin: an Internet-based location from which a coherent set of objects (contents, revisions, releases, etc.)
archived by Software Heritage has been obtained. Origins are currently identified by URLs.
Visit: the passage of Software Heritage on a given origin, to retrieve all source code and metadata available
there at the time. A visit object stores the state of all visible branches (if any) available at the origin at visit
time; each of them points to a revision object in the archive. Future visits of the same origin will create new visit
objects, without removing previous ones.
Person: an entity referenced by a revision as either the author or the committer of the corresponding change.
A person is associated to a full name and/or an email address.
Version
The current version of the API is
v1.
Warning: this version of the API is not to be considered stable yet. Non-backward compatible changes might happen
even without changing the API version number.
The response format can be overridden using the
Accept request header. In particular,
Accept: text/html (that web browsers send by default) requests HTML pretty-printing, whereas
Accept: application/yaml requests YAML-encoded responses.
Some API endpoints can be tweaked by passing optional parameters. For GET requests, optional parameters can be passed as
an HTTP query string.
The optional parameter
fields is accepted by all endpoints that return dictionaries and can be used to restrict the list of fields returned
by the API, in case you are not interested in all of them. By default, all available fields are returned.
Unavailability of the underlying storage backend will result in a
503 Service Unavailable HTTP response.
Pagination
Requests that might potentially return many items will be paginated.
Page size is set to a default (usually: 10 items), but might be overridden with the
per_page query parameter up to a maximum (usually: 50 items). Example:
Due to limited resource availability on the back end side, API usage is currently rate limited. Furthermore, as API usage
is currently entirely anonymous (i.e., without any authentication), API "users" are currently identified by their
origin IP address.
Three HTTP response fields will inform you about the current state of limits that apply to your current rate limiting bucket:
X-RateLimit-Limit: maximum number of permitted requests per hour
X-RateLimit-Remaining: number of permitted requests remaining before the next reset
X-RateLimit-Reset: the time (expressed in
Unix time seconds) at which the current rate limiting will expire, resetting to a fresh
X-RateLimit-Limit
\ No newline at end of file
diff --git a/swh/web/templates/includes/breadcrumbs.html b/swh/web/templates/includes/breadcrumbs.html
index 798c5d826..3d6bf68ab 100644
--- a/swh/web/templates/includes/breadcrumbs.html
+++ b/swh/web/templates/includes/breadcrumbs.html
@@ -1,12 +1,19 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
\ No newline at end of file
diff --git a/swh/web/templates/includes/browse-help.html b/swh/web/templates/includes/browse-help.html
index 9d5579329..daa03f5ee 100644
--- a/swh/web/templates/includes/browse-help.html
+++ b/swh/web/templates/includes/browse-help.html
@@ -1,160 +1,167 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
This web application aims to provide HTML views to easily navigate in the Software Heritage archive. This is an ongoing development
and new features and improvements will be progressively added over the time.
URI scheme
The current URI scheme of that web application is described below and depends on the type of Software Heritage object to
browse. Its exhaustive documentation can be consulted from the official
Software Heritage development documentation
Context-independent browsing
Context-independent URLs provide information about SWH objects (e.g., revisions, directories, contents, persons, …), independently
of the contexts where they have been found (e.g., specific software origins, branches, commits, …).
Below are some examples of endpoints used to just render the corresponding information for user consumption:
In order to facilitate the browsing of the archive and generate relevant entry points to it, a
search interface is available. Currently, it enables to search software origins from the URLs they were retrieved
from. More search criteria will be added in the future.
\ No newline at end of file
diff --git a/swh/web/templates/includes/content-display.html b/swh/web/templates/includes/content-display.html
index 363ed9219..4f8427496 100644
--- a/swh/web/templates/includes/content-display.html
+++ b/swh/web/templates/includes/content-display.html
@@ -1,19 +1,26 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% if content_size > max_content_size %}
Content is too large to be displayed (size is greater than {{ max_content_size|filesizeformat }}).
{% elif "inode/x-empty" == mimetype %}
File is empty
{% elif "text/" in mimetype %}
{{ content }}
{% elif "image/" in mimetype and content %}
{% else %}
Content with mime type {{ mimetype }} can not be displayed.
{% endif %}
diff --git a/swh/web/templates/includes/directory-display.html b/swh/web/templates/includes/directory-display.html
index eb3f547f3..b1e21be1b 100644
--- a/swh/web/templates/includes/directory-display.html
+++ b/swh/web/templates/includes/directory-display.html
@@ -1,36 +1,43 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
\ No newline at end of file
diff --git a/swh/web/templates/includes/origins-search.html b/swh/web/templates/includes/origins-search.html
index 03eaa3cdf..d297199b4 100644
--- a/swh/web/templates/includes/origins-search.html
+++ b/swh/web/templates/includes/origins-search.html
@@ -1,50 +1,57 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% load static %}
Search Software Heritage origins to browse
Searching origins ...
Origin type
Origin browse url
Visit status
No origins matching the search criteria were found
\ No newline at end of file
diff --git a/swh/web/templates/includes/readme-display.html b/swh/web/templates/includes/readme-display.html
index 7b6ac6b27..ec4354227 100644
--- a/swh/web/templates/includes/readme-display.html
+++ b/swh/web/templates/includes/readme-display.html
@@ -1,15 +1,22 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% if readme_name %}
{{ readme_name }}
{% endif %}
diff --git a/swh/web/templates/includes/snapshot-context.html b/swh/web/templates/includes/snapshot-context.html
index 6b7b33e8e..70450f3cb 100644
--- a/swh/web/templates/includes/snapshot-context.html
+++ b/swh/web/templates/includes/snapshot-context.html
@@ -1,33 +1,40 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% if snapshot_context.origin_info %}
{% if snapshot_context.origin_info.url|slice:"0:4" == "http" %}
diff --git a/swh/web/templates/includes/top-navigation.html b/swh/web/templates/includes/top-navigation.html
index a27f61d53..c206291ff 100644
--- a/swh/web/templates/includes/top-navigation.html
+++ b/swh/web/templates/includes/top-navigation.html
@@ -1,87 +1,94 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% load swh_templatetags %}
{% if snapshot_context %}
{% if snapshot_context.branch or snapshot_context.release %}
diff --git a/swh/web/templates/includes/vault-create-tasks.html b/swh/web/templates/includes/vault-create-tasks.html
index ffaa714b6..9d9bddf18 100644
--- a/swh/web/templates/includes/vault-create-tasks.html
+++ b/swh/web/templates/includes/vault-create-tasks.html
@@ -1,103 +1,110 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% if vault_cooking %}
{% if vault_cooking.directory_context %}
{% endif %}
{% if vault_cooking.revision_context %}
{% endif %}
Cook and download a directory from the Software Heritage Vault
You have requested the cooking of the directory with identifier {{ vault_cooking.directory_id }}
into a standard tar.gz archive.
Are you sure you want to continue ?
Cook and download a revision from the Software Heritage Vault
You have requested the cooking of the revision with identifier {{ vault_cooking.revision_id }}
into a git fast-import archive.
Are you sure you want to continue ?
Invalid Email !
The provided email is not well-formed.
{% endif %}
\ No newline at end of file
diff --git a/swh/web/templates/includes/vault-ui.html b/swh/web/templates/includes/vault-ui.html
index 224531edb..cbe999405 100644
--- a/swh/web/templates/includes/vault-ui.html
+++ b/swh/web/templates/includes/vault-ui.html
@@ -1,35 +1,42 @@
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
Download content from the Software Heritage Vault
This interface enables to track the status of the different Software Heritage
Vault cooking tasks created while browsing the archive.
Once a cooking task is finished, a link will be made available in order to
download the associated archive.
Object type
Object id
Cooking status
diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html
index 8de05c5a7..6a4c39e15 100644
--- a/swh/web/templates/layout.html
+++ b/swh/web/templates/layout.html
@@ -1,68 +1,75 @@
+{% comment %}
+Copyright (C) 2015-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% load static %}
{% load render_bundle from webpack_loader %}
{% block title %}{% endblock %}
{% render_bundle 'vendors' %}
{% render_bundle 'webapp' %}
{% block header %}{% endblock %}
{% block content %}{% endblock %}
diff --git a/swh/web/templates/origin.html b/swh/web/templates/origin.html
index 5e7c6e812..2d95ba390 100644
--- a/swh/web/templates/origin.html
+++ b/swh/web/templates/origin.html
@@ -1,94 +1,101 @@
{% extends "browse.html" %}
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% load static %}
{% load swh_templatetags %}
{% load render_bundle from webpack_loader %}
{% block header %}
{{ block.super }}
{% render_bundle 'origin' %}
{% endblock %}
{% block swh-browse-before-panels %}
Total number of visits: {{ origin_visits|length }}
Last full visit:
First full visit:
Last visit:
Visits history
Timeline
Calendar
List
{% endblock %}
diff --git a/swh/web/templates/person.html b/swh/web/templates/person.html
index f40323ad8..8dfae303c 100644
--- a/swh/web/templates/person.html
+++ b/swh/web/templates/person.html
@@ -1 +1,9 @@
{% extends "browse.html" %}
+
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
diff --git a/swh/web/templates/release.html b/swh/web/templates/release.html
index bbe9f87dd..a51f28171 100644
--- a/swh/web/templates/release.html
+++ b/swh/web/templates/release.html
@@ -1,22 +1,29 @@
{% extends "browse.html" %}
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% load swh_templatetags %}
{% block swh-browse-before-panels %}
{% if snapshot_context %}
{% include "includes/snapshot-context.html" %}
{% endif %}
{% endblock %}
{% block swh-browse-main-panel-content %}
Release {{ swh_object_metadata.name }}
created by {{ swh_object_metadata.author }} on {{ swh_object_metadata.date }}
{{ release_note_header }}
{{ release_note_body }}
{{ release_target_link }}
{% endblock %}
diff --git a/swh/web/templates/releases.html b/swh/web/templates/releases.html
index 99ed6cd76..264366a23 100644
--- a/swh/web/templates/releases.html
+++ b/swh/web/templates/releases.html
@@ -1,59 +1,66 @@
{% extends "browse.html" %}
+{% comment %}
+Copyright (C) 2017-2018 The Software Heritage developers
+See the AUTHORS file at the top-level directory of this distribution
+License: GNU Affero General Public License version 3, or any later version
+See top-level LICENSE file for more information
+{% endcomment %}
+
{% block swh-browse-before-panels %}
{% if snapshot_context %}
{% include "includes/snapshot-context.html" %}
{% endif %}
{% endblock %}
{% block swh-browse-main-panel-content %}
',
count=len(origin_releases))
for release in origin_releases:
browse_release_url = reverse('browse-release',
kwargs={'sha1_git': release['id']},
query_params={'origin_type': stub_origin_info['type'],
'origin_url': stub_origin_info['url']})
self.assertContains(resp, '%s' % (escape(browse_release_url), release['name']))
diff --git a/swh/web/tests/browse/views/test_person.py b/swh/web/tests/browse/views/test_person.py
index d3cab7e60..dbc685b67 100644
--- a/swh/web/tests/browse/views/test_person.py
+++ b/swh/web/tests/browse/views/test_person.py
@@ -1,56 +1,56 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from unittest.mock import patch
from nose.tools import istest
from django.test import TestCase
from swh.web.common.exc import NotFoundExc
from swh.web.common.utils import reverse
from swh.web.tests.testbase import SWHWebTestBase
class SwhBrowsePersonTest(SWHWebTestBase, TestCase):
@patch('swh.web.browse.views.person.service')
@istest
def person_browse(self, mock_service):
test_person_data = \
{
"email": "j.adams440@gmail.com",
"fullname": "oysterCrusher ",
"id": 457587,
"name": "oysterCrusher"
}
mock_service.lookup_person.return_value = test_person_data
url = reverse('browse-person', kwargs={'person_id': 457587})
resp = self.client.get(url)
self.assertEquals(resp.status_code, 200)
self.assertTemplateUsed('person.html')
self.assertContains(resp, '
' %
escape(prev_page_url))
@patch('swh.web.browse.utils.service')
@patch('swh.web.browse.views.revision.service')
@istest
def revision_request_errors(self, mock_service, mock_utils_service):
mock_service.lookup_revision.side_effect = \
NotFoundExc('Revision not found')
url = reverse('browse-revision',
kwargs={'sha1_git': revision_id_test})
resp = self.client.get(url)
self.assertEquals(resp.status_code, 404)
self.assertTemplateUsed('error.html')
self.assertContains(resp, 'Revision not found', status_code=404)
mock_service.lookup_revision_log.side_effect = \
NotFoundExc('Revision not found')
url = reverse('browse-revision-log',
kwargs={'sha1_git': revision_id_test})
resp = self.client.get(url)
self.assertEquals(resp.status_code, 404)
self.assertTemplateUsed('error.html')
self.assertContains(resp, 'Revision not found', status_code=404)
url = reverse('browse-revision',
kwargs={'sha1_git': revision_id_test},
query_params={'origin_type': 'git',
'origin_url': 'https://github.com/foo/bar'})
mock_service.lookup_revision.side_effect = None
mock_utils_service.lookup_origin.side_effect = \
NotFoundExc('Origin not found')
resp = self.client.get(url)
self.assertEquals(resp.status_code, 404)
self.assertTemplateUsed('error.html')
self.assertContains(resp, 'Origin not found', status_code=404)
diff --git a/swh/web/tests/common/test_throttling.py b/swh/web/tests/common/test_throttling.py
index b5a764ea5..a4175cbaa 100644
--- a/swh/web/tests/common/test_throttling.py
+++ b/swh/web/tests/common/test_throttling.py
@@ -1,137 +1,137 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.web.settings.tests import (
scope1_limiter_rate, scope1_limiter_rate_post,
scope2_limiter_rate, scope2_limiter_rate_post,
scope3_limiter_rate, scope3_limiter_rate_post
)
from django.test import TestCase
from django.core.cache import cache
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework.test import APIRequestFactory
from rest_framework.decorators import api_view
from nose.tools import istest
from swh.web.common.throttling import (
SwhWebRateThrottle, throttle_scope
)
class MockViewScope1(APIView):
throttle_classes = (SwhWebRateThrottle,)
throttle_scope = 'scope1'
def get(self, request):
return Response('foo_get')
def post(self, request):
return Response('foo_post')
@api_view(['GET', 'POST'])
@throttle_scope('scope2')
def mock_view_scope2(request):
if request.method == 'GET':
return Response('bar_get')
elif request.method == 'POST':
return Response('bar_post')
class MockViewScope3(APIView):
throttle_classes = (SwhWebRateThrottle,)
throttle_scope = 'scope3'
def get(self, request):
return Response('foo_get')
def post(self, request):
return Response('foo_post')
@api_view(['GET', 'POST'])
@throttle_scope('scope3')
def mock_view_scope3(request):
if request.method == 'GET':
return Response('bar_get')
elif request.method == 'POST':
return Response('bar_post')
class ThrottlingTests(TestCase):
def setUp(self):
"""
Reset the cache so that no throttles will be active
"""
cache.clear()
self.factory = APIRequestFactory()
@istest
def scope1_requests_are_throttled(self):
"""
Ensure request rate is limited in scope1
"""
request = self.factory.get('/')
for _ in range(scope1_limiter_rate):
response = MockViewScope1.as_view()(request)
assert response.status_code == 200
response = MockViewScope1.as_view()(request)
assert response.status_code == 429
request = self.factory.post('/')
for _ in range(scope1_limiter_rate_post):
response = MockViewScope1.as_view()(request)
assert response.status_code == 200
response = MockViewScope1.as_view()(request)
assert response.status_code == 429
@istest
def scope2_requests_are_throttled(self):
"""
Ensure request rate is limited in scope2
"""
request = self.factory.get('/')
for _ in range(scope2_limiter_rate):
response = mock_view_scope2(request)
assert response.status_code == 200
response = mock_view_scope2(request)
assert response.status_code == 429
request = self.factory.post('/')
for _ in range(scope2_limiter_rate_post):
response = mock_view_scope2(request)
assert response.status_code == 200
response = mock_view_scope2(request)
assert response.status_code == 429
@istest
def scope3_requests_are_throttled_exempted(self):
"""
Ensure request rate is not limited in scope3 as
requests coming from localhost are exempted from rate limit.
"""
request = self.factory.get('/')
for _ in range(scope3_limiter_rate+1):
response = MockViewScope3.as_view()(request)
assert response.status_code == 200
request = self.factory.post('/')
for _ in range(scope3_limiter_rate_post+1):
response = MockViewScope3.as_view()(request)
assert response.status_code == 200
request = self.factory.get('/')
for _ in range(scope3_limiter_rate+1):
response = mock_view_scope3(request)
assert response.status_code == 200
request = self.factory.post('/')
for _ in range(scope3_limiter_rate_post+1):
response = mock_view_scope3(request)
assert response.status_code == 200
diff --git a/swh/web/tests/common/test_utils.py b/swh/web/tests/common/test_utils.py
index be9cb9d7b..8393b934a 100644
--- a/swh/web/tests/common/test_utils.py
+++ b/swh/web/tests/common/test_utils.py
@@ -1,107 +1,107 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import datetime
import unittest
from nose.tools import istest
from swh.web.common import utils
class UtilsTestCase(unittest.TestCase):
@istest
def shorten_path_noop(self):
noops = [
'/api/',
'/browse/',
'/content/symbol/foobar/'
]
for noop in noops:
self.assertEqual(
utils.shorten_path(noop),
noop
)
@istest
def shorten_path_sha1(self):
sha1 = 'aafb16d69fd30ff58afdd69036a26047f3aebdc6'
short_sha1 = sha1[:8] + '...'
templates = [
'/api/1/content/sha1:%s/',
'/api/1/content/sha1_git:%s/',
'/api/1/directory/%s/',
'/api/1/content/sha1:%s/ctags/',
]
for template in templates:
self.assertEqual(
utils.shorten_path(template % sha1),
template % short_sha1
)
@istest
def shorten_path_sha256(self):
sha256 = ('aafb16d69fd30ff58afdd69036a26047'
'213add102934013a014dfca031c41aef')
short_sha256 = sha256[:8] + '...'
templates = [
'/api/1/content/sha256:%s/',
'/api/1/directory/%s/',
'/api/1/content/sha256:%s/filetype/',
]
for template in templates:
self.assertEqual(
utils.shorten_path(template % sha256),
template % short_sha256
)
@istest
def parse_timestamp(self):
input_timestamps = [
None,
'2016-01-12',
'2016-01-12T09:19:12+0100',
'Today is January 1, 2047 at 8:21:00AM',
'1452591542',
]
output_dates = [
None,
datetime.datetime(2016, 1, 12, 0, 0),
datetime.datetime(2016, 1, 12, 8, 19, 12,
tzinfo=datetime.timezone.utc),
datetime.datetime(2047, 1, 1, 8, 21),
datetime.datetime(2016, 1, 12, 9, 39, 2,
tzinfo=datetime.timezone.utc),
]
for ts, exp_date in zip(input_timestamps, output_dates):
self.assertEquals(utils.parse_timestamp(ts), exp_date)
@istest
def format_utc_iso_date(self):
self.assertEqual(utils.format_utc_iso_date('2017-05-04T13:27:13+02:00'), # noqa
'04 May 2017, 11:27 UTC')
@istest
def gen_path_info(self):
input_path = '/home/user/swh-environment/swh-web/'
expected_result = [
{'name': 'home', 'path': 'home'},
{'name': 'user', 'path': 'home/user'},
{'name': 'swh-environment', 'path': 'home/user/swh-environment'},
{'name': 'swh-web', 'path': 'home/user/swh-environment/swh-web'}
]
path_info = utils.gen_path_info(input_path)
self.assertEquals(path_info, expected_result)
input_path = 'home/user/swh-environment/swh-web'
path_info = utils.gen_path_info(input_path)
self.assertEquals(path_info, expected_result)
diff --git a/swh/web/urls.py b/swh/web/urls.py
index 002e5a84d..7976eb814 100644
--- a/swh/web/urls.py
+++ b/swh/web/urls.py
@@ -1,47 +1,47 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.conf import settings
from django.conf.urls import (
url, include, handler400, handler403, handler404, handler500
)
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.contrib.staticfiles.views import serve
from django.shortcuts import render
from django.views.generic.base import RedirectView
from django_js_reverse.views import urls_js
from swh.web.common.exc import (
swh_handle400, swh_handle403, swh_handle404, swh_handle500
)
favicon_view = RedirectView.as_view(url='/static/img/icons/swh-logo-32x32.png',
permanent=True)
def default_view(request):
return render(request, "homepage.html")
urlpatterns = [
url(r'^favicon\.ico$', favicon_view),
url(r'^api/', include('swh.web.api.urls')),
url(r'^browse/', include('swh.web.browse.urls')),
url(r'^$', default_view, name='swh-web-homepage'),
url(r'^jsreverse/$', urls_js, name='js_reverse')
]
# enable to serve compressed assets through django development server
if settings.DEBUG:
static_pattern = r'^%s(?P.*)$' % settings.STATIC_URL[1:]
urlpatterns.append(url(static_pattern, serve))
else:
urlpatterns += staticfiles_urlpatterns()
handler400 = swh_handle400 # noqa
handler403 = swh_handle403 # noqa
handler404 = swh_handle404 # noqa
handler500 = swh_handle500 # noqa
diff --git a/swh/web/wsgi.py b/swh/web/wsgi.py
index 87f37a9e6..ffd4f9beb 100644
--- a/swh/web/wsgi.py
+++ b/swh/web/wsgi.py
@@ -1,12 +1,12 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
+# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "swh.web.settings.production")
application = get_wsgi_application()