diff --git a/.gitignore b/.gitignore index 185099d9..9e6b2f74 100644 --- a/.gitignore +++ b/.gitignore @@ -1,40 +1,40 @@ *.pyc *.sw? *~ \#* .\#* /.coverage /.coverage.* .eggs/ resources/test/ __pycache__ version.txt swh.web.egg-info docs/build/ docs/uri-scheme.md docs/dev-info.md *.sqlite3 .vscode/ .directory node_modules/ -swh/web/static/*.* -swh/web/static/js/ -swh/web/static/css/ -swh/web/static/fonts/ -swh/web/static/jssources/ -swh/web/static/img/thirdParty/ +static/*.* +static/js/ +static/css/ +static/fonts/ +static/jssources/ +static/img/thirdParty/ .cache-loader/ build/ dist/ .hypothesis .cache .pytest_cache .tox/ .mypy_cache/ debian/ package-lock.json yarn-error.log cypress/mochawesome/ .nyc_output/ cypress/coverage/ cypress/fixtures/source*.json diff --git a/MANIFEST.in b/MANIFEST.in index 57f60b4e..9d93da96 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,14 +1,13 @@ -include Makefile include pytest.ini include README.md include requirements*.txt include tox.ini include version.txt recursive-include swh py.typed + recursive-include swh/web/assets * -recursive-include swh/web/static * recursive-include swh/web/templates * recursive-include swh/web/tests/resources * include package.json include yarn.lock diff --git a/setup.py b/setup.py index 5640f9fc..2c293178 100755 --- a/setup.py +++ b/setup.py @@ -1,66 +1,75 @@ #!/usr/bin/env python3 -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License v3 or later # See top-level LICENSE file for more information +import os + from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements +# package generated static assets as module data files +data_files = [] +for root, _, files in os.walk('static/'): + root_files = [os.path.join(root, i) for i in files] + data_files.append((os.path.join('share/swh/web', root), root_files)) + setup( name='swh.web', description='Software Heritage Web UI', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DWUI/', packages=find_packages(), scripts=[], install_requires=parse_requirements() + parse_requirements('swh'), setup_requires=['vcversioner'], extras_require={'testing': parse_requirements('test')}, vcversioner={}, include_package_data=True, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", # noqa "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", "Framework :: Django", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-web', }, + data_files=data_files ) diff --git a/swh/web/static/img/arrow-up-small.png b/static/img/arrow-up-small.png similarity index 100% rename from swh/web/static/img/arrow-up-small.png rename to static/img/arrow-up-small.png diff --git a/swh/web/static/img/icons/swh-logo-32x32.png b/static/img/icons/swh-logo-32x32.png similarity index 100% rename from swh/web/static/img/icons/swh-logo-32x32.png rename to static/img/icons/swh-logo-32x32.png diff --git a/swh/web/static/img/icons/swh-logo-archive-180x180.png b/static/img/icons/swh-logo-archive-180x180.png similarity index 100% rename from swh/web/static/img/icons/swh-logo-archive-180x180.png rename to static/img/icons/swh-logo-archive-180x180.png diff --git a/swh/web/static/img/icons/swh-logo-archive-192x192.png b/static/img/icons/swh-logo-archive-192x192.png similarity index 100% rename from swh/web/static/img/icons/swh-logo-archive-192x192.png rename to static/img/icons/swh-logo-archive-192x192.png diff --git a/swh/web/static/img/icons/swh-logo-archive-270x270.png b/static/img/icons/swh-logo-archive-270x270.png similarity index 100% rename from swh/web/static/img/icons/swh-logo-archive-270x270.png rename to static/img/icons/swh-logo-archive-270x270.png diff --git a/swh/web/static/img/logos/bitbucket.png b/static/img/logos/bitbucket.png similarity index 100% rename from swh/web/static/img/logos/bitbucket.png rename to static/img/logos/bitbucket.png diff --git a/swh/web/static/img/logos/debian.png b/static/img/logos/debian.png similarity index 100% rename from swh/web/static/img/logos/debian.png rename to static/img/logos/debian.png diff --git a/swh/web/static/img/logos/framagit.png b/static/img/logos/framagit.png similarity index 100% rename from swh/web/static/img/logos/framagit.png rename to static/img/logos/framagit.png diff --git a/swh/web/static/img/logos/github.png b/static/img/logos/github.png similarity index 100% rename from swh/web/static/img/logos/github.png rename to static/img/logos/github.png diff --git a/swh/web/static/img/logos/gitlab.svg b/static/img/logos/gitlab.svg similarity index 100% rename from swh/web/static/img/logos/gitlab.svg rename to static/img/logos/gitlab.svg diff --git a/swh/web/static/img/logos/gitorious.png b/static/img/logos/gitorious.png similarity index 100% rename from swh/web/static/img/logos/gitorious.png rename to static/img/logos/gitorious.png diff --git a/swh/web/static/img/logos/gnu.png b/static/img/logos/gnu.png similarity index 100% rename from swh/web/static/img/logos/gnu.png rename to static/img/logos/gnu.png diff --git a/swh/web/static/img/logos/googlecode.png b/static/img/logos/googlecode.png similarity index 100% rename from swh/web/static/img/logos/googlecode.png rename to static/img/logos/googlecode.png diff --git a/swh/web/static/img/logos/hal.png b/static/img/logos/hal.png similarity index 100% rename from swh/web/static/img/logos/hal.png rename to static/img/logos/hal.png diff --git a/swh/web/static/img/logos/inria.jpg b/static/img/logos/inria.jpg similarity index 100% rename from swh/web/static/img/logos/inria.jpg rename to static/img/logos/inria.jpg diff --git a/swh/web/static/img/logos/npm.png b/static/img/logos/npm.png similarity index 100% rename from swh/web/static/img/logos/npm.png rename to static/img/logos/npm.png diff --git a/swh/web/static/img/logos/pypi.svg b/static/img/logos/pypi.svg similarity index 100% rename from swh/web/static/img/logos/pypi.svg rename to static/img/logos/pypi.svg diff --git a/swh/web/static/img/swh-api.png b/static/img/swh-api.png similarity index 100% rename from swh/web/static/img/swh-api.png rename to static/img/swh-api.png diff --git a/swh/web/static/img/swh-browse.png b/static/img/swh-browse.png similarity index 100% rename from swh/web/static/img/swh-browse.png rename to static/img/swh-browse.png diff --git a/swh/web/static/img/swh-logo-white.svg b/static/img/swh-logo-white.svg similarity index 100% rename from swh/web/static/img/swh-logo-white.svg rename to static/img/swh-logo-white.svg diff --git a/swh/web/static/img/swh-logo.png b/static/img/swh-logo.png similarity index 100% rename from swh/web/static/img/swh-logo.png rename to static/img/swh-logo.png diff --git a/swh/web/static/img/swh-logo.svg b/static/img/swh-logo.svg similarity index 100% rename from swh/web/static/img/swh-logo.svg rename to static/img/swh-logo.svg diff --git a/swh/web/static/img/swh-spinner-small.gif b/static/img/swh-spinner-small.gif similarity index 100% rename from swh/web/static/img/swh-spinner-small.gif rename to static/img/swh-spinner-small.gif diff --git a/swh/web/static/img/swh-spinner.gif b/static/img/swh-spinner.gif similarity index 100% rename from swh/web/static/img/swh-spinner.gif rename to static/img/swh-spinner.gif diff --git a/swh/web/static/img/swh-support.png b/static/img/swh-support.png similarity index 100% rename from swh/web/static/img/swh-support.png rename to static/img/swh-support.png diff --git a/swh/web/static/img/swh-vault.png b/static/img/swh-vault.png similarity index 100% rename from swh/web/static/img/swh-vault.png rename to static/img/swh-vault.png diff --git a/swh/web/static/xml/swh-opensearch.xml b/static/xml/swh-opensearch.xml similarity index 100% rename from swh/web/static/xml/swh-opensearch.xml rename to static/xml/swh-opensearch.xml diff --git a/swh/web/assets/config/webpack.config.development.js b/swh/web/assets/config/webpack.config.development.js index 7751a054..8edef8b2 100644 --- a/swh/web/assets/config/webpack.config.development.js +++ b/swh/web/assets/config/webpack.config.development.js @@ -1,438 +1,438 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // webpack configuration for compiling static assets in development mode // import required node modules and webpack plugins const chalk = require('chalk'); const fs = require('fs'); const path = require('path'); const webpack = require('webpack'); const BundleTracker = require('webpack-bundle-tracker'); const RobotstxtPlugin = require('robotstxt-webpack-plugin'); const CleanWebpackPlugin = require('clean-webpack-plugin').CleanWebpackPlugin; const MiniCssExtractPlugin = require('mini-css-extract-plugin'); const FixSwhSourceMapsPlugin = require('./webpack-plugins/fix-swh-source-maps-webpack-plugin'); const CopyWebpackPlugin = require('copy-webpack-plugin'); const GenerateWebLabelsPlugin = require('./webpack-plugins/generate-weblabels-webpack-plugin'); const ProgressBarPlugin = require('progress-bar-webpack-plugin'); const loadedMathJaxJsFiles = require('./mathjax-js-files'); // are we running webpack-dev-server ? const isDevServer = process.argv.find(v => v.includes('webpack-dev-server')); // webpack-dev-server configuration const devServerPort = 3000; const devServerPublicPath = 'http://localhost:' + devServerPort + '/static/'; // set publicPath according if we are using webpack-dev-server to serve // our assets or not const publicPath = isDevServer ? devServerPublicPath : '/static/'; const nodeModules = path.resolve(__dirname, '../../../../node_modules/'); // collect all bundles we want to produce with webpack var bundles = {}; const bundlesDir = path.join(__dirname, '../src/bundles'); fs.readdirSync(bundlesDir).forEach(file => { bundles[file] = ['bundles/' + file + '/index.js']; }); // common loaders for css related assets (css, sass, less) let cssLoaders = [ MiniCssExtractPlugin.loader, { loader: 'cache-loader' }, { loader: 'css-loader', options: { sourceMap: !isDevServer } }, { loader: 'postcss-loader', options: { ident: 'postcss', sourceMap: !isDevServer, plugins: [ // lint swh-web stylesheets require('stylelint')({ 'config': { 'extends': 'stylelint-config-standard', 'rules': { 'indentation': 4, 'font-family-no-missing-generic-family-keyword': null, 'no-descending-specificity': null }, 'ignoreFiles': ['node_modules/**/*.css', 'swh/web/assets/src/thirdparty/**/*.css'] } }), // automatically add vendor prefixes to css rules require('autoprefixer')(), require('postcss-normalize')(), require('postcss-reporter')({ clearReportedMessages: true }) ] } } ]; // webpack development configuration module.exports = { // use caching to speedup incremental builds cache: true, // set mode to development mode: 'development', // use eval source maps when using webpack-dev-server for quick debugging, // otherwise generate source map files (more expensive) devtool: isDevServer ? 'eval' : 'source-map', // webpack-dev-server configuration devServer: { clientLogLevel: 'warning', port: devServerPort, publicPath: devServerPublicPath, // enable to serve static assets not managed by webpack contentBase: path.resolve('./swh/web/'), // we do not use hot reloading here (as a framework like React needs to be used in order to fully benefit from that feature) // and prefer to fully reload the frontend application in the browser instead hot: false, inline: true, historyApiFallback: true, headers: { 'Access-Control-Allow-Origin': '*' }, compress: true, stats: 'errors-only', overlay: { warnings: true, errors: true } }, // set entries to the bundles we want to produce entry: bundles, // assets output configuration output: { - path: path.resolve('./swh/web/static/'), + path: path.resolve('./static/'), filename: 'js/[name].[chunkhash].js', chunkFilename: 'js/[name].[chunkhash].js', publicPath: publicPath, // each bundle will be compiled as a umd module with its own namespace // in order to easily use them in django templates library: ['swh', '[name]'], libraryTarget: 'umd' }, // module resolving configuration resolve: { // alias pdfjs to its minified version alias: { 'pdfjs-dist': 'pdfjs-dist/build/pdf.min.js' }, // configure base paths for resolving modules with webpack modules: [ 'node_modules', path.resolve(__dirname, '../src') ] }, stats: 'errors-warnings', // module import configuration module: { rules: [ { // Preprocess all js files with eslint for consistent code style // and avoid bad js development practices. enforce: 'pre', test: /\.js$/, exclude: /node_modules/, use: [{ loader: 'eslint-loader', options: { configFile: path.join(__dirname, '.eslintrc'), ignorePath: path.join(__dirname, '.eslintignore'), cache: true, emitWarning: true } }] }, { // Use babel-loader in order to use es6 syntax in js files // but also advanced js features like async/await syntax. // All code get transpiled to es5 in order to be executed // in a large majority of browsers. test: /\.js$/, exclude: /node_modules/, use: [ { loader: 'cache-loader' }, { loader: 'babel-loader', options: { presets: [ // use env babel presets to benefit from es6 syntax ['@babel/preset-env', { // Do not transform es6 module syntax to another module type // in order to benefit from dead code elimination (aka tree shaking) // when running webpack in production mode 'loose': true, 'modules': false }] ], plugins: [ // use babel transform-runtime plugin in order to use aync/await syntax ['@babel/plugin-transform-runtime', { 'regenerator': true }], // use other babel plugins to benefit from advanced js features (es2017) '@babel/plugin-syntax-dynamic-import' ], env: { test: { plugins: ['istanbul'] } } } }] }, // expose jquery to the global context as $ and jQuery when importing it { test: require.resolve('jquery'), use: [{ loader: 'expose-loader', options: 'jQuery' }, { loader: 'expose-loader', options: '$' }] }, // expose highlightjs to the global context as hljs when importing it { test: require.resolve('highlight.js'), use: [{ loader: 'expose-loader', options: 'hljs' }] }, { test: require.resolve('js-cookie'), use: [{ loader: 'expose-loader', options: 'Cookies' }] }, // css import configuration: // - first process it with postcss // - then extract it to a dedicated file associated to each bundle { test: /\.css$/, use: cssLoaders }, // sass import configuration: // - generate css with sass-loader // - process it with postcss // - then extract it to a dedicated file associated to each bundle { test: /\.scss$/, use: cssLoaders.concat([ { loader: 'sass-loader', options: { sourceMap: !isDevServer } } ]) }, // less import configuration: // - generate css with less-loader // - process it with postcss // - then extract it to a dedicated file associated to each bundle { test: /\.less$/, use: cssLoaders.concat([ { loader: 'less-loader', options: { sourceMap: !isDevServer } } ]) }, // web fonts import configuration { test: /\.(woff|woff2)(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] }, { test: /\.ttf(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] }, { test: /\.eot(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] }, { test: /\.svg(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] }, { test: /\.otf(\?v=\d+\.\d+\.\d+)?$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'fonts/' } }] }, { test: /\.png$/, use: [{ loader: 'file-loader', options: { name: '[name].[ext]', outputPath: 'img/thirdParty/' } }] } ], // tell webpack to not parse minified pdfjs file to speedup build process noParse: [path.resolve(nodeModules, 'pdfjs-dist/build/pdf.min.js')] }, // webpack plugins plugins: [ // cleanup previously generated assets new CleanWebpackPlugin({ cleanOnceBeforeBuildPatterns: ['**/*', '!xml', '!xml/*', '!img', '!img/*', '!img/logos', '!img/logos/*', '!img/icons', '!img/icons/*'] }), // needed in order to use django_webpack_loader new BundleTracker({ - filename: './swh/web/static/webpack-stats.json' + filename: './static/webpack-stats.json' }), // for generating the robots.txt file new RobotstxtPlugin({ policy: [{ userAgent: '*', disallow: '/api/' }] }), // for extracting all stylesheets in separate css files new MiniCssExtractPlugin({ filename: 'css/[name].[chunkhash].css', chunkFilename: 'css/[name].[chunkhash].css' }), // fix generated asset sourcemaps to workaround a Firefox issue new FixSwhSourceMapsPlugin(), // define some global variables accessible from js code new webpack.DefinePlugin({ __STATIC__: JSON.stringify(publicPath) }), // needed in order to use bootstrap 4.x new webpack.ProvidePlugin({ Popper: ['popper.js', 'default'], Alert: 'exports-loader?Alert!bootstrap/js/dist/alert', Button: 'exports-loader?Button!bootstrap/js/dist/button', Carousel: 'exports-loader?Carousel!bootstrap/js/dist/carousel', Collapse: 'exports-loader?Collapse!bootstrap/js/dist/collapse', Dropdown: 'exports-loader?Dropdown!bootstrap/js/dist/dropdown', Modal: 'exports-loader?Modal!bootstrap/js/dist/modal', Popover: 'exports-loader?Popover!bootstrap/js/dist/popover', Scrollspy: 'exports-loader?Scrollspy!bootstrap/js/dist/scrollspy', Tab: 'exports-loader?Tab!bootstrap/js/dist/tab', Tooltip: 'exports-loader?Tooltip!bootstrap/js/dist/tooltip', Util: 'exports-loader?Util!bootstrap/js/dist/util' }), // needed in order to use pdf.js new webpack.IgnorePlugin(/^\.\/pdf.worker.js$/), new CopyWebpackPlugin([{ from: path.resolve(nodeModules, 'pdfjs-dist/build/pdf.worker.min.js'), - to: path.resolve(__dirname, '../../static/js/') + to: path.resolve(__dirname, '../../../../static/js/') }]), new GenerateWebLabelsPlugin({ outputType: 'json', exclude: ['mini-css-extract-plugin', 'bootstrap-loader'], srcReplace: { './node_modules/pdfjs-dist/build/pdf.min.js': './node_modules/pdfjs-dist/build/pdf.js', './node_modules/admin-lte/dist/js/adminlte.min.js': './node_modules/admin-lte/dist/js/adminlte.js' }, licenseOverride: { './swh/web/assets/src/thirdparty/jquery.tabSlideOut/jquery.tabSlideOut.js': { 'spdxLicenseExpression': 'GPL-3.0', 'licenseFilePath': './swh/web/assets/src/thirdparty/jquery.tabSlideOut/LICENSE' } }, additionalScripts: Object.assign( { 'js/pdf.worker.min.js': [ { 'id': 'pdfjs-dist/build/pdf.worker.js', 'path': './node_modules/pdfjs-dist/build/pdf.worker.js', 'spdxLicenseExpression': 'Apache-2.0', 'licenseFilePath': './node_modules/pdfjs-dist/LICENSE' } ], '/jsreverse/': [ { 'id': 'jsreverse', 'path': '/jsreverse/', 'spdxLicenseExpression': 'AGPL-3.0-or-later', 'licenseFilePath': './LICENSE' } ] }, loadedMathJaxJsFiles ) }), new ProgressBarPlugin({ format: chalk.cyan.bold('webpack build of swh-web assets') + ' [:bar] ' + chalk.green.bold(':percent') + ' (:elapsed seconds)', width: 50 }) ], // webpack optimizations optimization: { // ensure the vendors bundle gets emitted in a single chunk splitChunks: { cacheGroups: { vendors: { test: 'vendors', chunks: 'all', name: 'vendors', enforce: true } } } }, // disable webpack warnings about bundle sizes performance: { hints: false } }; diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py index 0f7f8654..4e8ba26f 100644 --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -1,284 +1,288 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django common settings for swh-web. """ import os +import sys from swh.web.config import get_config swh_web_config = get_config() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = swh_web_config['secret_key'] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = swh_web_config['debug'] DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config['debug'] ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] + swh_web_config['allowed_hosts'] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', 'swh.web.common', 'swh.web.api', 'swh.web.browse', 'webpack_loader', 'django_js_reverse', 'corsheaders' ] MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'corsheaders.middleware.CorsMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'swh.web.common.middlewares.ThrottlingHeadersMiddleware' ] # Compress all assets (static ones and dynamically generated html) # served by django in a local development environment context. # In a production environment, assets compression will be directly # handled by web servers like apache or nginx. if swh_web_config['serve_assets']: MIDDLEWARE.insert(0, 'django.middleware.gzip.GZipMiddleware') ROOT_URLCONF = 'swh.web.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [os.path.join(PROJECT_DIR, "../templates")], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', 'swh.web.common.utils.context_processor' ], 'libraries': { 'swh_templatetags': 'swh.web.common.swh_templatetags', }, }, }, ] DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': swh_web_config['development_db'], } } # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa }, ] # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' -STATICFILES_DIRS = [ - os.path.join(PROJECT_DIR, "../static") -] +# static folder location when swh-web has been installed with pip +STATIC_DIR = os.path.join(sys.prefix, 'share/swh/web/static') +if not os.path.exists(STATIC_DIR): + # static folder location when developping swh-web + STATIC_DIR = os.path.join(PROJECT_DIR, '../../../static') +STATICFILES_DIRS = [STATIC_DIR] INTERNAL_IPS = ['127.0.0.1'] throttle_rates = {} http_requests = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'] throttling = swh_web_config['throttling'] for limiter_scope, limiter_conf in throttling['scopes'].items(): if 'default' in limiter_conf['limiter_rate']: throttle_rates[limiter_scope] = limiter_conf['limiter_rate']['default'] # for backward compatibility else: throttle_rates[limiter_scope] = limiter_conf['limiter_rate'] # register sub scopes specific for HTTP request types for http_request in http_requests: if http_request in limiter_conf['limiter_rate']: throttle_rates[limiter_scope + '_' + http_request.lower()] = \ limiter_conf['limiter_rate'][http_request] REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', 'swh.web.api.renderers.YAMLRenderer', 'rest_framework.renderers.TemplateHTMLRenderer' ), 'DEFAULT_THROTTLE_CLASSES': ( 'swh.web.common.throttling.SwhWebRateThrottle', ), 'DEFAULT_THROTTLE_RATES': throttle_rates } LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse', }, 'require_debug_true': { '()': 'django.utils.log.RequireDebugTrue', }, }, 'formatters': { 'request': { 'format': '[%(asctime)s] [%(levelname)s] %(request)s %(status_code)s', # noqa 'datefmt': "%d/%b/%Y %H:%M:%S" }, 'simple': { 'format': '[%(asctime)s] [%(levelname)s] %(message)s', 'datefmt': "%d/%b/%Y %H:%M:%S" }, 'verbose': { 'format': '[%(asctime)s] [%(levelname)s] %(name)s.%(funcName)s:%(lineno)s - %(message)s', # noqa 'datefmt': "%d/%b/%Y %H:%M:%S" }, }, 'handlers': { 'console': { 'level': 'DEBUG', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', 'formatter': 'simple' }, 'file': { 'level': 'WARNING', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), 'formatter': 'simple' }, 'file_request': { 'level': 'WARNING', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), 'formatter': 'request' }, 'console_verbose': { 'level': 'DEBUG', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', 'formatter': 'verbose' }, 'file_verbose': { 'level': 'WARNING', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), 'formatter': 'verbose' }, 'null': { 'class': 'logging.NullHandler', }, }, 'loggers': { '': { 'handlers': ['console_verbose', 'file_verbose'], 'level': 'DEBUG' if DEBUG else 'WARNING', }, 'django': { 'handlers': ['console'], 'level': 'DEBUG' if DEBUG else 'WARNING', 'propagate': False, }, 'django.request': { 'handlers': ['file_request'], 'level': 'DEBUG' if DEBUG else 'WARNING', 'propagate': False, }, 'django.db.backends': { 'handlers': ['null'], 'propagate': False }, }, } -WEBPACK_LOADER = { # noqa +WEBPACK_LOADER = { 'DEFAULT': { 'CACHE': False, 'BUNDLE_DIR_NAME': './', - 'STATS_FILE': os.path.join(PROJECT_DIR, '../static/webpack-stats.json'), # noqa + 'STATS_FILE': os.path.join(STATIC_DIR, 'webpack-stats.json'), 'POLL_INTERVAL': 0.1, 'TIMEOUT': None, - 'IGNORE': ['.+\.hot-update.js', '.+\.map'] + 'IGNORE': ['.+\\.hot-update.js', '.+\\.map'] } } LOGIN_URL = '/admin/login/' LOGIN_REDIRECT_URL = 'admin' SESSION_ENGINE = 'django.contrib.sessions.backends.cache' CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache' }, 'db_cache': { 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'swh_web_cache', } } JS_REVERSE_JS_MINIFY = False CORS_ORIGIN_ALLOW_ALL = True CORS_URLS_REGEX = r'^/badge/.*$' diff --git a/swh/web/tests/conftest.py b/swh/web/tests/conftest.py index 1c01295d..44b8646d 100644 --- a/swh/web/tests/conftest.py +++ b/swh/web/tests/conftest.py @@ -1,301 +1,307 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import inspect import json import os import shutil +import sys from subprocess import run, PIPE import pytest from django.core.cache import cache from hypothesis import settings, HealthCheck from rest_framework.test import APIClient, APIRequestFactory from swh.model.hashutil import ALGORITHMS, hash_to_bytes from swh.web.common import converters from swh.web.tests.data import get_tests_data, override_storages # Used to skip some tests ctags_json_missing = ( shutil.which('ctags') is None or b'+json' not in run(['ctags', '--version'], stdout=PIPE).stdout ) fossology_missing = shutil.which('nomossa') is None # Register some hypothesis profiles settings.register_profile('default', settings()) settings.register_profile( 'swh-web', settings(deadline=None, suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much])) settings.register_profile( 'swh-web-fast', settings(deadline=None, max_examples=1, suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much])) def pytest_configure(config): # Small hack in order to be able to run the unit tests # without static assets generated by webpack. # Those assets are not really needed for the Python tests # but the django templates will fail to load due to missing # generated file webpack-stats.json describing the js and css # files to include. # So generate a dummy webpack-stats.json file to overcome # that issue. test_dir = os.path.dirname(__file__) - static_dir = os.path.join(test_dir, '../static') + # location of the static folder when running tests through tox + static_dir = os.path.join(sys.prefix, 'share/swh/web/static') + + if not os.path.exists(static_dir): + # location of the static folder when running tests locally with pytest + static_dir = os.path.join(test_dir, '../../../static') webpack_stats = os.path.join(static_dir, 'webpack-stats.json') if os.path.exists(webpack_stats): return bundles_dir = os.path.join(test_dir, '../assets/src/bundles') _, dirs, _ = next(os.walk(bundles_dir)) mock_webpack_stats = { 'status': 'done', 'publicPath': '/static', 'chunks': {} } for bundle in dirs: asset = 'js/%s.js' % bundle mock_webpack_stats['chunks'][bundle] = [{ 'name': asset, 'publicPath': '/static/%s' % asset, 'path': os.path.join(static_dir, asset) }] with open(webpack_stats, 'w') as outfile: json.dump(mock_webpack_stats, outfile) # Clear Django cache before each test @pytest.fixture(autouse=True) def django_cache_cleared(): cache.clear() # Alias rf fixture from pytest-django @pytest.fixture def request_factory(rf): return rf # Fixture to get test client from Django REST Framework @pytest.fixture(scope='module') def api_client(): return APIClient() # Fixture to get API request factory from Django REST Framework @pytest.fixture(scope='module') def api_request_factory(): return APIRequestFactory() # Initialize tests data @pytest.fixture(autouse=True) def tests_data(): data = get_tests_data(reset=True) # Update swh-web configuration to use the in-memory storages # instantiated in the tests.data module override_storages(data['storage'], data['idx_storage'], data['search']) return data # Fixture to manipulate data from a sample archive used in the tests @pytest.fixture def archive_data(tests_data): return _ArchiveData(tests_data) # Fixture to manipulate indexer data from a sample archive used in the tests @pytest.fixture def indexer_data(tests_data): return _IndexerData(tests_data) # Custom data directory for requests_mock @pytest.fixture def datadir(): return os.path.join(os.path.abspath(os.path.dirname(__file__)), 'resources') class _ArchiveData: """ Helper class to manage data from a sample test archive. It is initialized with a reference to an in-memory storage containing raw tests data. It is basically a proxy to Storage interface but it overrides some methods to retrieve those tests data in a json serializable format in order to ease tests implementation. """ def __init__(self, tests_data): self.storage = tests_data['storage'] def _call_storage_method(method): def call_storage_method(*args, **kwargs): return method(*args, **kwargs) return call_storage_method # Forward calls to non overridden Storage methods to wrapped # storage instance for method_name, method in inspect.getmembers( self.storage, predicate=inspect.ismethod): if (not hasattr(self, method_name) and not method_name.startswith('_')): setattr(self, method_name, _call_storage_method(method)) def content_find(self, content): cnt_ids_bytes = {algo_hash: hash_to_bytes(content[algo_hash]) for algo_hash in ALGORITHMS if content.get(algo_hash)} cnt = self.storage.content_find(cnt_ids_bytes) return converters.from_content(cnt[0]) if cnt else cnt def content_get_metadata(self, cnt_id): cnt_id_bytes = hash_to_bytes(cnt_id) metadata = next(self.storage.content_get_metadata([cnt_id_bytes])) return converters.from_swh(metadata, hashess={'sha1', 'sha1_git', 'sha256', 'blake2s256'}) def content_get(self, cnt_id): cnt_id_bytes = hash_to_bytes(cnt_id) cnt = next(self.storage.content_get([cnt_id_bytes])) return converters.from_content(cnt) def directory_get(self, dir_id): return { 'id': dir_id, 'content': self.directory_ls(dir_id) } def directory_ls(self, dir_id): cnt_id_bytes = hash_to_bytes(dir_id) dir_content = map(converters.from_directory_entry, self.storage.directory_ls(cnt_id_bytes)) return list(dir_content) def release_get(self, rel_id): rel_id_bytes = hash_to_bytes(rel_id) rel_data = next(self.storage.release_get([rel_id_bytes])) return converters.from_release(rel_data) def revision_get(self, rev_id): rev_id_bytes = hash_to_bytes(rev_id) rev_data = next(self.storage.revision_get([rev_id_bytes])) return converters.from_revision(rev_data) def revision_log(self, rev_id, limit=None): rev_id_bytes = hash_to_bytes(rev_id) return list(map(converters.from_revision, self.storage.revision_log([rev_id_bytes], limit=limit))) def snapshot_get_latest(self, origin_url): snp = self.storage.snapshot_get_latest(origin_url) return converters.from_snapshot(snp) def origin_get(self, origin_info): origin = self.storage.origin_get(origin_info) return converters.from_origin(origin) def origin_visit_get(self, origin_url): visits = self.storage.origin_visit_get(origin_url) return list(map(converters.from_origin_visit, visits)) def origin_visit_get_by(self, origin_url, visit_id): visit = self.storage.origin_visit_get_by(origin_url, visit_id) return converters.from_origin_visit(visit) def snapshot_get(self, snapshot_id): snp = self.storage.snapshot_get(hash_to_bytes(snapshot_id)) return converters.from_snapshot(snp) def snapshot_get_branches(self, snapshot_id, branches_from='', branches_count=1000, target_types=None): snp = self.storage.snapshot_get_branches( hash_to_bytes(snapshot_id), branches_from.encode(), branches_count, target_types) return converters.from_snapshot(snp) def snapshot_get_head(self, snapshot): if snapshot['branches']['HEAD']['target_type'] == 'alias': target = snapshot['branches']['HEAD']['target'] head = snapshot['branches'][target]['target'] else: head = snapshot['branches']['HEAD']['target'] return head class _IndexerData: """ Helper class to manage indexer tests data It is initialized with a reference to an in-memory indexer storage containing raw tests data. It also defines class methods to retrieve those tests data in a json serializable format in order to ease tests implementation. """ def __init__(self, tests_data): self.idx_storage = tests_data['idx_storage'] self.mimetype_indexer = tests_data['mimetype_indexer'] self.license_indexer = tests_data['license_indexer'] self.ctags_indexer = tests_data['ctags_indexer'] def content_add_mimetype(self, cnt_id): self.mimetype_indexer.run([hash_to_bytes(cnt_id)], 'update-dups') def content_get_mimetype(self, cnt_id): mimetype = next(self.idx_storage.content_mimetype_get( [hash_to_bytes(cnt_id)])) return converters.from_filetype(mimetype) def content_add_language(self, cnt_id): raise NotImplementedError('Language indexer is disabled.') self.language_indexer.run([hash_to_bytes(cnt_id)], 'update-dups') def content_get_language(self, cnt_id): lang = next(self.idx_storage.content_language_get( [hash_to_bytes(cnt_id)])) return converters.from_swh(lang, hashess={'id'}) def content_add_license(self, cnt_id): self.license_indexer.run([hash_to_bytes(cnt_id)], 'update-dups') def content_get_license(self, cnt_id): cnt_id_bytes = hash_to_bytes(cnt_id) lic = next(self.idx_storage.content_fossology_license_get( [cnt_id_bytes])) return converters.from_swh({'id': cnt_id_bytes, 'facts': lic[cnt_id_bytes]}, hashess={'id'}) def content_add_ctags(self, cnt_id): self.ctags_indexer.run([hash_to_bytes(cnt_id)], 'update-dups') def content_get_ctags(self, cnt_id): cnt_id_bytes = hash_to_bytes(cnt_id) ctags = self.idx_storage.content_ctags_get([cnt_id_bytes]) for ctag in ctags: yield converters.from_swh(ctag, hashess={'id'})