diff --git a/README.dev b/README.dev index d6c2c7b7..ba833e99 100644 --- a/README.dev +++ b/README.dev @@ -1,77 +1,77 @@ README.dev ========== A test server could locally be running for tests. # Sample configuration In either /etc/softwareheritage/storage/storage.yml, ~/.config/swh/storage.yml or ~/.swh/storage.yml: ``` storage: cls: local args: db: "dbname=softwareheritage-dev user=" objstorage: cls: pathslicing args: root: /home/storage/swh-storage/ slicing: 0:2/2:4/4:6 ``` which means, this uses: - a local storage instance whose db connection is to softwareheritage-dev local instance - the objstorage uses a local objstorage instance whose: - root path is /home/storage/swh-storage - slicing scheme is 0:2/2:4/4:6. This means that the identifier of the content (sha1) which will be stored on disk at first level with the first 2 hex characters, the second level with the next 2 hex characters and the third level with the next 2 hex characters. And finally the complete hash file holding the raw content. For example: 00062f8bd330715c4f819373653d97b3cd34394c will be stored at 00/06/2f/00062f8bd330715c4f819373653d97b3cd34394c Note that the 'root' path should exist on disk. # Run server Command: ``` python3 -m swh.storage.api.server ~/.config/swh/storage.yml ``` -This runs a local swh-storage api at 5000 port. +This runs a local swh-storage api at 5002 port. # And then what? In your upper layer (loader-git, loader-svn, etc...), you can define a remote storage with this snippet of yaml configuration. ``` storage: cls: remote args: - url: http://localhost:5000/ + url: http://localhost:5002/ ``` You could directly define a local storage with the following snippet: ``` storage: cls: local args: db: service=swh-dev objstorage: cls: pathslicing args: root: /home/storage/swh-storage/ slicing: 0:2/2:4/4:6 ``` diff --git a/swh/storage/api/server.py b/swh/storage/api/server.py index 1618ef21..88b3db4a 100644 --- a/swh/storage/api/server.py +++ b/swh/storage/api/server.py @@ -1,409 +1,409 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json import logging import click from flask import g, request from swh.core import config from swh.storage import get_storage from swh.core.api import (SWHServerAPIApp, decode_request, error_handler, encode_data_server as encode_data) DEFAULT_CONFIG = { 'storage': ('dict', { 'cls': 'local', 'args': { 'db': 'dbname=softwareheritage-dev', 'objstorage': { 'cls': 'pathslicing', 'args': { 'root': '/srv/softwareheritage/objects', 'slicing': '0:2/2:4/4:6', }, }, }, }) } app = SWHServerAPIApp(__name__) @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) @app.before_request def before_request(): g.storage = get_storage(**app.config['storage']) @app.route('/') def index(): return 'SWH Storage API server' @app.route('/check_config', methods=['POST']) def check_config(): return encode_data(g.storage.check_config(**decode_request(request))) @app.route('/content/missing', methods=['POST']) def content_missing(): return encode_data(g.storage.content_missing(**decode_request(request))) @app.route('/content/missing/sha1', methods=['POST']) def content_missing_per_sha1(): return encode_data(g.storage.content_missing_per_sha1( **decode_request(request))) @app.route('/content/present', methods=['POST']) def content_find(): return encode_data(g.storage.content_find(**decode_request(request))) @app.route('/content/provenance', methods=['POST']) def content_find_provenance(): res = g.storage.content_find_provenance(**decode_request(request)) return encode_data(res) @app.route('/content/add', methods=['POST']) def content_add(): return encode_data(g.storage.content_add(**decode_request(request))) @app.route('/content/data', methods=['POST']) def content_get(): return encode_data(g.storage.content_get(**decode_request(request))) @app.route('/content/metadata', methods=['POST']) def content_get_metadata(): return encode_data(g.storage.content_get_metadata( **decode_request(request))) @app.route('/directory', methods=['POST']) def directory_get(): return encode_data(g.storage.directory_get(**decode_request(request))) @app.route('/directory/missing', methods=['POST']) def directory_missing(): return encode_data(g.storage.directory_missing(**decode_request(request))) @app.route('/directory/add', methods=['POST']) def directory_add(): return encode_data(g.storage.directory_add(**decode_request(request))) @app.route('/directory/path', methods=['POST']) def directory_entry_get_by_path(): return encode_data(g.storage.directory_entry_get_by_path( **decode_request(request))) @app.route('/directory/ls', methods=['GET']) def directory_ls(): dir = request.args['directory'].encode('utf-8', 'surrogateescape') rec = json.loads(request.args.get('recursive', 'False').lower()) return encode_data(g.storage.directory_ls(dir, recursive=rec)) @app.route('/revision/add', methods=['POST']) def revision_add(): return encode_data(g.storage.revision_add(**decode_request(request))) @app.route('/revision', methods=['POST']) def revision_get(): return encode_data(g.storage.revision_get(**decode_request(request))) @app.route('/revision/by', methods=['POST']) def revision_get_by(): return encode_data(g.storage.revision_get_by(**decode_request(request))) @app.route('/revision/log', methods=['POST']) def revision_log(): return encode_data(g.storage.revision_log(**decode_request(request))) @app.route('/revision/logby', methods=['POST']) def revision_log_by(): return encode_data(g.storage.revision_log_by(**decode_request(request))) @app.route('/revision/shortlog', methods=['POST']) def revision_shortlog(): return encode_data(g.storage.revision_shortlog(**decode_request(request))) @app.route('/revision/missing', methods=['POST']) def revision_missing(): return encode_data(g.storage.revision_missing(**decode_request(request))) @app.route('/cache/content_revision', methods=['POST']) def cache_content_revision_add(): return encode_data(g.storage.cache_content_revision_add( **decode_request(request))) @app.route('/cache/contents', methods=['GET']) def cache_content_get_all(): return encode_data(g.storage.cache_content_get_all()) @app.route('/cache/content', methods=['POST']) def cache_content_get(): return encode_data(g.storage.cache_content_get( **decode_request(request))) @app.route('/cache/revision_origin', methods=['POST']) def cache_revision_origin_add(): return encode_data(g.storage.cache_revision_origin_add( **decode_request(request))) @app.route('/release/add', methods=['POST']) def release_add(): return encode_data(g.storage.release_add(**decode_request(request))) @app.route('/release', methods=['POST']) def release_get(): return encode_data(g.storage.release_get(**decode_request(request))) @app.route('/release/by', methods=['POST']) def release_get_by(): return encode_data(g.storage.release_get_by(**decode_request(request))) @app.route('/release/missing', methods=['POST']) def release_missing(): return encode_data(g.storage.release_missing(**decode_request(request))) @app.route('/object/find_by_sha1_git', methods=['POST']) def object_find_by_sha1_git(): return encode_data(g.storage.object_find_by_sha1_git( **decode_request(request))) @app.route('/occurrence', methods=['POST']) def occurrence_get(): return encode_data(g.storage.occurrence_get(**decode_request(request))) @app.route('/occurrence/add', methods=['POST']) def occurrence_add(): return encode_data(g.storage.occurrence_add(**decode_request(request))) @app.route('/origin/get', methods=['POST']) def origin_get(): return encode_data(g.storage.origin_get(**decode_request(request))) @app.route('/origin/add_multi', methods=['POST']) def origin_add(): return encode_data(g.storage.origin_add(**decode_request(request))) @app.route('/origin/add', methods=['POST']) def origin_add_one(): return encode_data(g.storage.origin_add_one(**decode_request(request))) @app.route('/origin/visit/get', methods=['POST']) def origin_visit_get(): return encode_data(g.storage.origin_visit_get(**decode_request(request))) @app.route('/origin/visit/getby', methods=['POST']) def origin_visit_get_by(): return encode_data( g.storage.origin_visit_get_by(**decode_request(request))) @app.route('/origin/visit/add', methods=['POST']) def origin_visit_add(): return encode_data(g.storage.origin_visit_add(**decode_request(request))) @app.route('/origin/visit/update', methods=['POST']) def origin_visit_update(): return encode_data(g.storage.origin_visit_update( **decode_request(request))) @app.route('/person', methods=['POST']) def person_get(): return encode_data(g.storage.person_get(**decode_request(request))) @app.route('/fetch_history', methods=['GET']) def fetch_history_get(): return encode_data(g.storage.fetch_history_get(request.args['id'])) @app.route('/fetch_history/start', methods=['POST']) def fetch_history_start(): return encode_data( g.storage.fetch_history_start(**decode_request(request))) @app.route('/fetch_history/end', methods=['POST']) def fetch_history_end(): return encode_data( g.storage.fetch_history_end(**decode_request(request))) @app.route('/entity/add', methods=['POST']) def entity_add(): return encode_data( g.storage.entity_add(**decode_request(request))) @app.route('/entity/get', methods=['POST']) def entity_get(): return encode_data( g.storage.entity_get(**decode_request(request))) @app.route('/entity', methods=['GET']) def entity_get_one(): return encode_data(g.storage.entity_get_one(request.args['uuid'])) @app.route('/entity/from_lister_metadata', methods=['POST']) def entity_from_lister_metadata(): return encode_data( g.storage.entity_get_from_lister_metadata(**decode_request(request))) @app.route('/content_mimetype/add', methods=['POST']) def content_mimetype_add(): return encode_data( g.storage.content_mimetype_add(**decode_request(request))) @app.route('/content_mimetype/missing', methods=['POST']) def content_mimetype_missing(): return encode_data( g.storage.content_mimetype_missing(**decode_request(request))) @app.route('/content_mimetype', methods=['POST']) def content_mimetype_get(): return encode_data( g.storage.content_mimetype_get(**decode_request(request))) @app.route('/content_language/add', methods=['POST']) def content_language_add(): return encode_data( g.storage.content_language_add(**decode_request(request))) @app.route('/content_language/missing', methods=['POST']) def content_language_missing(): return encode_data( g.storage.content_language_missing(**decode_request(request))) @app.route('/content_language', methods=['POST']) def content_language_get(): return encode_data( g.storage.content_language_get(**decode_request(request))) @app.route('/content/ctags/add', methods=['POST']) def content_ctags_add(): return encode_data( g.storage.content_ctags_add(**decode_request(request))) @app.route('/content/ctags/search', methods=['POST']) def content_ctags_search(): return encode_data( g.storage.content_ctags_search(**decode_request(request))) @app.route('/content/ctags/missing', methods=['POST']) def content_ctags_missing(): return encode_data( g.storage.content_ctags_missing(**decode_request(request))) @app.route('/content/ctags', methods=['POST']) def content_ctags_get(): return encode_data( g.storage.content_ctags_get(**decode_request(request))) @app.route('/content/fossology_license/add', methods=['POST']) def content_fossology_license_add(): return encode_data( g.storage.content_fossology_license_add(**decode_request(request))) @app.route('/content/fossology_license/missing', methods=['POST']) def content_fossology_license_missing(): return encode_data( g.storage.content_fossology_license_missing(**decode_request(request))) @app.route('/content/fossology_license', methods=['POST']) def content_fossology_license_get(): return encode_data( g.storage.content_fossology_license_get(**decode_request(request))) @app.route('/stat/counters', methods=['GET']) def stat_counters(): return encode_data(g.storage.stat_counters()) def run_from_webserver(environ, start_response): """Run the WSGI app from the webserver, loading the configuration.""" config_path = '/etc/softwareheritage/storage/storage.yml' app.config.update(config.read(config_path, DEFAULT_CONFIG)) handler = logging.StreamHandler() app.logger.addHandler(handler) return app(environ, start_response) @click.command() @click.argument('config-path', required=1) @click.option('--host', default='0.0.0.0', help="Host to run the server") -@click.option('--port', default=5000, type=click.INT, +@click.option('--port', default=5002, type=click.INT, help="Binding port of the server") @click.option('--debug/--nodebug', default=True, help="Indicates if the server should run in debug mode") def launch(config_path, host, port, debug): app.config.update(config.read(config_path, DEFAULT_CONFIG)) app.run(host, port=int(port), debug=bool(debug)) if __name__ == '__main__': launch() diff --git a/swh/storage/provenance/tasks.py b/swh/storage/provenance/tasks.py index b01c7806..b1dd2651 100644 --- a/swh/storage/provenance/tasks.py +++ b/swh/storage/provenance/tasks.py @@ -1,112 +1,112 @@ # Copyright (C) 2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import group from swh.core import hashutil from swh.core.config import load_named_config from swh.scheduler.task import Task from swh.storage import get_storage BASE_CONFIG_PATH = 'storage/provenance_cache' DEFAULT_CONFIG = { 'storage': ('dict', { 'cls': 'remote', 'args': { - 'url': 'http://localhost:5000/' + 'url': 'http://localhost:5002/' }, }), 'revision_packet_size': ('int', 100), } class PopulateCacheContentRevision(Task): """Populate the content -> revision provenance cache for some revisions""" task_queue = 'swh_populate_cache_content_revision' @property def config(self): if not hasattr(self, '__config'): self.__config = load_named_config(BASE_CONFIG_PATH, DEFAULT_CONFIG) return self.__config def run(self, revisions): """Cache the cache_content_revision table for the revisions provided. Args: revisions: List of revisions to cache populate. """ config = self.config storage = get_storage(**config['storage']) storage.cache_content_revision_add( hashutil.hex_to_hash(revision) for revision in revisions ) class PopulateCacheRevisionOrigin(Task): """Populate the revision -> origin provenance cache for one origin's visit""" task_queue = 'swh_populate_cache_revision_origin' @property def config(self): if not hasattr(self, '__config'): self.__config = load_named_config(BASE_CONFIG_PATH, DEFAULT_CONFIG) return self.__config def run(self, origin_id, visit_id): """Cache the cache_revision_origin for the given origin visit Args: origin_id: the origin id to cache visit_id: the visit id to cache This task also creates the revision cache tasks, as well as the task to cache the next origin visit available """ config = self.config storage = get_storage(**config['storage']) packet_size = config['revision_packet_size'] pipelined_tasks = [] visits = sorted( visit['visit'] for visit in storage.origin_visit_get(origin_id) ) if visit_id in visits: revision_task = PopulateCacheContentRevision() new_revisions = [ hashutil.hash_to_hex(revision) for revision in storage.cache_revision_origin_add( origin_id, visit_id) ] if new_revisions: split_new_revisions = [ new_revisions[i:i + packet_size] for i in range(0, packet_size, len(new_revisions)) ] for packet in split_new_revisions: pipelined_tasks.append(revision_task.s(packet)) try: next_visit = min(visit for visit in visits if visit > visit_id) except ValueError: # no next visit, stop pipelining further visits pass else: pipelined_tasks.append(self.s(origin_id, next_visit)) if pipelined_tasks: group(pipelined_tasks).delay() diff --git a/swh/storage/vault/api/server.py b/swh/storage/vault/api/server.py index fe1dc523..6ec9843d 100644 --- a/swh/storage/vault/api/server.py +++ b/swh/storage/vault/api/server.py @@ -1,104 +1,104 @@ # Copyright (C) 2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click from flask import abort, g from werkzeug.routing import BaseConverter from swh.core import config from swh.core.api import (SWHServerAPIApp, error_handler, encode_data_server as encode_data) from swh.storage import get_storage from swh.storage.vault.api import cooking_tasks # NOQA from swh.storage.vault.cache import VaultCache from swh.storage.vault.cooker import DirectoryVaultCooker from swh.scheduler.celery_backend.config import app as celery_app cooking_task_name = 'swh.storage.vault.api.cooking_tasks.SWHCookingTask' DEFAULT_CONFIG = { 'storage': ('dict', { 'cls': 'local', 'args': { 'db': 'dbname=softwareheritage-dev', 'objstorage': { 'root': '/tmp/objects', 'slicing': '0:2/2:4/4:6', }, }, }), 'cache': ('dict', {'root': '/tmp/vaultcache'}) } class RegexConverter(BaseConverter): def __init__(self, url_map, *items): super().__init__(url_map) self.regex = items[0] app = SWHServerAPIApp(__name__) app.url_map.converters['regex'] = RegexConverter @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) @app.before_request def before_request(): g.cache = VaultCache(**app.config['cache']) g.cooker = DirectoryVaultCooker( get_storage(**app.config['storage']), g.cache ) @app.route('/') def index(): return 'SWH vault API server' @app.route('/vault//', methods=['GET']) def ls_directory(type): return encode_data(list( g.cache.ls(type) )) @app.route('/vault///', methods=['GET']) def get_cooked_directory(type, id): if not g.cache.is_cached(type, id): abort(404) return encode_data(g.cache.get(type, id).decode()) @app.route('/vault///', methods=['POST']) def cook_request_directory(type, id): task = celery_app.tasks[cooking_task_name] task.delay(type, id, app.config['storage'], app.config['cache']) # Return url to get the content and 201 CREATED return encode_data('/vault/%s/%s/' % (type, id)), 201 @click.command() @click.argument('config-path', required=1) @click.option('--host', default='0.0.0.0', help="Host to run the server") -@click.option('--port', default=5000, type=click.INT, +@click.option('--port', default=5005, type=click.INT, help="Binding port of the server") @click.option('--debug/--nodebug', default=True, help="Indicates if the server should run in debug mode") def launch(config_path, host, port, debug): app.config.update(config.read(config_path, DEFAULT_CONFIG)) app.run(host, port=int(port), debug=bool(debug)) if __name__ == '__main__': launch()