diff --git a/swh/loader/git/backend/api.py b/swh/loader/git/backend/api.py index 9a4b679..8d1f43f 100755 --- a/swh/loader/git/backend/api.py +++ b/swh/loader/git/backend/api.py @@ -1,280 +1,280 @@ #!/usr/bin/env python3 # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging from flask import Flask, Response, make_response, request from swh.loader.git.storage import storage, db, service from swh.loader.git.protocols import serial # api's definition app = Flask(__name__) def read_request_payload(request): """Read the request's payload. """ # TODO: Check the signed pickled data? return serial.load(request.stream) def write_response(data): """Write response from data. """ return Response(serial.dumps(data), mimetype=serial.MIMETYPE) @app.route('/') def hello(): """A simple api to define what the server is all about. FIXME: A redirect towards a static page defining the routes would be nice. """ return 'Dev SWH API' # from uri to type _uri_types = { 'revisions': storage.Type.revision, 'directories': storage.Type.directory, 'contents': storage.Type.content, 'releases': storage.Type.release, 'occurrences': storage.Type.occurrence, 'persons': storage.Type.person } def _do_action_with_payload(conf, action_fn, uri_type, id, map_result_fn): uri_type_ok = _uri_types.get(uri_type, None) if uri_type_ok is None: return make_response('Bad request!', 400) vcs_object = read_request_payload(request) vcs_object.update({'id': id, 'type': uri_type_ok}) return action_fn(conf, vcs_object, map_result_fn) # occurrence type is not dealt the same way _post_all_uri_types = { 'revisions': storage.Type.revision, 'directories': storage.Type.directory, 'contents': storage.Type.content } @app.route('/vcs//', methods=['POST']) def filter_unknowns_type(uri_type): """Filters unknown sha1 to the backend and returns them. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected ' '%s data!' % serial.MIMETYPE, 400) obj_type = _post_all_uri_types.get(uri_type) if obj_type is None: return make_response('Bad request. Type not supported!', 400) sha1s = read_request_payload(request) config = app.config['conf'] with db.connect(config['db_url']) as db_conn: unknowns_sha1s = service.filter_unknowns_type(db_conn, obj_type, sha1s) if unknowns_sha1s is None: return make_response('Bad request!', 400) else: return write_response(unknowns_sha1s) @app.route('/vcs/persons/', methods=['POST']) def post_person(): """Find a person. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected ' '%s data!' % serial.MIMETYPE, 400) origin = read_request_payload(request) config = app.config['conf'] with db.connect(config['db_url']) as db_conn: try: person_found = service.find_person(db_conn, origin) if person_found: return write_response(person_found) else: return make_response('Person not found!', 404) except: return make_response('Bad request!', 400) -@app.route('/origins/', methods=['POST']) +@app.route('/vcs/origins/', methods=['POST']) def post_origin(): """Find an origin. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected ' '%s data!' % serial.MIMETYPE, 400) origin = read_request_payload(request) config = app.config['conf'] with db.connect(config['db_url']) as db_conn: try: origin_found = service.find_origin(db_conn, origin) if origin_found: return write_response(origin_found) else: return make_response('Origin not found!', 404) except: return make_response('Bad request!', 400) -@app.route('/origins/', methods=['PUT']) +@app.route('/vcs/origins/', methods=['PUT']) def put_origin(): """Create an origin or returns it if already existing. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected ' '%s data!' % serial.MIMETYPE, 400) origin = read_request_payload(request) config = app.config['conf'] with db.connect(config['db_url']) as db_conn: try: origin_found = service.add_origin(db_conn, origin) return write_response(origin_found) # FIXME: 204 except: return make_response('Bad request!', 400) @app.route('/vcs//', methods=['PUT']) def put_all(uri_type): """Store/update objects (uri_type in {contents, directories, releases}). """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected ' '%s data!' % serial.MIMETYPE, 400) payload = read_request_payload(request) obj_type = _uri_types[uri_type] config = app.config['conf'] with db.connect(config['db_url']) as db_conn: service.persist(db_conn, config, obj_type, payload) return make_response('Successful creation!', 204) def add_object(config, vcs_object, map_result_fn): """Add object in storage. - config is the configuration needed for the backend to execute query - vcs_object is the object to look for in the backend - map_result_fn is a mapping function which takes the backend's result and transform its output accordingly. This function returns an http response of the result. """ type = vcs_object['type'] id = vcs_object['id'] logging.debug('storage %s %s' % (type, id)) with db.connect(config['db_url']) as db_conn: res = service.persist(db_conn, config, type, [vcs_object]) return make_response(map_result_fn(id, res), 204) def _do_lookup(conf, uri_type, id, map_result_fn): """Looking up type object with sha1. - config is the configuration needed for the backend to execute query - vcs_object is the object to look for in the backend - map_result_fn is a mapping function which takes the backend's result and transform its output accordingly. This function returns an http response of the result. """ uri_type_ok = _uri_types.get(uri_type, None) if not uri_type_ok: return make_response('Bad request!', 400) with db.connect(conf['db_url']) as db_conn: res = storage.find(db_conn, id, uri_type_ok) if res: return write_response(map_result_fn(id, res)) # 200 return make_response('Not found!', 404) @app.route('/vcs/occurrences/') def list_occurrences_for(id): """Return the occurrences pointing to the revision id. """ return _do_lookup(app.config['conf'], 'occurrences', id, lambda _, result: list(map(lambda col: col[1], result))) @app.route('/vcs//') def object_exists_p(uri_type, id): """Assert if the object with sha1 id, of type uri_type, exists. """ return _do_lookup(app.config['conf'], uri_type, id, lambda sha1, _: {'id': sha1}) @app.route('/vcs//', methods=['PUT']) def put_object(uri_type, id): """Put an object in storage. """ return _do_action_with_payload(app.config['conf'], add_object, uri_type, id, # FIXME: use id or result instead lambda sha1, _2: sha1) def run(conf): """Run the api's server. conf is a dictionary of keywords: - 'db_url' the db url's access (through psycopg2 format) - 'content_storage_dir' revisions/directories/contents storage on disk - 'host' to override the default 127.0.0.1 to open or not the server to the world - 'port' to override the default of 5000 (from the underlying layer: flask) - 'debug' activate the verbose logs """ print("""SWH Api run host: %s port: %s debug: %s""" % (conf['host'], conf.get('port', None), conf['debug'])) # app.config is the app's state (accessible) app.config.update({'conf': conf}) app.run(host=conf['host'], port=conf.get('port', None), debug=conf['debug'] == 'true') diff --git a/swh/loader/git/client/http.py b/swh/loader/git/client/http.py index 65fde00..4e23a2b 100755 --- a/swh/loader/git/client/http.py +++ b/swh/loader/git/client/http.py @@ -1,97 +1,97 @@ #!/usr/bin/env python3 # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import requests from retrying import retry from swh.loader.git.retry import policy from swh.loader.git.storage import storage from swh.loader.git.protocols import serial session_swh = requests.Session() def compute_simple_url(base_url, type): """Compute the api url. """ return '%s%s' % (base_url, type) @retry(retry_on_exception=policy.retry_if_connection_error, wrap_exception=True, stop_max_attempt_number=3) def execute(map_type_url, method_fn, base_url, obj_type, data, result_fn=lambda result: result.ok): """Execute a query to the backend. - map_type_url is a map of {type: url backend} - method_fn is swh_session.post or swh_session.put - base_url is the base url of the backend - obj_type is the nature of the data - data is the data to send to the backend - result_fn is a function which takes the response result and do something with it. The default function is to return if the server is ok or not. """ if not data: return data res = method_fn(compute_simple_url(base_url, map_type_url[obj_type]), data=serial.dumps(data), headers={'Content-Type': serial.MIMETYPE}) return result_fn(res) # url mapping for lookup url_lookup_per_type = { storage.Type.origin: "/origins/", storage.Type.content: "/vcs/contents/", storage.Type.directory: "/vcs/directories/", storage.Type.revision: "/vcs/revisions/", } def post(base_url, obj_type, obj_sha1s): """Retrieve the objects of type type with sha1 sha1hex. """ return execute(url_lookup_per_type, session_swh.post, base_url, obj_type, obj_sha1s, result_fn=lambda res: serial.loads(res.content)) # url mapping for storage url_store_per_type = { - storage.Type.origin: "/origins/", + storage.Type.origin: "/vcs/origins/", storage.Type.content: "/vcs/contents/", storage.Type.directory: "/vcs/directories/", storage.Type.revision: "/vcs/revisions/", storage.Type.release: "/vcs/releases/", storage.Type.occurrence: "/vcs/occurrences/", storage.Type.person: "/vcs/persons/", } def put(base_url, obj_type, obj): """Given an obj (map, simple object) of obj_type, PUT it in the backend. """ return execute(url_store_per_type, session_swh.put, base_url, obj_type, obj) diff --git a/swh/loader/git/tests/test_api_origin.py b/swh/loader/git/tests/test_api_origin.py index 3792139..5689147 100644 --- a/swh/loader/git/tests/test_api_origin.py +++ b/swh/loader/git/tests/test_api_origin.py @@ -1,98 +1,98 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from nose.tools import istest from nose.plugins.attrib import attr from swh.loader.git.storage import db, models from swh.loader.git.protocols import serial from test_utils import app_client @attr('slow') class OriginTestCase(unittest.TestCase): def setUp(self): self.app, db_url, _ = app_client() with db.connect(db_url) as db_conn: self.origin_url = 'https://github.com/torvalds/linux.git' self.origin_type = 'git' self.origin_id = models.add_origin(db_conn, self.origin_url, self.origin_type) @istest def get_origin_ok(self): # when payload = {'url': self.origin_url, 'type': self.origin_type} - rv = self.app.post('/origins/', + rv = self.app.post('/vcs/origins/', data=serial.dumps(payload), headers={'Content-Type': serial.MIMETYPE}) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == self.origin_id @istest def get_origin_not_found(self): # when payload = {'url': 'unknown', 'type': 'blah'} - rv = self.app.post('/origins/', + rv = self.app.post('/vcs/origins/', data=serial.dumps(payload), headers={'Content-Type': serial.MIMETYPE}) # then assert rv.status_code == 404 assert rv.data == b'Origin not found!' @istest def get_origin_not_found_with_bad_format(self): # when - rv = self.app.post('/origins/', + rv = self.app.post('/vcs/origins/', data=serial.dumps({'url': 'unknown'}), headers={'Content-Type': serial.MIMETYPE}) # then assert rv.status_code == 400 @istest def put_origin(self): # when payload = {'url': 'unknown', 'type': 'blah'} - rv = self.app.post('/origins/', + rv = self.app.post('/vcs/origins/', data=serial.dumps(payload), headers={'Content-Type': serial.MIMETYPE}) # then assert rv.status_code == 404 assert rv.data == b'Origin not found!' # when - rv = self.app.put('/origins/', + rv = self.app.put('/vcs/origins/', data=serial.dumps(payload), headers={'Content-Type': serial.MIMETYPE}) # then assert rv.status_code == 200 # FIXME: 201 assert serial.loads(rv.data)['id'] payload = {'url': 'unknown', 'type': 'blah'} - rv = self.app.post('/origins/', + rv = self.app.post('/vcs/origins/', data=serial.dumps(payload), headers={'Content-Type': serial.MIMETYPE}) # then assert rv.status_code == 200 origin_id = serial.loads(rv.data)['id'] assert origin_id # when - rv = self.app.put('/origins/', + rv = self.app.put('/vcs/origins/', data=serial.dumps(payload), headers={'Content-Type': serial.MIMETYPE}) # then assert rv.status_code == 200 # FIXME: 204 assert serial.loads(rv.data)['id'] == origin_id