diff --git a/swh/backend/api.py b/swh/backend/api.py index a43a085..4ece97f 100755 --- a/swh/backend/api.py +++ b/swh/backend/api.py @@ -1,303 +1,289 @@ #!/usr/bin/env python3 # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging from flask import Flask, Response, make_response, request from swh.store import store, db, service from swh.protocols import serial # api's definition app = Flask(__name__) def read_request_payload(request): """Read the request's payload. """ # TODO: Check the signed pickled data? return serial.load(request.stream) def write_response(data): """Write response from data. """ return Response(serial.dumps(data), mimetype=serial.MIMETYPE) @app.route('/') def hello(): """A simple api to define what the server is all about. FIXME: A redirect towards a static page defining the routes would be nice. """ return 'Dev SWH API' # from uri to type _uri_types = {'revisions': store.Type.revision, 'directories': store.Type.directory, 'contents': store.Type.content, 'releases': store.Type.release, 'occurrences': store.Type.occurrence} def _do_action_with_payload(conf, action_fn, uri_type, id, map_result_fn): uri_type_ok = _uri_types.get(uri_type, None) if uri_type_ok is None: return make_response('Bad request!', 400) vcs_object = read_request_payload(request) vcs_object.update({'id': id, 'type': uri_type_ok}) return action_fn(conf, vcs_object, map_result_fn) # occurrence type is not dealt the same way _post_all_uri_types = {'revisions': store.Type.revision, 'directories': store.Type.directory, 'contents': store.Type.content} @app.route('/vcs//', methods=['POST']) def filter_unknowns_type(uri_type): """Filters unknown sha1 to the backend and returns them. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected %s data!' % serial.MIMETYPE, 400) obj_type = _post_all_uri_types.get(uri_type) if obj_type is None: return make_response('Bad request. Type not supported!', 400) sha1s = read_request_payload(request) config = app.config['conf'] with db.connect(config['db_url']) as db_conn: unknowns_sha1s = service.filter_unknowns_type(db_conn, obj_type, sha1s) if unknowns_sha1s is None: return make_response('Bad request!', 400) else: return write_response(unknowns_sha1s) @app.route('/vcs/persons/', methods=['POST']) def post_person(): """Find a person. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected %s data!' % serial.MIMETYPE, 400) origin = read_request_payload(request) config = app.config['conf'] with db.connect(config['db_url']) as db_conn: try: person_found = service.find_person(db_conn, origin) if person_found: return write_response(person_found) else: return make_response('Person not found!', 404) except: return make_response('Bad request!', 400) @app.route('/origins/', methods=['POST']) def post_origin(): """Find an origin. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected %s data!' % serial.MIMETYPE, 400) origin = read_request_payload(request) config = app.config['conf'] with db.connect(config['db_url']) as db_conn: try: origin_found = service.find_origin(db_conn, origin) if origin_found: return write_response(origin_found) else: return make_response('Origin not found!', 404) except: return make_response('Bad request!', 400) @app.route('/origins/', methods=['PUT']) def put_origin(): """Create an origin or returns it if already existing. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected %s data!' % serial.MIMETYPE, 400) origin = read_request_payload(request) config = app.config['conf'] with db.connect(config['db_url']) as db_conn: try: origin_found = service.add_origin(db_conn, origin) return write_response(origin_found) # FIXME: 204 except: return make_response('Bad request!', 400) @app.route('/vcs/persons/', methods=['PUT']) def put_all_persons(): """Store or update given revisions. FIXME: Refactor same behavior with `put_all`. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected %s data!' % serial.MIMETYPE, 400) payload = read_request_payload(request) obj_type = store.Type.person config = app.config['conf'] with db.connect(config['db_url']) as db_conn: service.add_persons(db_conn, config, obj_type, payload) return make_response('Successful creation!', 204) @app.route('/vcs/revisions/', methods=['PUT']) def put_all_revisions(): """Store or update given revisions. FIXME: Refactor same behavior with `put_all`. """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected %s data!' % serial.MIMETYPE, 400) payload = read_request_payload(request) obj_type = store.Type.revision config = app.config['conf'] with db.connect(config['db_url']) as db_conn: service.add_revisions(db_conn, config, obj_type, payload) return make_response('Successful creation!', 204) @app.route('/vcs//', methods=['PUT']) def put_all(uri_type): """Store or update given objects (uri_type in {contents, directories, releases). """ if request.headers.get('Content-Type') != serial.MIMETYPE: return make_response('Bad request. Expected %s data!' % serial.MIMETYPE, 400) payload = read_request_payload(request) obj_type = _uri_types[uri_type] config = app.config['conf'] with db.connect(config['db_url']) as db_conn: service.add_objects(db_conn, config, obj_type, payload) return make_response('Successful creation!', 204) def add_object(config, vcs_object, map_result_fn): """Add object in storage. - config is the configuration needed for the backend to execute query - vcs_object is the object to look for in the backend - map_result_fn is a mapping function which takes the backend's result and transform its output accordingly. This function returns an http response of the result. """ type = vcs_object['type'] id = vcs_object['id'] logging.debug('store %s %s' % (type, id)) with db.connect(config['db_url']) as db_conn: - if store.find(db_conn, vcs_object): - logging.debug('update %s %s' % (id, type)) - return make_response('Successful update!', 200) # immutable - else: - logging.debug('store %s %s' % (id, type)) - res = store.add(db_conn, config, vcs_object) - if res is None: - return make_response('Bad request!', 400) - elif res is False: - logging.error('store %s %s' % (id, type)) - return make_response('Internal server error!', 500) - else: - return make_response(map_result_fn(id, res), 204) + res = service.add_objects(db_conn, config, type, [vcs_object]) + return make_response(map_result_fn(id, res), 204) def _do_lookup(conf, uri_type, id, map_result_fn): """Looking up type object with sha1. - config is the configuration needed for the backend to execute query - vcs_object is the object to look for in the backend - map_result_fn is a mapping function which takes the backend's result and transform its output accordingly. This function returns an http response of the result. """ uri_type_ok = _uri_types.get(uri_type, None) if not uri_type_ok: return make_response('Bad request!', 400) - vcs_object = {'id': id, - 'type': uri_type_ok} - with db.connect(conf['db_url']) as db_conn: - res = store.find(db_conn, vcs_object) + res = store.find(db_conn, id, uri_type_ok) if res: return write_response(map_result_fn(id, res)) # 200 return make_response('Not found!', 404) @app.route('/vcs/occurrences/') def list_occurrences_for(id): """Return the occurrences pointing to the revision id. """ return _do_lookup(app.config['conf'], 'occurrences', id, lambda _, result: list(map(lambda col: col[1], result))) @app.route('/vcs//') def object_exists_p(uri_type, id): """Assert if the object with sha1 id, of type uri_type, exists. """ return _do_lookup(app.config['conf'], uri_type, id, lambda sha1, _: {'id': sha1}) @app.route('/vcs//', methods=['PUT']) def put_object(uri_type, id): """Put an object in storage. """ return _do_action_with_payload(app.config['conf'], add_object, uri_type, id, - lambda _1, _2: 'Successful Creation!') # FIXME: use id or result instead + lambda sha1, _2: sha1) # FIXME: use id or result instead def run(conf): """Run the api's server. conf is a dictionary of keywords: - 'db_url' the db url's access (through psycopg2 format) - 'content_storage_dir' revisions/directories/contents storage on disk - 'host' to override the default 127.0.0.1 to open or not the server to the world - 'port' to override the default of 5000 (from the underlying layer: flask) - 'debug' activate the verbose logs """ print("""SWH Api run host: %s port: %s debug: %s""" % (conf['host'], conf.get('port', None), conf['debug'])) # app.config is the app's state (accessible) app.config.update({'conf': conf}) app.run(host=conf['host'], port=conf.get('port', None), debug=conf['debug'] == 'true') diff --git a/swh/gitloader/local_store.py b/swh/gitloader/local_store.py index c25533f..7bff576 100644 --- a/swh/gitloader/local_store.py +++ b/swh/gitloader/local_store.py @@ -1,96 +1,95 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.store import store, db, service from swh.conf import reader from swh.storage.objstorage import ObjStorage # FIXME: duplicated from bin/swh-backend... # Default configuration file DEFAULT_CONF_FILE = '~/.config/swh/back.ini' # default configuration DEFAULT_CONF = { 'content_storage_dir': ('string', '/tmp/swh-loader-git/content-storage'), 'log_dir': ('string', '/tmp/swh-loader-git/log'), 'db_url': ('string', 'dbname=softwareheritage-dev'), 'folder_depth': ('int', 4), 'debug': ('bool', None), 'host': ('string', '127.0.0.1'), 'port': ('int', 5000) } def store_only_new(db_conn, conf, obj_type, obj): """Store object if not already present. """ - obj.update({'type': obj_type}) - if not store.find(db_conn, obj): + if not store.find(db_conn, obj['id'], obj_type): store.add(db_conn, conf, obj) _obj_to_persist_fn = {store.Type.revision: service.add_revisions} def store_unknown_objects(db_conn, conf, obj_type, swhmap): """Load objects to the backend. """ sha1s = swhmap.keys() # have: filter unknown obj unknown_obj_sha1s = service.filter_unknowns_type(db_conn, obj_type, sha1s) if not unknown_obj_sha1s: return True # seen: now store in backend persist_fn = _obj_to_persist_fn.get(obj_type, service.add_objects) obj_fulls = map(swhmap.get, unknown_obj_sha1s) return persist_fn(db_conn, conf, obj_type, obj_fulls) def load_to_back(conf, swh_repo): """Load to the backend the repository swh_repo. """ with db.connect(conf['db_url']) as db_conn: # First, store/retrieve the origin identifier # FIXME: should be done by the cloner worker (which is not yet plugged # on the right swh db ftm) service.add_origin(db_conn, swh_repo.get_origin()) # First reference all unknown persons service.add_persons(db_conn, conf, store.Type.person, swh_repo.get_persons()) res = store_unknown_objects(db_conn, conf, store.Type.content, swh_repo.get_contents()) if res: res = store_unknown_objects(db_conn, conf, store.Type.directory, swh_repo.get_directories()) if res: res = store_unknown_objects(db_conn, conf, store.Type.revision, swh_repo.get_revisions()) if res: # brutally send all remaining occurrences service.add_objects(db_conn, conf, store.Type.occurrence, swh_repo.get_occurrences()) # and releases (the idea here is that compared to existing # objects, the quantity is less) service.add_objects(db_conn, conf, store.Type.release, swh_repo.get_releases()) def prepare_and_load_to_back(backend_setup_file, swh_repo): # Read the configuration file (no check yet) conf = reader.read(backend_setup_file or DEFAULT_CONF_FILE, DEFAULT_CONF) reader.prepare_folders(conf, 'content_storage_dir') conf.update({ 'objstorage': ObjStorage(conf['content_storage_dir'], conf['folder_depth']) }) load_to_back(conf, swh_repo) diff --git a/swh/store/service.py b/swh/store/service.py index 9b413b6..8b00ed4 100644 --- a/swh/store/service.py +++ b/swh/store/service.py @@ -1,74 +1,84 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.store import store filter_unknowns_type = store.find_unknowns def find_origin(db_conn, origin): """Find origin. """ origin_found = store.find_origin(db_conn, origin) return None if not origin_found else {'id': origin_found[0]} def find_person(db_conn, person): """Find person. """ person_found = store.find_person(db_conn, person) return None if not person_found else {'id': person_found[0]} def add_origin(db_conn, origin): """Add origin if not already existing. """ origin_found = store.find_origin(db_conn, origin) id = origin_found[0] if origin_found else store.add_origin(db_conn, origin) return {'id': id} def add_revisions(db_conn, conf, obj_type, objs): """Add Revisions. """ couple_parents = [] for obj in objs: # iterate over objects of type uri_type - - obj_found = store.find(db_conn, obj) + obj_id = obj['id'] + obj_found = store.find(db_conn, obj_id, obj_type) if not obj_found: - store.add(db_conn, conf, obj) + store.add(db_conn, conf, obj_id, obj_type, obj) # deal with revision history par_shas = obj.get('parent-sha1s', None) if par_shas: - couple_parents.extend([(obj['id'], p) for p in par_shas]) + couple_parents.extend([(obj_id, p) for p in par_shas]) store.add_revision_history(db_conn, couple_parents) return True def add_persons(db_conn, conf, obj_type, objs): """Add persons. conf, obj_type are not used (implementation detail.) """ for obj in objs: obj_found = store.find_person(db_conn, obj) if not obj_found: store.add_person(db_conn, obj) return True +# dispatch map to add in storage with fs or not +_add_fn = {store.Type.content: store.add_with_fs_storage} + + def add_objects(db_conn, conf, obj_type, objs): - """Add objects. + """Add objects if not already present in the storage. """ + add_fn = _add_fn.get(obj_type, store.add) + res = [] for obj in objs: # iterate over objects of type uri_type - obj_found = store.find(db_conn, obj) + obj_id = obj['id'] + obj_found = store.find(db_conn, obj_id, obj_type) if not obj_found: - store.add(db_conn, conf, obj) + obj = add_fn(db_conn, conf, obj_id, obj_type, obj) + res.append(obj) + else: + res.append(obj_found) - return True + return res diff --git a/swh/store/store.py b/swh/store/store.py index f75ae6c..51afaed 100755 --- a/swh/store/store.py +++ b/swh/store/store.py @@ -1,194 +1,198 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from io import StringIO from swh.store import models Type = models.Type _find_object = {Type.occurrence: models.find_occurrences_for_revision} -def find(db_conn, vcs_object): +def find(db_conn, id, type): """Find an object according to its sha1hex and type. """ - id = vcs_object['id'] # sha1 for every object except for origin (url) - type = vcs_object['type'] - - find_fn = _find_object.get(type, models.find_object) + find_fn = _find_object.get(type, models.find_object) return find_fn(db_conn, id, type) _find_unknown = {Type.revision: models.find_unknown_revisions, Type.content: models.find_unknown_contents, Type.directory: models.find_unknown_directories} def find_unknowns(db_conn, obj_type, sha1s_hex): """Given a list of sha1s, return the non presents one in storage. """ def row_to_sha1(row): """Convert a row (memoryview) to a string sha1. """ return row[0] vals = '\n'.join(sha1s_hex) cpy_data_buffer = StringIO() cpy_data_buffer.write(vals) cpy_data_buffer.seek(0) # move file cursor back at start of file find_unknown_fn = _find_unknown[obj_type] unknowns = find_unknown_fn(db_conn, cpy_data_buffer) cpy_data_buffer.close() return list(map(row_to_sha1, unknowns)) def _add_content(db_conn, vcs_object, sha1hex): """Add a blob to storage. Designed to be wrapped in a db transaction. Returns: - the sha1 if everything went alright. - None if something went wrong Writing exceptions can also be raised and expected to be handled by the caller. """ models.add_content(db_conn, sha1hex, vcs_object['content-sha1'], vcs_object['content-sha256'], vcs_object['size']) return sha1hex def _add_directory(db_conn, vcs_object, sha1hex): """Add a directory to storage. Designed to be wrapped in a db transaction. """ models.add_directory(db_conn, sha1hex) for directory_entry in vcs_object['entries']: _add_directory_entry(db_conn, directory_entry) return sha1hex def _add_directory_entry(db_conn, vcs_object): """Add a directory to storage. Designed to be wrapped in a db transaction. Returns: - the sha1 if everything went alright. - None if something went wrong Writing exceptions can also be raised and expected to be handled by the caller. """ name = vcs_object['name'] parent = vcs_object['parent'] models.add_directory_entry(db_conn, name, vcs_object['target-sha1'], vcs_object['nature'], vcs_object['perms'], vcs_object['atime'], vcs_object['mtime'], vcs_object['ctime'], parent) return name, parent def _add_revision(db_conn, vcs_object, sha1hex): """Add a revision to storage. Designed to be wrapped in a db transaction. Returns: - the sha1 if everything went alright. - None if something went wrong Writing exceptions can also be raised and expected to be handled by the caller. """ models.add_revision(db_conn, sha1hex, vcs_object['date'], vcs_object['directory'], vcs_object['message'], vcs_object['author'], vcs_object['committer'], vcs_object['parent-sha1s']) return sha1hex def _add_release(db_conn, vcs_object, sha1hex): """Add a release. """ models.add_release(db_conn, sha1hex, vcs_object['revision'], vcs_object['date'], vcs_object['name'], vcs_object['comment'], vcs_object['author']) return sha1hex def _add_occurrence(db_conn, vcs_object, sha1hex): """Add an occurrence. """ models.add_occurrence(db_conn, vcs_object['url-origin'], vcs_object['reference'], vcs_object['revision']) return sha1hex def add_person(db_conn, vcs_object): """Add an author. """ return models.add_person(db_conn, vcs_object['name'], vcs_object['email']) -_store_fn = {Type.content: _add_content, - Type.directory: _add_directory, +_store_fn = {Type.directory: _add_directory, Type.revision: _add_revision, Type.release: _add_release, Type.occurrence: _add_occurrence} def add_origin(db_conn, origin): """A a new origin and returns its id. """ return models.add_origin(db_conn, origin['url'], origin['type']) def find_origin(db_conn, origin): """Find an existing origin. """ return models.find_origin(db_conn, origin['url'], origin['type']) def find_person(db_conn, person): """Find an existing person. """ return models.find_person(db_conn, person['email'], person['name']) -def add(db_conn, config, vcs_object): +def add_with_fs_storage(db_conn, config, id, type, vcs_object): + """Add vcs_object in the storage + - db_conn is the opened connection to the db + - config is the map of configuration needed for core layer + - type is not used here but represent the type of vcs_object + - vcs_object is the object meant to be persisted in fs and db + """ + config['objstorage'].add_bytes(vcs_object['content'], id) # FIXME use this id + return _add_content(db_conn, vcs_object, id) + + +def add(db_conn, config, id, type, vcs_object): """Given a sha1hex, type and content, store a given object in the store. + - db_conn is the opened connection to the db + - config is not used here + - type is the object's type + - vcs_object is the object meant to be persisted in db """ - type = vcs_object['type'] - sha1hex = vcs_object['id'] - obj_content = vcs_object.get('content') - - if obj_content: - config['objstorage'].add_bytes(obj_content, sha1hex) - return _store_fn[type](db_conn, vcs_object, sha1hex) - return _store_fn[type](db_conn, vcs_object, sha1hex) + return _store_fn[type](db_conn, vcs_object, id) def add_revision_history(db_conn, couple_parents): """Given a list of tuple (sha, parent_sha), store in revision_history. """ if len(couple_parents) > 0: models.add_revision_history(db_conn, couple_parents) diff --git a/swh/tests/test_api_content.py b/swh/tests/test_api_content.py index 2e3c921..2fffb27 100644 --- a/swh/tests/test_api_content.py +++ b/swh/tests/test_api_content.py @@ -1,110 +1,110 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from nose.tools import istest from nose.plugins.attrib import attr from swh.store import db, models from swh.protocols import serial from test_utils import app_client, app_client_teardown @attr('slow') class ContentTestCase(unittest.TestCase): def setUp(self): self.app, db_url, self.content_storage_dir = app_client() with db.connect(db_url) as db_conn: self.content_sha1_id = '222222f9dd5dc46ee476a8be155ab049994f717e' content_sha1_id = 'blabliblablo' self.content_sha256_hex = '222222f9dd5dc46ee476a8be155ab049994f717e' models.add_content(db_conn, self.content_sha1_id, content_sha1_id, self.content_sha256_hex, 10) def tearDown(self): app_client_teardown(self.content_storage_dir) @istest def get_content_ok(self): # when rv = self.app.get('/vcs/contents/%s' % self.content_sha1_id) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == '222222f9dd5dc46ee476a8be155ab049994f717e' @istest def get_content_not_found(self): # when rv = self.app.get('/vcs/contents/222222f9dd5dc46ee476a8be155ab049994f7170') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def get_content_not_found_with_bad_format(self): # when rv = self.app.get('/vcs/contents/1') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def put_content_create_and_update(self): content_sha1 = 'sha1-contentc46ee476a8be155ab03333333333' # does not exist rv = self.app.get('/vcs/contents/%s' % content_sha1) # then assert rv.status_code == 404 assert rv.data == b'Not found!' # we create it body = {'id': content_sha1, 'content-sha1': 'content-sha1c46ee476a8be155ab03333333333', 'content-sha256': 'content-sha2566ee476a8be155ab03333333333', 'content': b'bar', 'size': '3'} rv = self.app.put('/vcs/contents/%s' % content_sha1, data=serial.dumps(body), headers={'Content-Type': serial.MIMETYPE}) assert rv.status_code == 204 assert rv.data == b'' # now it exists rv = self.app.get('/vcs/contents/%s' % content_sha1) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == 'sha1-contentc46ee476a8be155ab03333333333' # # we update it body = {'id': content_sha1, 'content-sha1': 'content-sha1c46ee476a8be155ab03333333333', 'content-sha256': 'content-sha2566ee476a8be155ab03333333333', 'content': b'bar', 'size': '3'} rv = self.app.put('/vcs/contents/%s' % content_sha1, data=serial.dumps(body), headers={'Content-Type': serial.MIMETYPE}) - assert rv.status_code == 200 - assert rv.data == b'Successful update!' + assert rv.status_code == 204 + assert rv.data == b'' # still the same rv = self.app.get('/vcs/contents/%s' % content_sha1) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == 'sha1-contentc46ee476a8be155ab03333333333' diff --git a/swh/tests/test_api_directory.py b/swh/tests/test_api_directory.py index 9222010..f73b0c1 100644 --- a/swh/tests/test_api_directory.py +++ b/swh/tests/test_api_directory.py @@ -1,119 +1,119 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from nose.tools import istest from nose.plugins.attrib import attr from swh.store import db, models from swh.protocols import serial from test_utils import now, app_client, app_client_teardown @attr('slow') class DirectoryTestCase(unittest.TestCase): def setUp(self): self.app, db_url, self.content_storage_dir = app_client() with db.connect(db_url) as db_conn: self.content_sha1_id = 'content-sha1c46ee476a8be155ab049994f717e' content_sha1_hex = 'content-sha1c46ee476a8be155ab049994f717e' content_sha256_hex = 'content-sha2566ee476a8be155ab049994f717e' models.add_content(db_conn, self.content_sha1_id, content_sha1_hex, content_sha256_hex, 10) self.directory_sha1_hex = 'directory-sha16ee476a8be155ab049994f717e' models.add_directory(db_conn, self.directory_sha1_hex) def tearDown(self): app_client_teardown(self.content_storage_dir) @istest def get_directory_ok(self): # when rv = self.app.get('/vcs/directories/%s' % self.directory_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == 'directory-sha16ee476a8be155ab049994f717e' @istest def get_directory_not_found(self): # when rv = self.app.get('/vcs/directories/111111f9dd5dc46ee476a8be155ab049994f7170') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def get_directory_not_found_with_bad_format(self): # when rv = self.app.get('/vcs/directories/1') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def put_directory_create_and_update(self): directory_sha1='directory-sha16ee476a8be155ab049994f7170' # does not exist rv = self.app.get('/vcs/directories/%s' % directory_sha1) # then assert rv.status_code == 404 assert rv.data == b'Not found!' # we create it body = serial.dumps({'entries': [{'name': 'filename', 'target-sha1': self.content_sha1_id, 'nature': 'file', 'perms': '000', 'atime': now(), 'mtime': now(), 'ctime': now(), 'parent': directory_sha1}, {'name': 'dirname', 'target-sha1': self.directory_sha1_hex, 'nature': 'directory', 'perms': '012', 'atime': now(), 'mtime': now(), 'ctime': now(), 'parent': directory_sha1} ]}) rv = self.app.put('/vcs/directories/%s' % directory_sha1, data=body, headers={'Content-Type': serial.MIMETYPE}) assert rv.status_code == 204 assert rv.data == b'' # now it exists rv = self.app.get('/vcs/directories/%s' % directory_sha1) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == 'directory-sha16ee476a8be155ab049994f7170' # we update it rv = self.app.put('/vcs/directories/directory-sha16ee476a8be155ab049994f7170', data=serial.dumps({'entry': 'directory-bar'}), headers={'Content-Type': serial.MIMETYPE}) - assert rv.status_code == 200 - assert rv.data == b'Successful update!' + assert rv.status_code == 204 + assert rv.data == b'' # still the same rv = self.app.get('/vcs/directories/directory-sha16ee476a8be155ab049994f7170') # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == 'directory-sha16ee476a8be155ab049994f7170' diff --git a/swh/tests/test_api_occurrence.py b/swh/tests/test_api_occurrence.py index b3f9de1..d9b18b0 100644 --- a/swh/tests/test_api_occurrence.py +++ b/swh/tests/test_api_occurrence.py @@ -1,131 +1,131 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from nose.tools import istest from nose.plugins.attrib import attr from swh.store import db, models from swh.protocols import serial from test_utils import now, app_client, app_client_teardown @attr('slow') class OccurrenceTestCase(unittest.TestCase): def setUp(self): self.app, db_url, self.content_storage_dir = app_client() with db.connect(db_url) as db_conn: self.directory_sha1_hex = 'directory-sha16ee476a8be155ab049994f717e' models.add_directory(db_conn, self.directory_sha1_hex) authorAndCommitter = {'name': 'some-name', 'email': 'some-email'} models.add_person(db_conn, authorAndCommitter['name'], authorAndCommitter['email']) self.revision_sha1_hex = 'revision-sha1-to-test-existence9994f717e' models.add_revision(db_conn, self.revision_sha1_hex, now(), self.directory_sha1_hex, "revision message", authorAndCommitter, authorAndCommitter) self.origin_url = "https://github.com/user/repo" models.add_origin(db_conn, self.origin_url, 'git') self.reference_name = 'master' models.add_occurrence(db_conn, self.origin_url, self.reference_name, self.revision_sha1_hex) self.reference_name2 = 'master2' models.add_occurrence(db_conn, self.origin_url, self.reference_name2, self.revision_sha1_hex) self.revision_sha1_hex_2 = '2-revision-sha1-to-test-existence9994f71' models.add_revision(db_conn, self.revision_sha1_hex_2, now(), self.directory_sha1_hex, "revision message 2", authorAndCommitter, authorAndCommitter) def tearDown(self): app_client_teardown(self.content_storage_dir) @istest def get_occurrence_ok(self): # when rv = self.app.get('/vcs/occurrences/%s' % self.revision_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data) == [self.reference_name, self.reference_name2] @istest def get_occurrence_not_found(self): # when rv = self.app.get('/vcs/occurrences/inexistant-sha1') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def get_occurrence_not_found_with_bad_format(self): # when rv = self.app.get('/vcs/occurrences/1') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def put_occurrence_create_and_update(self): occ_revision_sha1_hex = self.revision_sha1_hex_2 rv = self.app.get('/vcs/occurrences/%s' % occ_revision_sha1_hex) # then assert rv.status_code == 404 assert rv.data == b'Not found!' # we create it body = serial.dumps({'revision': occ_revision_sha1_hex, # FIXME: redundant with the one from uri.. 'reference': 'master', 'url-origin': self.origin_url}) rv = self.app.put('/vcs/occurrences/%s' % occ_revision_sha1_hex, # ... here data=body, headers={'Content-Type': serial.MIMETYPE}) assert rv.status_code == 204 assert rv.data == b'' # now it exists rv = self.app.get('/vcs/occurrences/%s' % occ_revision_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data) == ['master'] # we update it rv = self.app.put('/vcs/occurrences/%s' % occ_revision_sha1_hex, data=body, headers={'Content-Type': serial.MIMETYPE}) - assert rv.status_code == 200 - assert rv.data == b'Successful update!' + assert rv.status_code == 204 + assert rv.data == b'' # still the same rv = self.app.get('/vcs/occurrences/%s' % occ_revision_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data) == ['master'] diff --git a/swh/tests/test_api_release.py b/swh/tests/test_api_release.py index 877d73d..4c1fc47 100644 --- a/swh/tests/test_api_release.py +++ b/swh/tests/test_api_release.py @@ -1,119 +1,119 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from nose.tools import istest from nose.plugins.attrib import attr from swh.store import db, models from swh.protocols import serial from test_utils import now, app_client, app_client_teardown @attr('slow') class ReleaseTestCase(unittest.TestCase): def setUp(self): self.app, db_url, self.content_storage_dir = app_client() with db.connect(db_url) as db_conn: self.directory_sha1_hex = 'directory-sha16ee476a8be155ab049994f717e' models.add_directory(db_conn, self.directory_sha1_hex) self.tagAuthor = {'name': 'tony', 'email': 'tony@mail.org'} models.add_person(db_conn, self.tagAuthor['name'], self.tagAuthor['email']) self.revision_sha1_hex = 'revision-sha1-to-test-existence9994f717e' models.add_revision(db_conn, self.revision_sha1_hex, now(), self.directory_sha1_hex, "revision message", self.tagAuthor, self.tagAuthor) self.release_sha1_hex = 'release-sha1-to-test-existence1234567901' models.add_release(db_conn, self.release_sha1_hex, self.revision_sha1_hex, now(), "0.0.1", "Super release tagged by tony", self.tagAuthor) def tearDown(self): app_client_teardown(self.content_storage_dir) @istest def get_release_ok(self): # when rv = self.app.get('/vcs/releases/%s' % self.release_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == self.release_sha1_hex @istest def get_release_not_found(self): # when rv = self.app.get('/vcs/releases/inexistant-sha1') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def get_release_not_found_with_bad_format(self): # when rv = self.app.get('/vcs/releases/1') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def put_release_create_and_update(self): release_sha1_hex = 'sha1-release46ee476a8be155ab049994f717e' rv = self.app.get('/vcs/releases/%s' % release_sha1_hex) # then assert rv.status_code == 404 assert rv.data == b'Not found!' # we create it body = serial.dumps({'id': release_sha1_hex, 'revision': self.revision_sha1_hex, 'date': now(), 'name': '0.0.1', 'comment': 'super release tagged by ardumont', 'author': self.tagAuthor}) rv = self.app.put('/vcs/releases/%s' % release_sha1_hex, data=body, headers={'Content-Type': serial.MIMETYPE}) assert rv.status_code == 204 assert rv.data == b'' # now it exists rv = self.app.get('/vcs/releases/%s' % release_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == release_sha1_hex # we update it rv = self.app.put('/vcs/releases/%s' % release_sha1_hex, data=body, headers={'Content-Type': serial.MIMETYPE}) - assert rv.status_code == 200 - assert rv.data == b'Successful update!' + assert rv.status_code == 204 + assert rv.data == b'' # still the same rv = self.app.get('/vcs/releases/%s' % release_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == release_sha1_hex diff --git a/swh/tests/test_api_revision.py b/swh/tests/test_api_revision.py index 6fa16ae..e29051d 100644 --- a/swh/tests/test_api_revision.py +++ b/swh/tests/test_api_revision.py @@ -1,110 +1,110 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from nose.tools import istest from nose.plugins.attrib import attr from swh.store import db, models from swh.protocols import serial from test_utils import now, app_client, app_client_teardown @attr('slow') class RevisionTestCase(unittest.TestCase): def setUp(self): self.app, db_url, self.content_storage_dir = app_client() with db.connect(db_url) as db_conn: self.directory_sha1_hex = 'directory-sha16ee476a8be155ab049994f717e' models.add_directory(db_conn, self.directory_sha1_hex) self.authorAndCommitter = {'name': 'some-name', 'email': 'some-email'} models.add_person(db_conn, self.authorAndCommitter['name'], self.authorAndCommitter['email']) self.revision_sha1_hex = 'revision-sha1-to-test-existence9994f717e' models.add_revision(db_conn, self.revision_sha1_hex, now(), self.directory_sha1_hex, "revision message", self.authorAndCommitter, self.authorAndCommitter) def tearDown(self): app_client_teardown(self.content_storage_dir) @istest def get_revision_ok(self): # when rv = self.app.get('/vcs/revisions/%s' % self.revision_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == self.revision_sha1_hex @istest def get_revision_not_found(self): # when rv = self.app.get('/vcs/revisions/inexistant-sha1') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def get_revision_not_found_with_bad_format(self): # when rv = self.app.get('/vcs/revisions/1') # then assert rv.status_code == 404 assert rv.data == b'Not found!' @istest def put_revision_create_and_update(self): revision_sha1_hex = 'sha1-revision46ee476a8be155ab049994f717e' rv = self.app.get('/vcs/revisions/%s' % revision_sha1_hex) # then assert rv.status_code == 404 assert rv.data == b'Not found!' # we create it body = serial.dumps({'date': now(), 'directory': self.directory_sha1_hex, 'message': 'revision message describing it', 'committer': self.authorAndCommitter, 'author': self.authorAndCommitter, 'parent-sha1s': [self.revision_sha1_hex]}) rv = self.app.put('/vcs/revisions/%s' % revision_sha1_hex, data=body, headers={'Content-Type': serial.MIMETYPE}) assert rv.status_code == 204 assert rv.data == b'' # now it exists rv = self.app.get('/vcs/revisions/%s' % revision_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == revision_sha1_hex # we update it rv = self.app.put('/vcs/revisions/%s' % revision_sha1_hex, data=body, headers={'Content-Type': serial.MIMETYPE}) - assert rv.status_code == 200 - assert rv.data == b'Successful update!' + assert rv.status_code == 204 + assert rv.data == b'' # still the same rv = self.app.get('/vcs/revisions/%s' % revision_sha1_hex) # then assert rv.status_code == 200 assert serial.loads(rv.data)['id'] == revision_sha1_hex