diff --git a/resources/antelink/s3-injecter.ini b/resources/antelink/s3-injecter.ini index c1edb0f..d3c625f 100644 --- a/resources/antelink/s3-injecter.ini +++ b/resources/antelink/s3-injecter.ini @@ -1,4 +1,4 @@ [main] -storage_args = http://localhost:5000/ +storage_args = http://localhost:5002/ storage_class = remote_storage s3_folder = /home/storage/antelink/s3/ diff --git a/resources/antelink/sesi.ini b/resources/antelink/sesi.ini index 38057b1..a6d6122 100644 --- a/resources/antelink/sesi.ini +++ b/resources/antelink/sesi.ini @@ -1,3 +1,3 @@ [main] -storage_args = http://localhost:5000/ +storage_args = http://localhost:5002/ storage_class = remote_storage diff --git a/swh/loader/antelink/s3injecter.py b/swh/loader/antelink/s3injecter.py index 3c8f522..cfd765c 100644 --- a/swh/loader/antelink/s3injecter.py +++ b/swh/loader/antelink/s3injecter.py @@ -1,68 +1,68 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os from swh.core import config, hashutil from swh.storage import get_storage from swh.loader.antelink import utils class AntelinkS3Injecter(config.SWHConfig): """A bulk loader for downloading some file from s3. """ DEFAULT_CONFIG = { 'storage_class': ('str', 'remote_storage'), - 'storage_args': ('list[str]', ['http://localhost:5000/']), + 'storage_args': ('list[str]', ['http://localhost:5002/']), 's3_folder': ('string', '/srv/storage/space/antelink/s3'), } def __init__(self, config): self.config = config s3_folder = self.config['s3_folder'] if not s3_folder.endswith('/'): self.config['s3_folder'] = s3_folder + '/' self.storage = get_storage(config['storage_class'], config['storage_args']) self.log = logging.getLogger( 'swh.antelink.loader.AntelinkS3Injecter') def process_paths(self, paths): for localpath in paths: if not os.path.exists(localpath): self.log.error('%s does not exist!' % localpath) continue try: data = utils.to_content(localpath, log=self.log) # Check for corruption on sha1 origin_sha1 = utils.sha1_from_path(localpath) sha1 = hashutil.hash_to_hex(data['sha1']) if origin_sha1 != sha1: self.log.warn('%s corrupted - %s != %s. Skipping!' % (localpath, origin_sha1, sha1)) continue self.log.debug('%s -> swh' % sha1) yield data except Exception as e: self.log.error('Problem during checksums computation %s - %s' % (localpath, e)) continue def process(self, paths): # Then process them and store in swh data = self.process_paths( (self.config['s3_folder'] + p for p in paths)) self.storage.content_add(data) self.log.info('s3 - %s contents -> swh' % len(paths)) diff --git a/swh/loader/antelink/sesiinjecter.py b/swh/loader/antelink/sesiinjecter.py index fc155ec..50cc5d5 100644 --- a/swh/loader/antelink/sesiinjecter.py +++ b/swh/loader/antelink/sesiinjecter.py @@ -1,62 +1,62 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os from swh.core import config, hashutil from swh.storage import get_storage from swh.loader.antelink import utils class AntelinkSesiInjecter(config.SWHConfig): """A bulk loader for downloading some file from s3. """ DEFAULT_CONFIG = { 'storage_class': ('str', 'remote_storage'), - 'storage_args': ('list[str]', ['http://localhost:5000/']), + 'storage_args': ('list[str]', ['http://localhost:5002/']), } def __init__(self, config): self.config = config self.storage = get_storage(config['storage_class'], config['storage_args']) self.log = logging.getLogger( 'swh.antelink.loader.AntelinkSesiInjecter') def process_paths(self, paths): for localpath in paths: if not os.path.exists(localpath): self.log.warn('%s does not exist!' % localpath) continue try: data = utils.to_content(localpath, log=self.log) # Check for corruption on sha1 origin_sha1 = utils.sha1_from_path(localpath) sha1 = hashutil.hash_to_hex(data['sha1']) if origin_sha1 != sha1: self.log.warn('%s corrupted - %s != %s. Skipping!' % (localpath, origin_sha1, sha1)) continue self.log.debug('%s -> swh' % sha1) yield data except Exception as e: self.log.error('Problem during checksums computation %s - %s' % (localpath, e)) continue def process(self, paths): # Then process them and store in swh data = self.process_paths(paths) self.storage.content_add(data) self.log.info('sesi - %s contents -> swh' % len(paths))