Page MenuHomeSoftware Heritage
Files F11012862

No OneTemporary
Actions

View Options

diff --git a/setup.py b/setup.py
index 90b0b1f..c743ab7 100755
--- a/setup.py
+++ b/setup.py
@@ -1,69 +1,71 @@
#!/usr/bin/env python3
# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from setuptools import setup, find_packages
from os import path
from io import open
here = path.abspath(path.dirname(__file__))
# Get the long description from the README file
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
long_description = f.read()
def parse_requirements(name=None):
if name:
reqf = 'requirements-%s.txt' % name
else:
reqf = 'requirements.txt'
requirements = []
if not path.exists(reqf):
return requirements
with open(reqf) as f:
for line in f.readlines():
line = line.strip()
if not line or line.startswith('#'):
continue
requirements.append(line)
return requirements
setup(
name='swh.journal',
description='Software Heritage Journal utilities',
long_description=long_description,
long_description_content_type='text/markdown',
author='Software Heritage developers',
author_email='swh-devel@inria.fr',
url='https://forge.softwareheritage.org/diffusion/DJNL/',
packages=find_packages(),
scripts=[],
entry_points='''
[console_scripts]
swh-journal=swh.journal.cli:main
+ [swh.cli.subcommands]
+ journal=swh.journal.cli:cli
''',
install_requires=parse_requirements() + parse_requirements('swh'),
setup_requires=['vcversioner'],
extras_require={'testing': parse_requirements('test')},
vcversioner={},
include_package_data=True,
classifiers=[
"Programming Language :: Python :: 3",
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Operating System :: OS Independent",
"Development Status :: 5 - Production/Stable",
],
project_urls={
'Bug Reports': 'https://forge.softwareheritage.org/maniphest',
'Funding': 'https://www.softwareheritage.org/donate',
'Source': 'https://forge.softwareheritage.org/source/swh-journal',
},
)
diff --git a/swh/journal/cli.py b/swh/journal/cli.py
index 984acdb..f4e8acf 100644
--- a/swh/journal/cli.py
+++ b/swh/journal/cli.py
@@ -1,118 +1,118 @@
# Copyright (C) 2016-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import click
import functools
import logging
import os
from swh.core import config
from swh.storage import get_storage
from swh.journal.client import JournalClient
from swh.journal.replay import process_replay_objects
from swh.journal.backfill import JournalBackfiller
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
-@click.group(context_settings=CONTEXT_SETTINGS)
+@click.group(name='journal', context_settings=CONTEXT_SETTINGS)
@click.option('--config-file', '-C', default=None,
type=click.Path(exists=True, dir_okay=False,),
help="Configuration file.")
@click.option('--log-level', '-l', default='INFO',
type=click.Choice(logging._nameToLevel.keys()),
help="Log level (default to INFO)")
@click.pass_context
def cli(ctx, config_file, log_level):
"""Software Heritage Scheduler CLI interface
Default to use the the local scheduler instance (plugged to the
main scheduler db).
"""
if not config_file:
config_file = os.environ.get('SWH_CONFIG_FILENAME')
if not config_file:
raise ValueError('You must either pass a config-file parameter '
'or set SWH_CONFIG_FILENAME to target '
'the config-file')
if not os.path.exists(config_file):
raise ValueError('%s does not exist' % config_file)
conf = config.read(config_file)
ctx.ensure_object(dict)
logging.basicConfig(
level=log_level,
format='%(asctime)s %(levelname)s %(name)s %(message)s',
)
logging.getLogger('kafka').setLevel(logging.INFO)
ctx.obj['config'] = conf
ctx.obj['loglevel'] = log_level
@cli.command()
@click.option('--max-messages', '-m', default=None, type=int,
help='Maximum number of objects to replay. Default is to '
'run forever.')
@click.option('--broker', 'brokers', type=str, multiple=True,
help='Kafka broker to connect to.')
@click.option('--prefix', type=str, default='swh.journal.objects',
help='Prefix of Kafka topic names to read from.')
@click.option('--consumer-id', type=str,
help='Name of the consumer/group id for reading from Kafka.')
@click.pass_context
def replay(ctx, brokers, prefix, consumer_id, max_messages):
"""Fill a new storage by reading a journal.
"""
conf = ctx.obj['config']
logger = logging.getLogger(__name__)
logger.setLevel(ctx.obj['loglevel'])
storage = get_storage(**conf.pop('storage'))
client = JournalClient(brokers, prefix, consumer_id)
worker_fn = functools.partial(process_replay_objects, storage=storage)
try:
nb_messages = 0
while not max_messages or nb_messages < max_messages:
nb_messages += client.process(worker_fn)
logger.info('Processed %d messages.' % nb_messages)
except KeyboardInterrupt:
ctx.exit(0)
else:
print('Done.')
@cli.command()
@click.argument('object_type')
@click.option('--start-object', default=None)
@click.option('--end-object', default=None)
@click.option('--dry-run', is_flag=True, default=False)
@click.pass_context
def backfiller(ctx, object_type, start_object, end_object, dry_run):
"""Manipulate backfiller
"""
conf = ctx.obj['config']
backfiller = JournalBackfiller(conf)
try:
backfiller.run(
object_type=object_type,
start_object=start_object, end_object=end_object,
dry_run=dry_run)
except KeyboardInterrupt:
ctx.exit(0)
def main():
return cli(auto_envvar_prefix='SWH_JOURNAL')
if __name__ == '__main__':
main()

File Metadata

Mime Type
text/x-diff
Expires
Thu, Sep 18, 4:39 AM (1 d, 12 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3297248

Event Timeline

About · Developer information · Legal