Page MenuHomeSoftware Heritage
Differential D702

Add an empty tool configuration for the orig-int-meta-indexer.
ClosedPublic
Actions

Authored by vlorentz on Nov 23 2018, 1:29 PM.

Details

Reviewers
ardumont
Group Reviewers
Reviewers
Commits
rDCIDX519286b73291: Add an empty tool configuration for the orig-int-meta-indexer.
Summary

Also fixes docstrings.

Diff Detail

Repository
rDCIDX Metadata indexer
Branch
origin-intrinsic-metadata-indexer-tool
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 2552
Build 3172: tox-on-jenkinsJenkins
Build 3171: arc lint + arc unit

Event Timeline

vlorentz created this revision.Nov 23 2018, 1:29 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptNov 23 2018, 1:29 PM
swh-public-ci added a subscriber: swh-public-ci.Nov 23 2018, 1:32 PM

Build is green
See https://jenkins.softwareheritage.org/job/DCIDX/job/tox/60/ for more details.

Harbormaster completed remote builds in B2550: Diff 2214.Nov 23 2018, 1:32 PM
vlorentz updated this revision to Diff 2216.Nov 23 2018, 2:03 PM
  • Declare an empty list of tools, instead of an empty tool.
Harbormaster completed remote builds in B2552: Diff 2216.Nov 23 2018, 2:07 PM
swh-public-ci added a comment.Nov 23 2018, 2:07 PM

Build is green
See https://jenkins.softwareheritage.org/job/DCIDX/job/tox/61/ for more details.

ardumont accepted this revision.Nov 23 2018, 2:09 PM
ardumont added a subscriber: ardumont.

I have the feeling that tool setup is getting out of hand.
We will need to address it at some point.

This revision is now accepted and ready to land.Nov 23 2018, 2:09 PM
Closed by commit rDCIDX519286b73291: Add an empty tool configuration for the orig-int-meta-indexer. (authored by vlorentz). · Explain WhyNov 23 2018, 2:16 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathSize
swh/
indexer/
7 lines
20 lines
storage/
2 lines
tests/
7 lines
19 lines

Diff 2216

View Options

swh/indexer/indexer.py

Show First 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
def pretty_print_dict(dict, indent): def pretty_print_dict(dict, indent):
"""Pretty-print a list""" """Pretty-print a list"""
return ''.join('%s%s: %s\n' % return ''.join('%s%s: %s\n' %
(' ' * indent, click.style(key, bold=True), value) (' ' * indent, click.style(key, bold=True), value)
for key, value in dict.items()) for key, value in dict.items())
def pretty_print_task(task): def pretty_print_task(task, full=False):
"""Pretty-print a task""" """Pretty-print a task"""
vlorentzUnsubmitted
Not Done
Inline Actions

full could be documented.

vlorentz: `full` could be documented.
next_run = arrow.get(task['next_run']) next_run = arrow.get(task['next_run'])
lines = [ lines = [
'%s %s\n' % (click.style('Task', bold=True), task['id']), '%s %s\n' % (click.style('Task', bold=True), task['id']),
click.style(' Next run: ', bold=True), click.style(' Next run: ', bold=True),
"%s (%s)" % (next_run.humanize(locale=ARROW_LOCALE), "%s (%s)" % (next_run.humanize(locale=ARROW_LOCALE),
next_run.format()), next_run.format()),
'\n', '\n',
click.style(' Interval: ', bold=True), click.style(' Interval: ', bold=True),
str(task['current_interval']), '\n', str(task['current_interval']), '\n',
click.style(' Type: ', bold=True), task['type'], '\n', click.style(' Type: ', bold=True), task['type'] or '', '\n',
click.style(' Policy: ', bold=True), task['policy'], '\n', click.style(' Policy: ', bold=True), task['policy'] or '', '\n',
]
if full:
lines += [
click.style(' Status: ', bold=True),
task['status'] or '', '\n',
click.style(' Priority: ', bold=True),
task['priority'] or '', '\n',
]
lines += [
click.style(' Args:\n', bold=True), click.style(' Args:\n', bold=True),
pretty_print_list(task['arguments']['args'], indent=4), pretty_print_list(task['arguments']['args'], indent=4),
click.style(' Keyword args:\n', bold=True), click.style(' Keyword args:\n', bold=True),
pretty_print_dict(task['arguments']['kwargs'], indent=4), pretty_print_dict(task['arguments']['kwargs'], indent=4),
] ]
return ''.join(lines) return ''.join(lines)
▲ Show 20 Lines • Show All 197 Lines • ▼ Show 20 Lines for task_type in task_types:
len(pending), task_type)) len(pending), task_type))
for task in pending: for task in pending:
output.append(pretty_print_task(task)) output.append(pretty_print_task(task))
click.echo('\n'.join(output)) click.echo('\n'.join(output))
@task.command('list')
@click.option('--task-id', '-i', default=None, multiple=True,
help='List only tasks which id is given')
@click.option('--task-type', '-t', default=None, multiple=True,
help='List only tasks of type task-type')
@click.option('--limit', '-l', required=False, type=click.INT,
help='The maximum number of tasks to fetch')
@click.option('--status', '-s', multiple=True,
default=None,
ardumontUnsubmitted
Not Done
Inline Actions

List tasks whose status ...

Same for options below ;)

ardumont: List tasks whose status ... Same for options below ;)
help='List tasks which status is STATUS')
@click.option('--policy', '-p', default=None,
type=click.Choice(['recurring', 'oneshot']),
help='List tasks which policy is POLICY')
@click.option('--priority', '-P', default=None, multiple=True,
type=click.Choice(['all', 'low', 'normal', 'high']),
help='List tasks which priority is PRIORITY')
@click.option('--before', '-b', required=False, type=DATETIME,
help='List all tasks supposed to run before the given date')
@click.option('--after', '-a', required=False, type=DATETIME,
help='List all tasks supposed to run after the given date')
@click.pass_context
def list_tasks(ctx, task_id, task_type, limit, status, policy, priority,
before, after):
"""List tasks.
"""
scheduler = ctx.obj['scheduler']
if not scheduler:
raise ValueError('Scheduler class (local/remote) must be instantiated')
if not task_type:
task_type = [x['type'] for x in scheduler.get_task_types()]
# if task_id is not given, default value for status is
# 'next_run_not_scheduled'
# if task_id is given, default status is 'all'
if task_id is None and status is None:
status = ['next_run_not_scheduled']
if status and 'all' in status:
status = None
if priority and 'all' in priority:
priority = None
output = []
tasks = scheduler.search_tasks(
task_id=task_id,
task_type=task_type,
status=status, priority=priority, policy=policy,
before=before, after=after,
limit=limit)
output.append('Found %d tasks\n' % (
len(tasks)))
for task in tasks:
output.append(pretty_print_task(task, full=True))
click.echo('\n'.join(output))
@task.command('archive') @task.command('archive')
@click.option('--before', '-b', default=None, @click.option('--before', '-b', default=None,
help='''Task whose ended date is anterior will be archived. help='''Task whose ended date is anterior will be archived.
Default to current month's first day.''') Default to current month's first day.''')
@click.option('--after', '-a', default=None, @click.option('--after', '-a', default=None,
help='''Task whose ended date is after the specified date will help='''Task whose ended date is after the specified date will
be archived. Default to prior month's first day.''') be archived. Default to prior month's first day.''')
@click.option('--batch-index', default=1000, type=click.INT, @click.option('--batch-index', default=1000, type=click.INT,
▲ Show 20 Lines • Show All 242 Lines • Show Last 20 Lines
View Options

swh/indexer/metadata.py

Show First 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
def pretty_print_dict(dict, indent): def pretty_print_dict(dict, indent):
"""Pretty-print a list""" """Pretty-print a list"""
return ''.join('%s%s: %s\n' % return ''.join('%s%s: %s\n' %
(' ' * indent, click.style(key, bold=True), value) (' ' * indent, click.style(key, bold=True), value)
for key, value in dict.items()) for key, value in dict.items())
def pretty_print_task(task): def pretty_print_task(task, full=False):
"""Pretty-print a task""" """Pretty-print a task"""
vlorentzUnsubmitted
Not Done
Inline Actions

full could be documented.

vlorentz: `full` could be documented.
next_run = arrow.get(task['next_run']) next_run = arrow.get(task['next_run'])
lines = [ lines = [
'%s %s\n' % (click.style('Task', bold=True), task['id']), '%s %s\n' % (click.style('Task', bold=True), task['id']),
click.style(' Next run: ', bold=True), click.style(' Next run: ', bold=True),
"%s (%s)" % (next_run.humanize(locale=ARROW_LOCALE), "%s (%s)" % (next_run.humanize(locale=ARROW_LOCALE),
next_run.format()), next_run.format()),
'\n', '\n',
click.style(' Interval: ', bold=True), click.style(' Interval: ', bold=True),
str(task['current_interval']), '\n', str(task['current_interval']), '\n',
click.style(' Type: ', bold=True), task['type'], '\n', click.style(' Type: ', bold=True), task['type'] or '', '\n',
click.style(' Policy: ', bold=True), task['policy'], '\n', click.style(' Policy: ', bold=True), task['policy'] or '', '\n',
]
if full:
lines += [
click.style(' Status: ', bold=True),
task['status'] or '', '\n',
click.style(' Priority: ', bold=True),
task['priority'] or '', '\n',
]
lines += [
click.style(' Args:\n', bold=True), click.style(' Args:\n', bold=True),
pretty_print_list(task['arguments']['args'], indent=4), pretty_print_list(task['arguments']['args'], indent=4),
click.style(' Keyword args:\n', bold=True), click.style(' Keyword args:\n', bold=True),
pretty_print_dict(task['arguments']['kwargs'], indent=4), pretty_print_dict(task['arguments']['kwargs'], indent=4),
] ]
return ''.join(lines) return ''.join(lines)
▲ Show 20 Lines • Show All 197 Lines • ▼ Show 20 Lines for task_type in task_types:
len(pending), task_type)) len(pending), task_type))
for task in pending: for task in pending:
output.append(pretty_print_task(task)) output.append(pretty_print_task(task))
click.echo('\n'.join(output)) click.echo('\n'.join(output))
@task.command('list')
@click.option('--task-id', '-i', default=None, multiple=True,
help='List only tasks which id is given')
@click.option('--task-type', '-t', default=None, multiple=True,
help='List only tasks of type task-type')
@click.option('--limit', '-l', required=False, type=click.INT,
help='The maximum number of tasks to fetch')
@click.option('--status', '-s', multiple=True,
default=None,
ardumontUnsubmitted
Not Done
Inline Actions

List tasks whose status ...

Same for options below ;)

ardumont: List tasks whose status ... Same for options below ;)
help='List tasks which status is STATUS')
@click.option('--policy', '-p', default=None,
type=click.Choice(['recurring', 'oneshot']),
help='List tasks which policy is POLICY')
@click.option('--priority', '-P', default=None, multiple=True,
type=click.Choice(['all', 'low', 'normal', 'high']),
help='List tasks which priority is PRIORITY')
@click.option('--before', '-b', required=False, type=DATETIME,
help='List all tasks supposed to run before the given date')
@click.option('--after', '-a', required=False, type=DATETIME,
help='List all tasks supposed to run after the given date')
@click.pass_context
def list_tasks(ctx, task_id, task_type, limit, status, policy, priority,
before, after):
"""List tasks.
"""
scheduler = ctx.obj['scheduler']
if not scheduler:
raise ValueError('Scheduler class (local/remote) must be instantiated')
if not task_type:
task_type = [x['type'] for x in scheduler.get_task_types()]
# if task_id is not given, default value for status is
# 'next_run_not_scheduled'
# if task_id is given, default status is 'all'
if task_id is None and status is None:
status = ['next_run_not_scheduled']
if status and 'all' in status:
status = None
if priority and 'all' in priority:
priority = None
output = []
tasks = scheduler.search_tasks(
task_id=task_id,
task_type=task_type,
status=status, priority=priority, policy=policy,
before=before, after=after,
limit=limit)
output.append('Found %d tasks\n' % (
len(tasks)))
for task in tasks:
output.append(pretty_print_task(task, full=True))
click.echo('\n'.join(output))
@task.command('archive') @task.command('archive')
@click.option('--before', '-b', default=None, @click.option('--before', '-b', default=None,
help='''Task whose ended date is anterior will be archived. help='''Task whose ended date is anterior will be archived.
Default to current month's first day.''') Default to current month's first day.''')
@click.option('--after', '-a', default=None, @click.option('--after', '-a', default=None,
help='''Task whose ended date is after the specified date will help='''Task whose ended date is after the specified date will
be archived. Default to prior month's first day.''') be archived. Default to prior month's first day.''')
@click.option('--batch-index', default=1000, type=click.INT, @click.option('--batch-index', default=1000, type=click.INT,
▲ Show 20 Lines • Show All 242 Lines • Show Last 20 Lines
View Options

swh/indexer/storage/__init__.py

Show First 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
def pretty_print_dict(dict, indent): def pretty_print_dict(dict, indent):
"""Pretty-print a list""" """Pretty-print a list"""
return ''.join('%s%s: %s\n' % return ''.join('%s%s: %s\n' %
(' ' * indent, click.style(key, bold=True), value) (' ' * indent, click.style(key, bold=True), value)
for key, value in dict.items()) for key, value in dict.items())
def pretty_print_task(task): def pretty_print_task(task, full=False):
"""Pretty-print a task""" """Pretty-print a task"""
vlorentzUnsubmitted
Not Done
Inline Actions

full could be documented.

vlorentz: `full` could be documented.
next_run = arrow.get(task['next_run']) next_run = arrow.get(task['next_run'])
lines = [ lines = [
'%s %s\n' % (click.style('Task', bold=True), task['id']), '%s %s\n' % (click.style('Task', bold=True), task['id']),
click.style(' Next run: ', bold=True), click.style(' Next run: ', bold=True),
"%s (%s)" % (next_run.humanize(locale=ARROW_LOCALE), "%s (%s)" % (next_run.humanize(locale=ARROW_LOCALE),
next_run.format()), next_run.format()),
'\n', '\n',
click.style(' Interval: ', bold=True), click.style(' Interval: ', bold=True),
str(task['current_interval']), '\n', str(task['current_interval']), '\n',
click.style(' Type: ', bold=True), task['type'], '\n', click.style(' Type: ', bold=True), task['type'] or '', '\n',
click.style(' Policy: ', bold=True), task['policy'], '\n', click.style(' Policy: ', bold=True), task['policy'] or '', '\n',
]
if full:
lines += [
click.style(' Status: ', bold=True),
task['status'] or '', '\n',
click.style(' Priority: ', bold=True),
task['priority'] or '', '\n',
]
lines += [
click.style(' Args:\n', bold=True), click.style(' Args:\n', bold=True),
pretty_print_list(task['arguments']['args'], indent=4), pretty_print_list(task['arguments']['args'], indent=4),
click.style(' Keyword args:\n', bold=True), click.style(' Keyword args:\n', bold=True),
pretty_print_dict(task['arguments']['kwargs'], indent=4), pretty_print_dict(task['arguments']['kwargs'], indent=4),
] ]
return ''.join(lines) return ''.join(lines)
▲ Show 20 Lines • Show All 197 Lines • ▼ Show 20 Lines for task_type in task_types:
len(pending), task_type)) len(pending), task_type))
for task in pending: for task in pending:
output.append(pretty_print_task(task)) output.append(pretty_print_task(task))
click.echo('\n'.join(output)) click.echo('\n'.join(output))
@task.command('list')
@click.option('--task-id', '-i', default=None, multiple=True,
help='List only tasks which id is given')
@click.option('--task-type', '-t', default=None, multiple=True,
help='List only tasks of type task-type')
@click.option('--limit', '-l', required=False, type=click.INT,
help='The maximum number of tasks to fetch')
@click.option('--status', '-s', multiple=True,
default=None,
ardumontUnsubmitted
Not Done
Inline Actions

List tasks whose status ...

Same for options below ;)

ardumont: List tasks whose status ... Same for options below ;)
help='List tasks which status is STATUS')
@click.option('--policy', '-p', default=None,
type=click.Choice(['recurring', 'oneshot']),
help='List tasks which policy is POLICY')
@click.option('--priority', '-P', default=None, multiple=True,
type=click.Choice(['all', 'low', 'normal', 'high']),
help='List tasks which priority is PRIORITY')
@click.option('--before', '-b', required=False, type=DATETIME,
help='List all tasks supposed to run before the given date')
@click.option('--after', '-a', required=False, type=DATETIME,
help='List all tasks supposed to run after the given date')
@click.pass_context
def list_tasks(ctx, task_id, task_type, limit, status, policy, priority,
before, after):
"""List tasks.
"""
scheduler = ctx.obj['scheduler']
if not scheduler:
raise ValueError('Scheduler class (local/remote) must be instantiated')
if not task_type:
task_type = [x['type'] for x in scheduler.get_task_types()]
# if task_id is not given, default value for status is
# 'next_run_not_scheduled'
# if task_id is given, default status is 'all'
if task_id is None and status is None:
status = ['next_run_not_scheduled']
if status and 'all' in status:
status = None
if priority and 'all' in priority:
priority = None
output = []
tasks = scheduler.search_tasks(
task_id=task_id,
task_type=task_type,
status=status, priority=priority, policy=policy,
before=before, after=after,
limit=limit)
output.append('Found %d tasks\n' % (
len(tasks)))
for task in tasks:
output.append(pretty_print_task(task, full=True))
click.echo('\n'.join(output))
@task.command('archive') @task.command('archive')
@click.option('--before', '-b', default=None, @click.option('--before', '-b', default=None,
help='''Task whose ended date is anterior will be archived. help='''Task whose ended date is anterior will be archived.
Default to current month's first day.''') Default to current month's first day.''')
@click.option('--after', '-a', default=None, @click.option('--after', '-a', default=None,
help='''Task whose ended date is after the specified date will help='''Task whose ended date is after the specified date will
be archived. Default to prior month's first day.''') be archived. Default to prior month's first day.''')
@click.option('--batch-index', default=1000, type=click.INT, @click.option('--batch-index', default=1000, type=click.INT,
▲ Show 20 Lines • Show All 242 Lines • Show Last 20 Lines
View Options

swh/indexer/tests/test_origin_metadata.py

Show First 20 Lines • Show All 42 Lines • ▼ Show 20 Lines
def pretty_print_dict(dict, indent): def pretty_print_dict(dict, indent):
"""Pretty-print a list""" """Pretty-print a list"""
return ''.join('%s%s: %s\n' % return ''.join('%s%s: %s\n' %
(' ' * indent, click.style(key, bold=True), value) (' ' * indent, click.style(key, bold=True), value)
for key, value in dict.items()) for key, value in dict.items())
def pretty_print_task(task): def pretty_print_task(task, full=False):
"""Pretty-print a task""" """Pretty-print a task"""
vlorentzUnsubmitted
Not Done
Inline Actions

full could be documented.

vlorentz: `full` could be documented.
next_run = arrow.get(task['next_run']) next_run = arrow.get(task['next_run'])
lines = [ lines = [
'%s %s\n' % (click.style('Task', bold=True), task['id']), '%s %s\n' % (click.style('Task', bold=True), task['id']),
click.style(' Next run: ', bold=True), click.style(' Next run: ', bold=True),
"%s (%s)" % (next_run.humanize(locale=ARROW_LOCALE), "%s (%s)" % (next_run.humanize(locale=ARROW_LOCALE),
next_run.format()), next_run.format()),
'\n', '\n',
click.style(' Interval: ', bold=True), click.style(' Interval: ', bold=True),
str(task['current_interval']), '\n', str(task['current_interval']), '\n',
click.style(' Type: ', bold=True), task['type'], '\n', click.style(' Type: ', bold=True), task['type'] or '', '\n',
click.style(' Policy: ', bold=True), task['policy'], '\n', click.style(' Policy: ', bold=True), task['policy'] or '', '\n',
]
if full:
lines += [
click.style(' Status: ', bold=True),
task['status'] or '', '\n',
click.style(' Priority: ', bold=True),
task['priority'] or '', '\n',
]
lines += [
click.style(' Args:\n', bold=True), click.style(' Args:\n', bold=True),
pretty_print_list(task['arguments']['args'], indent=4), pretty_print_list(task['arguments']['args'], indent=4),
click.style(' Keyword args:\n', bold=True), click.style(' Keyword args:\n', bold=True),
pretty_print_dict(task['arguments']['kwargs'], indent=4), pretty_print_dict(task['arguments']['kwargs'], indent=4),
] ]
return ''.join(lines) return ''.join(lines)
▲ Show 20 Lines • Show All 197 Lines • ▼ Show 20 Lines for task_type in task_types:
len(pending), task_type)) len(pending), task_type))
for task in pending: for task in pending:
output.append(pretty_print_task(task)) output.append(pretty_print_task(task))
click.echo('\n'.join(output)) click.echo('\n'.join(output))
@task.command('list')
@click.option('--task-id', '-i', default=None, multiple=True,
help='List only tasks which id is given')
@click.option('--task-type', '-t', default=None, multiple=True,
help='List only tasks of type task-type')
@click.option('--limit', '-l', required=False, type=click.INT,
help='The maximum number of tasks to fetch')
@click.option('--status', '-s', multiple=True,
default=None,
ardumontUnsubmitted
Not Done
Inline Actions

List tasks whose status ...

Same for options below ;)

ardumont: List tasks whose status ... Same for options below ;)
help='List tasks which status is STATUS')
@click.option('--policy', '-p', default=None,
type=click.Choice(['recurring', 'oneshot']),
help='List tasks which policy is POLICY')
@click.option('--priority', '-P', default=None, multiple=True,
type=click.Choice(['all', 'low', 'normal', 'high']),
help='List tasks which priority is PRIORITY')
@click.option('--before', '-b', required=False, type=DATETIME,
help='List all tasks supposed to run before the given date')
@click.option('--after', '-a', required=False, type=DATETIME,
help='List all tasks supposed to run after the given date')
@click.pass_context
def list_tasks(ctx, task_id, task_type, limit, status, policy, priority,
before, after):
"""List tasks.
"""
scheduler = ctx.obj['scheduler']
if not scheduler:
raise ValueError('Scheduler class (local/remote) must be instantiated')
if not task_type:
task_type = [x['type'] for x in scheduler.get_task_types()]
# if task_id is not given, default value for status is
# 'next_run_not_scheduled'
# if task_id is given, default status is 'all'
if task_id is None and status is None:
status = ['next_run_not_scheduled']
if status and 'all' in status:
status = None
if priority and 'all' in priority:
priority = None
output = []
tasks = scheduler.search_tasks(
task_id=task_id,
task_type=task_type,
status=status, priority=priority, policy=policy,
before=before, after=after,
limit=limit)
output.append('Found %d tasks\n' % (
len(tasks)))
for task in tasks:
output.append(pretty_print_task(task, full=True))
click.echo('\n'.join(output))
@task.command('archive') @task.command('archive')
@click.option('--before', '-b', default=None, @click.option('--before', '-b', default=None,
help='''Task whose ended date is anterior will be archived. help='''Task whose ended date is anterior will be archived.
Default to current month's first day.''') Default to current month's first day.''')
@click.option('--after', '-a', default=None, @click.option('--after', '-a', default=None,
help='''Task whose ended date is after the specified date will help='''Task whose ended date is after the specified date will
be archived. Default to prior month's first day.''') be archived. Default to prior month's first day.''')
@click.option('--batch-index', default=1000, type=click.INT, @click.option('--batch-index', default=1000, type=click.INT,
▲ Show 20 Lines • Show All 242 Lines • Show Last 20 Lines
View Options

swh/indexer/tests/test_utils.py

Loading...
About · Developer information · Legal