Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cli.py
Show First 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | def serve(ctx, config_path, host, port, debug): | ||||
""" | """ | ||||
if "log_level" in ctx.obj: | if "log_level" in ctx.obj: | ||||
logging.getLogger("werkzeug").setLevel(ctx.obj["log_level"]) | logging.getLogger("werkzeug").setLevel(ctx.obj["log_level"]) | ||||
api_cfg = load_and_check_config(config_path, type="any") | api_cfg = load_and_check_config(config_path, type="any") | ||||
app.config.update(api_cfg) | app.config.update(api_cfg) | ||||
app.run(host, port=int(port), debug=bool(debug)) | app.run(host, port=int(port), debug=bool(debug)) | ||||
@storage.command() | |||||
@click.argument("object_type") | |||||
@click.option("--start-object", default=None) | |||||
@click.option("--end-object", default=None) | |||||
@click.option("--dry-run", is_flag=True, default=False) | |||||
@click.pass_context | |||||
def backfiller(ctx, object_type, start_object, end_object, dry_run): | |||||
"""Run the backfiller | |||||
The backfiller list objects from a Storage and produce journal entries from | |||||
there. | |||||
Typically used to rebuild a journal or compensate for missing objects in a | |||||
journal (eg. due to a downtime of this later). | |||||
The configuration file requires the following entries: | |||||
- brokers: a list of kafka endpoints (the journal) in which entries will be | |||||
added. | |||||
- storage_dbconn: URL to connect to the storage DB. | |||||
- prefix: the prefix of the topics (topics will be <prefix>.<object_type>). | |||||
- client_id: the kafka client ID. | |||||
""" | |||||
# for "lazy" loading | |||||
from swh.storage.backfill import JournalBackfiller | |||||
try: | |||||
from systemd.daemon import notify | |||||
except ImportError: | |||||
notify = None | |||||
conf = ctx.obj["config"] | |||||
backfiller = JournalBackfiller(conf) | |||||
if notify: | |||||
notify("READY=1") | |||||
try: | |||||
backfiller.run( | |||||
object_type=object_type, | |||||
start_object=start_object, | |||||
end_object=end_object, | |||||
dry_run=dry_run, | |||||
) | |||||
except KeyboardInterrupt: | |||||
if notify: | |||||
notify("STOPPING=1") | |||||
ctx.exit(0) | |||||
def main(): | def main(): | ||||
logging.basicConfig() | logging.basicConfig() | ||||
return serve(auto_envvar_prefix="SWH_STORAGE") | return serve(auto_envvar_prefix="SWH_STORAGE") | ||||
if __name__ == "__main__": | if __name__ == "__main__": | ||||
main() | main() |