Changeset View
Changeset View
Standalone View
Standalone View
swh/dataset/luigi.py
| Show First 20 Lines • Show All 655 Lines • ▼ Show 20 Lines | Example invocation:: | ||||
| --UploadExportToS3-local-export-path=/tmp/export_2022-11-08_staging/ \ | --UploadExportToS3-local-export-path=/tmp/export_2022-11-08_staging/ \ | ||||
| --formats=edges \ | --formats=edges \ | ||||
| --s3-export-path=s3://softwareheritage/graph/swh_2022-11-08 \ | --s3-export-path=s3://softwareheritage/graph/swh_2022-11-08 \ | ||||
| --athena-db-name=swh_20221108 \ | --athena-db-name=swh_20221108 \ | ||||
| --object-types=origin,origin_visit \ | --object-types=origin,origin_visit \ | ||||
| --s3-athena-output-location=s3://softwareheritage/graph/tmp/athena | --s3-athena-output-location=s3://softwareheritage/graph/tmp/athena | ||||
| """ | """ | ||||
| formats = luigi.EnumListParameter(enum=Format, batch_method=merge_lists) | formats = luigi.EnumListParameter( | ||||
| enum=Format, batch_method=merge_lists, default=list(Format) | |||||
| ) | |||||
| object_types = luigi.EnumListParameter( | object_types = luigi.EnumListParameter( | ||||
| enum=ObjectType, default=list(ObjectType), batch_method=merge_lists | enum=ObjectType, default=list(ObjectType), batch_method=merge_lists | ||||
| ) | ) | ||||
| s3_export_path = S3PathParameter() | s3_export_path = S3PathParameter() | ||||
| s3_athena_output_location = S3PathParameter() | s3_athena_output_location = S3PathParameter() | ||||
| athena_db_name = luigi.Parameter() | athena_db_name = luigi.Parameter() | ||||
| def requires(self) -> List[luigi.Task]: | def requires(self) -> List[luigi.Task]: | ||||
| Show All 21 Lines | |||||