Changeset View
Changeset View
Standalone View
Standalone View
swh/dataset/luigi.py
Show First 20 Lines • Show All 265 Lines • ▼ Show 20 Lines | margin = FractionalFloatParameter( | ||||
in other words, start earlier than last committed position. | in other words, start earlier than last committed position. | ||||
""", | """, | ||||
) | ) | ||||
object_types = luigi.EnumListParameter( | object_types = luigi.EnumListParameter( | ||||
enum=ObjectType, default=list(ObjectType), batch_method=merge_lists | enum=ObjectType, default=list(ObjectType), batch_method=merge_lists | ||||
) | ) | ||||
def output(self) -> List[luigi.Target]: | def output(self) -> List[luigi.Target]: | ||||
"""Returns stamp and meta paths on the local FS.""" | """Returns path of `meta/export.json` on the local FS.""" | ||||
return self._stamps() + [self._meta()] | return [self._meta()] | ||||
def complete(self) -> bool: | |||||
return super().complete() and _export_metadata_has_object_types( | |||||
self._meta(), self.object_types | |||||
) | |||||
def _stamps(self): | def _stamps(self): | ||||
return [ | return [ | ||||
luigi.LocalTarget(self.local_export_path / path) | luigi.LocalTarget(self.local_export_path / path) | ||||
for path in stamps_paths(self.formats, self.object_types) | for path in stamps_paths(self.formats, self.object_types) | ||||
] | ] | ||||
def _meta(self): | def _meta(self): | ||||
▲ Show 20 Lines • Show All 412 Lines • Show Last 20 Lines |