Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/puppet/lister.py
| Show First 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | def get_origins_from_page(self, page: PuppetListerPage) -> Iterator[ListedOrigin]: | ||||
| dt_parse_pattern = "%Y-%m-%d %H:%M:%S %z" | dt_parse_pattern = "%Y-%m-%d %H:%M:%S %z" | ||||
| for entry in page: | for entry in page: | ||||
| last_update = datetime.strptime(entry["updated_at"], dt_parse_pattern) | last_update = datetime.strptime(entry["updated_at"], dt_parse_pattern) | ||||
| pkgname = entry["name"] | pkgname = entry["name"] | ||||
| owner = entry["owner"]["slug"] | owner = entry["owner"]["slug"] | ||||
| url = f"https://forge.puppet.com/modules/{owner}/{pkgname}" | url = f"https://forge.puppet.com/modules/{owner}/{pkgname}" | ||||
| artifacts = {} | artifacts = [] | ||||
| for release in entry["releases"]: | for release in entry["releases"]: | ||||
| # Build an artifact entry following original-artifacts-json specification | # Build an artifact entry following original-artifacts-json specification | ||||
| # https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950 | # https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950 | ||||
| checksums = {} | checksums = {} | ||||
| if release["version"] == entry["current_release"]["version"]: | if release["version"] == entry["current_release"]["version"]: | ||||
| # checksums are only available for current release | # checksums are only available for current release | ||||
| for checksum in ("md5", "sha256"): | for checksum in ("md5", "sha256"): | ||||
| checksums[checksum] = entry["current_release"][ | checksums[checksum] = entry["current_release"][ | ||||
| f"file_{checksum}" | f"file_{checksum}" | ||||
| ] | ] | ||||
| else: | else: | ||||
| # use file length as basic content check instead | # use file length as basic content check instead | ||||
| checksums["length"] = release["file_size"] | checksums["length"] = release["file_size"] | ||||
| artifacts[release["version"]] = { | artifacts.append( | ||||
| { | |||||
| "filename": release["file_uri"].split("/")[-1], | "filename": release["file_uri"].split("/")[-1], | ||||
| "url": urljoin(self.BASE_URL, release["file_uri"]), | "url": urljoin(self.BASE_URL, release["file_uri"]), | ||||
| "version": release["version"], | "version": release["version"], | ||||
| "last_update": datetime.strptime( | "last_update": datetime.strptime( | ||||
| release["created_at"], dt_parse_pattern | release["created_at"], dt_parse_pattern | ||||
| ).isoformat(), | ).isoformat(), | ||||
| "checksums": checksums, | "checksums": checksums, | ||||
| } | } | ||||
| ) | |||||
| yield ListedOrigin( | yield ListedOrigin( | ||||
| lister_id=self.lister_obj.id, | lister_id=self.lister_obj.id, | ||||
| visit_type=self.VISIT_TYPE, | visit_type=self.VISIT_TYPE, | ||||
| url=url, | url=url, | ||||
| last_update=last_update, | last_update=last_update, | ||||
| extra_loader_arguments={"artifacts": artifacts}, | extra_loader_arguments={"artifacts": artifacts}, | ||||
| ) | ) | ||||