Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/conda/lister.py
| Show First 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | class CondaLister(StatelessLister[CondaListerPage]): | |||||||||||
| def get_pages(self) -> Iterator[CondaListerPage]: | def get_pages(self) -> Iterator[CondaListerPage]: | |||||||||||
| """Yield an iterator which returns 'page'""" | """Yield an iterator which returns 'page'""" | |||||||||||
| for arch in self.archs: | for arch in self.archs: | |||||||||||
| repodata_url = self.REPO_URL_PATTERN.format( | repodata_url = self.REPO_URL_PATTERN.format( | |||||||||||
| url=self.url, channel=self.channel, arch=arch | url=self.url, channel=self.channel, arch=arch | |||||||||||
| ) | ) | |||||||||||
| response = self.http_request(url=repodata_url) | response = self.http_request(url=repodata_url) | |||||||||||
| packages = json.loads(bz2.decompress(response.content))["packages"] | packages: Dict[str, Any] = json.loads(bz2.decompress(response.content))[ | |||||||||||
| "packages" | ||||||||||||
| ] | ||||||||||||
| yield (arch, packages) | yield (arch, packages) | |||||||||||
| def get_origins_from_page(self, page: CondaListerPage) -> Iterator[ListedOrigin]: | def get_origins_from_page(self, page: CondaListerPage) -> Iterator[ListedOrigin]: | |||||||||||
| """Iterate on all pages and yield ListedOrigin instances.""" | """Iterate on all pages and yield ListedOrigin instances.""" | |||||||||||
| assert self.lister_obj.id is not None | assert self.lister_obj.id is not None | |||||||||||
| arch, packages = page | arch, packages = page | |||||||||||
| for filename, package_metadata in packages.items(): | for filename, package_metadata in packages.items(): | |||||||||||
| artifact = { | version_key = ( | |||||||||||
| f"{arch}/{package_metadata['version']}-{package_metadata['build']}" | ||||||||||||
| ) | ||||||||||||
| artifact: Dict[str, Any] = { | ||||||||||||
| "filename": filename, | "filename": filename, | |||||||||||
| "url": self.ARCHIVE_URL_PATTERN.format( | "url": self.ARCHIVE_URL_PATTERN.format( | |||||||||||
| url=self.url, | url=self.url, | |||||||||||
| channel=self.channel, | channel=self.channel, | |||||||||||
| filename=filename, | filename=filename, | |||||||||||
| arch=arch, | arch=arch, | |||||||||||
| ), | ), | |||||||||||
| "version": package_metadata["version"], | "version": version_key, | |||||||||||
| "checksums": {}, | "checksums": {}, | |||||||||||
| } | } | |||||||||||
| for checksum in ("md5", "sha256"): | for checksum in ("md5", "sha256"): | |||||||||||
| if checksum in package_metadata: | if checksum in package_metadata: | |||||||||||
| artifact["checksums"][checksum] = package_metadata[checksum] | artifact["checksums"][checksum] = package_metadata[checksum] | |||||||||||
| version_key = ( | ||||||||||||
| f"{arch}/{package_metadata['version']}-{package_metadata['build']}" | ||||||||||||
| ) | ||||||||||||
| self.packages[package_metadata["name"]][version_key] = artifact | self.packages[package_metadata["name"]][version_key] = artifact | |||||||||||
| package_date = None | package_date = None | |||||||||||
| if "timestamp" in package_metadata: | if "timestamp" in package_metadata: | |||||||||||
| package_date = datetime.datetime.fromtimestamp( | package_date = datetime.datetime.fromtimestamp( | |||||||||||
| package_metadata["timestamp"] / 1e3, datetime.timezone.utc | package_metadata["timestamp"] / 1e3, datetime.timezone.utc | |||||||||||
| ) | ) | |||||||||||
| elif "date" in package_metadata: | elif "date" in package_metadata: | |||||||||||
| package_date = iso8601.parse_date(package_metadata["date"]) | package_date = iso8601.parse_date(package_metadata["date"]) | |||||||||||
| last_update = None | last_update = None | |||||||||||
| if package_date: | if package_date: | |||||||||||
| artifact["date"] = package_date.isoformat() | artifact["date"] = package_date.isoformat() | |||||||||||
| self.package_dates[package_metadata["name"]].append(package_date) | self.package_dates[package_metadata["name"]].append(package_date) | |||||||||||
| last_update = max(self.package_dates[package_metadata["name"]]) | last_update = max(self.package_dates[package_metadata["name"]]) | |||||||||||
| yield ListedOrigin( | yield ListedOrigin( | |||||||||||
| lister_id=self.lister_obj.id, | lister_id=self.lister_obj.id, | |||||||||||
| visit_type=self.VISIT_TYPE, | visit_type=self.VISIT_TYPE, | |||||||||||
| url=self.ORIGIN_URL_PATTERN.format( | url=self.ORIGIN_URL_PATTERN.format( | |||||||||||
| channel=self.channel, pkgname=package_metadata["name"] | channel=self.channel, pkgname=package_metadata["name"] | |||||||||||
| ), | ), | |||||||||||
| last_update=last_update, | last_update=last_update, | |||||||||||
| extra_loader_arguments={ | extra_loader_arguments={ | |||||||||||
| "artifacts": self.packages[package_metadata["name"]], | "artifacts": [ | |||||||||||
| v for k, v in self.packages[package_metadata["name"]].items() | ||||||||||||
| ], | ||||||||||||
| }, | }, | |||||||||||
vlorentzUnsubmitted Not Done Inline Actions
vlorentz: | ||||||||||||
| ) | ) | |||||||||||