Differential D8531 Diff 30830 swh/lister/puppet/lister.py

Changeset View

Standalone View

View Options

swh/lister/puppet/lister.py

This file was added.

# See the AUTHORS file at the top-level directory of this distribution

# License: GNU General Public License version 3, or any later version

# See top-level LICENSE file for more information

from datetime import datetime

import logging

from typing import Any, Dict, Iterator, List, Optional

from urllib.parse import urljoin

from swh.scheduler.interface import SchedulerInterface

from swh.scheduler.model import ListedOrigin

from ..pattern import CredentialsType, StatelessLister

logger = logging.getLogger(__name__)

# Aliasing the page results returned by `get_pages` method from the lister.

PuppetListerPage = List[Dict[str, Any]]

class PuppetLister(StatelessLister[PuppetListerPage]):

"""The Puppet lister list origins from 'Puppet Forge'"""

LISTER_NAME = "puppet"

VISIT_TYPE = "puppet"

INSTANCE = "puppet"

vlorentzUnsubmitted

Done

reuse the value defined in in pattern.py

vlorentz: reuse the value defined in in `pattern.py`

franckbretAuthorUnsubmitted

Done

I've change code to use self.http_request instead.

franckbret: I've change code to use self.http_request instead.

BASE_URL = "https://forgeapi.puppet.com/"

def __init__(

self,

scheduler: SchedulerInterface,

credentials: Optional[CredentialsType] = None,

vlorentzUnsubmitted

Done

class PuppetLister(StatelessLister[PuppetListerPage]):

- """The Puppet lister list origins from `Puppet Forge`"""

+ """The Puppet lister list origins from 'Puppet Forge'"""

LISTER_NAME = "puppet"

vlorentz:

super().__init__(

scheduler=scheduler,

credentials=credentials,

instance=self.INSTANCE,

url=self.BASE_URL,

)

def get_pages(self) -> Iterator[PuppetListerPage]:

"""Yield an iterator which returns 'page'

It request the http api endpoint to get a paginated results of modules,

and retrieve a `next` url. It ends when `next` json value is `null`.

Open Api specification for getModules endpoint:

https://forgeapi.puppet.com/#tag/Module-Operations/operation/getModules

"""

# limit = 100 is the max value for pagination

limit: int = 100

response = self.http_request(

f"{self.BASE_URL}v3/modules", params={"limit": limit}

)

data: Dict[str, Any] = response.json()

yield data["results"]

while data["pagination"]["next"]:

response = self.http_request(

urljoin(self.BASE_URL, data["pagination"]["next"])

)

data = response.json()

yield data["results"]

def get_origins_from_page(self, page: PuppetListerPage) -> Iterator[ListedOrigin]:

"""Iterate on all pages and yield ListedOrigin instances."""

assert self.lister_obj.id is not None

dt_parse_pattern = "%Y-%m-%d %H:%M:%S %z"

for entry in page:

last_update = datetime.strptime(entry["updated_at"], dt_parse_pattern)

pkgname = entry["name"]

owner = entry["owner"]["slug"]

url = f"https://forge.puppet.com/modules/{owner}/{pkgname}"

artifacts = {}

for release in entry["releases"]:

# Build an artifact entry following original-artifacts-json specification

# https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950

artifacts[release["version"]] = {

"filename": release["file_uri"].split("/")[-1],

"url": urljoin(self.BASE_URL, release["file_uri"]),

"version": release["version"],

vlorentzUnsubmitted

Done

and retrieve a `next` url. It ends when `next` json value is `null`.

- Open Api specification for getModules endpoint::

- https://forgeapi.puppet.com/#tag/Module-Operations/operation/getModules

+ Open Api specification for getModules endpoint:

+ https://forgeapi.puppet.com/#tag/Module-Operations/operation/getModules

"""

# limit = 100 is the max value for pagination

to make it clickable

vlorentz: to make it clickable

"last_update": datetime.strptime(

release["created_at"], dt_parse_pattern

).isoformat(),

}

yield ListedOrigin(

lister_id=self.lister_obj.id,

visit_type=self.VISIT_TYPE,

url=url,

last_update=last_update,

extra_loader_arguments={"artifacts": artifacts},

)

vlorentzUnsubmitted

Done

lol, the Puppet Forge documentation advertises this format as ISO 8601

*head desk*

vlorentz: lol, the Puppet Forge documentation advertises this format as ISO 8601 *head desk*

franckbretAuthorUnsubmitted

Done

lol, the Puppet Forge documentation advertises this format as ISO 8601

*head desk*

yup

franckbret: > lol, the Puppet Forge documentation advertises this format as ISO 8601 > > *head desk* yup