diff --git a/swh/lister/packagist/lister.py b/swh/lister/packagist/lister.py --- a/swh/lister/packagist/lister.py +++ b/swh/lister/packagist/lister.py @@ -11,6 +11,7 @@ import iso8601 import requests +from swh.core.github.utils import GitHubSession from swh.scheduler.interface import SchedulerInterface from swh.scheduler.model import ListedOrigin @@ -63,6 +64,10 @@ self.session.headers.update({"Accept": "application/json"}) self.listing_date = datetime.now().astimezone(tz=timezone.utc) + self.github_session = GitHubSession( + credentials=self.credentials, + user_agent=str(self.session.headers["User-Agent"]), + ) def state_from_dict(self, d: Dict[str, Any]) -> PackagistListerState: last_listing_date = d.get("last_listing_date") diff --git a/swh/lister/packagist/tests/data/maxmilhas_specification-pattern.json b/swh/lister/packagist/tests/data/maxmilhas_specification-pattern.json new file mode 100644 --- /dev/null +++ b/swh/lister/packagist/tests/data/maxmilhas_specification-pattern.json @@ -0,0 +1,85 @@ +{ + "packages": { + "maxmilhas/specification-pattern": { + "1.0.0": { + "name": "maxmilhas/specification-pattern", + "description": "Implementation of the Specification Design Pettern in PHP", + "keywords": [], + "homepage": "", + "version": "1.0.0", + "version_normalized": "1.0.0.0", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "MaxMilhas", + "email": "ti@maxmilhas.com.br" + } + ], + "source": { + "url": "https://github.com/maxmilhas/SpecificationPatternPHP.git", + "type": "git", + "reference": "017aac2a7b245983e1863a3e9db65d1ad010de9b" + }, + "dist": { + "url": "https://api.github.com/repos/maxmilhas/SpecificationPatternPHP/zipball/017aac2a7b245983e1863a3e9db65d1ad010de9b", + "type": "zip", + "shasum": "", + "reference": "017aac2a7b245983e1863a3e9db65d1ad010de9b" + }, + "type": "library", + "time": "2015-07-03T21:01:33+00:00", + "autoload": { + "psr-4": { + "MaxMilhas\\DesignPatterns\\": [ + "src/", + "tests/src/" + ] + } + }, + "uid": 452100 + }, + "dev-master": { + "name": "maxmilhas/specification-pattern", + "description": "Implementation of the Specification Design Pettern in PHP", + "keywords": [], + "homepage": "", + "version": "dev-master", + "version_normalized": "9999999-dev", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "MaxMilhas", + "email": "ti@maxmilhas.com.br" + } + ], + "source": { + "url": "https://github.com/maxmilhas-admin/SpecificationPatternPHP.git", + "type": "git", + "reference": "9f22ef2ebff0bba687eb3e7dad6873ac8d3ee246" + }, + "dist": { + "url": "https://api.github.com/repos/maxmilhas-admin/SpecificationPatternPHP/zipball/9f22ef2ebff0bba687eb3e7dad6873ac8d3ee246", + "type": "zip", + "shasum": "", + "reference": "9f22ef2ebff0bba687eb3e7dad6873ac8d3ee246" + }, + "type": "library", + "time": "2015-08-04T14:20:59+00:00", + "autoload": { + "psr-4": { + "MaxMilhas\\DesignPatterns\\": [ + "src/", + "tests/src/" + ] + } + }, + "default-branch": true, + "uid": 4063205 + } + } + } +} diff --git a/swh/lister/packagist/tests/test_lister.py b/swh/lister/packagist/tests/test_lister.py --- a/swh/lister/packagist/tests/test_lister.py +++ b/swh/lister/packagist/tests/test_lister.py @@ -150,6 +150,38 @@ assert stats.origins == 0 +def test_packagist_lister_package_normalize_github_origin( + swh_scheduler, requests_mock, datadir +): + package_name = "maxmilhas/specification-pattern" + lister = PackagistLister(scheduler=swh_scheduler) + requests_mock.get( + lister.PACKAGIST_PACKAGES_LIST_URL, json={"packageNames": [package_name]} + ) + requests_mock.get( + f"{lister.PACKAGIST_REPO_BASE_URL}/{package_name}.json", + additional_matcher=_request_without_if_modified_since, + json=_package_metadata(datadir, package_name), + ) + + stats = lister.run() + + assert stats.pages == 1 + assert stats.origins == 1 + + expected_origins = { + ( + "https://github.com/maxmilhas-admin/SpecificationPatternPHP.git", + "git", + datetime.datetime.fromisoformat("2015-08-04T14:20:59+00:00"), + ), + } + assert expected_origins == { + (o.url, o.visit_type, o.last_update) + for o in swh_scheduler.get_listed_origins(lister.lister_obj.id).results + } + + def test_lister_from_configfile(swh_scheduler_config, mocker): load_from_envvar = mocker.patch("swh.lister.pattern.load_from_envvar") load_from_envvar.return_value = {