diff --git a/swh/web/client/client.py b/swh/web/client/client.py --- a/swh/web/client/client.py +++ b/swh/web/client/client.py @@ -46,6 +46,8 @@ PIDish = Union[PID, str] +VISIT = "visit" + def _get_pid(pidish: PIDish) -> PID: """Parse string to PID if needed""" @@ -106,6 +108,10 @@ entry['target']) elif obj_type == CONTENT: pass # nothing to do for contents + elif obj_type == VISIT: + data['date'] = to_date(data['date']) + if data['snapshot'] is not None: + data['snapshot'] = to_pid(SNAPSHOT, data['snapshot']) else: raise ValueError(f'invalid object type: {obj_type}') @@ -326,6 +332,46 @@ else: done = True + def visits(self, + origin: str, + per_page: Optional[int] = None, + last_visit: Optional[int] = None, + **req_args) -> Generator[Dict[str, Any], None, None]: + """List visits of an origin + + Args: + origin: an origin + per_page: specify the number of visits to list + last_visit: visit to start listing from + req_args: extra keyword arguments for requests.get() + + Returns: + an iterator over visits of the origin + + Raises: + requests.HTTPError: if HTTP request fails + + """ + done = False + r = None + + params = [] + if last_visit is not None: + params.append(("last_visit", last_visit)) + if per_page is not None: + params.append(("per_page", per_page)) + + query = f'origin/{origin}/visits' + + while not done: + r = self._call(query, http_method='get', params=params, **req_args) + yield from [typify(v, VISIT) for v in r.json()] + if 'next' in r.links and 'url' in r.links['next']: + params = [] + query = r.links['next']['url'] + else: + done = True + def content_exists(self, pid: PIDish, **req_args) -> bool: """Check if a content object exists in the archive diff --git a/swh/web/client/tests/api_data.py b/swh/web/client/tests/api_data.py --- a/swh/web/client/tests/api_data.py +++ b/swh/web/client/tests/api_data.py @@ -7518,4 +7518,214 @@ "next_branch": null } """, # NoQA: E501 + "origin/https://github.com/NixOS/nixpkgs/visits?last_visit=50&per_page=10": # NoQA: E501 + r""" +[ + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 49, + "date": "2018-07-31T04:34:23.298931+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/49/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 48, + "date": "2018-07-27T10:12:23.069912+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/48/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 47, + "date": "2018-07-24T15:34:14.433021+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/47/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 46, + "date": "2018-07-23T22:07:33.025195+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/46/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 45, + "date": "2018-07-21T16:50:41.944584+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/45/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 44, + "date": "2018-07-21T12:17:24.044885+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/44/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 43, + "date": "2018-07-20T13:58:53.926748+00:00", + "type": "git", + "status": "full", + "snapshot": "c5d63cd27b09068b1f6913a90dfe44f1276091c2", + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/43/", + "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/c5d63cd27b09068b1f6913a90dfe44f1276091c2/" + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 42, + "date": "2018-07-20T00:38:20.356513+00:00", + "type": "git", + "status": "full", + "snapshot": "456550ea74af4e2eecaa406629efaaf0b9b5f976", + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/42/", + "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/456550ea74af4e2eecaa406629efaaf0b9b5f976/" + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 41, + "date": "2018-07-18T12:42:33.408489+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/41/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 40, + "date": "2018-07-17T16:40:42.302353+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/40/", + "snapshot_url": null + } +] + """, # NoQA: E501 + "origin/https://github.com/NixOS/nixpkgs/visits/?last_visit=40&per_page=10": # NoQA: E501 + r""" +[ + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 39, + "date": "2018-07-16T23:24:15.061318+00:00", + "type": "git", + "status": "full", + "snapshot": "1c2d51e9cda9958736394454ce8875e5913a1925", + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/39/", + "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/1c2d51e9cda9958736394454ce8875e5913a1925/" + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 38, + "date": "2018-07-16T10:43:14.651509+00:00", + "type": "git", + "status": "full", + "snapshot": "8fc8f205ba2d16a798b85467bd9f3cc6b938cc5c", + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/38/", + "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/8fc8f205ba2d16a798b85467bd9f3cc6b938cc5c/" + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 37, + "date": "2018-07-15T18:45:41.744499+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/37/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 36, + "date": "2018-07-15T10:41:05.001029+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/36/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 35, + "date": "2018-07-14T10:42:58.232545+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/35/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 34, + "date": "2018-07-14T05:31:45.565907+00:00", + "type": "git", + "status": "full", + "snapshot": "588fa42fefd27047474072da57ac2c83c3a481a7", + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/34/", + "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/588fa42fefd27047474072da57ac2c83c3a481a7/" + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 33, + "date": "2018-07-12T22:17:10.133596+00:00", + "type": "git", + "status": "full", + "snapshot": "9f9c7b2d52038b1d1fb6b1ff7387f0446c3f9da0", + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/33/", + "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/9f9c7b2d52038b1d1fb6b1ff7387f0446c3f9da0/" + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 32, + "date": "2018-07-12T14:45:54.291987+00:00", + "type": "git", + "status": "partial", + "snapshot": null, + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/32/", + "snapshot_url": null + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 31, + "date": "2018-07-11T18:30:43.843317+00:00", + "type": "git", + "status": "full", + "snapshot": "3d4c9c5c9a7de4d174b8a02d23bf4a55b2d3a911", + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/31/", + "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/3d4c9c5c9a7de4d174b8a02d23bf4a55b2d3a911/" + }, + { + "origin": "https://github.com/NixOS/nixpkgs", + "visit": 30, + "date": "2018-07-10T07:37:24.991560+00:00", + "type": "git", + "status": "full", + "snapshot": "100de51846f317e6ab48da79d985cefa6fdefe42", + "origin_visit_url": "https://archive.softwareheritage.org/api/1/origin/https://github.com/NixOS/nixpkgs/visit/30/", + "snapshot_url": "https://archive.softwareheritage.org/api/1/snapshot/100de51846f317e6ab48da79d985cefa6fdefe42/" + } +] + """, # NoQA: E501 } diff --git a/swh/web/client/tests/conftest.py b/swh/web/client/tests/conftest.py --- a/swh/web/client/tests/conftest.py +++ b/swh/web/client/tests/conftest.py @@ -20,6 +20,13 @@ "Link": f"<{API_URL}/{api_call}?branches_count=1000&branches_from=refs/tags/v3.0-rc7>; rel=\"next\"" # NoQA: E501 } + elif api_call == "origin/https://github.com/NixOS/nixpkgs/visits?last_visit=50&per_page=10": # NoQA: E501 + # monkey patch the only URL that require a special response headers + # (to make the client init and follow pagination) + headers = { + "Link": + f"<{API_URL}/origin/https://github.com/NixOS/nixpkgs/visits/?last_visit=40&per_page=10>; rel=\"next\"" # NoQA: E501 + } requests_mock.get(f"{API_URL}/{api_call}", text=data, headers=headers) return requests_mock diff --git a/swh/web/client/tests/gen-api-data.sh b/swh/web/client/tests/gen-api-data.sh --- a/swh/web/client/tests/gen-api-data.sh +++ b/swh/web/client/tests/gen-api-data.sh @@ -18,6 +18,8 @@ urls="${urls} snapshot/6a3a2cf0b2b90ce7ae1cf0a221ed68035b686f5a/" urls="${urls} snapshot/cabcc7d7bf639bbe1cc3b41989e1806618dd5764/" urls="${urls} snapshot/cabcc7d7bf639bbe1cc3b41989e1806618dd5764/?branches_count=1000&branches_from=refs/tags/v3.0-rc7" +urls="${urls} origin/https://github.com/NixOS/nixpkgs/visits?last_visit=50&per_page=10" +urls="${urls} origin/https://github.com/NixOS/nixpkgs/visits/?last_visit=40&per_page=10" echo "# GENERATED FILE, DO NOT EDIT." echo "# Run './gen-api-data.sh > api_data.py' instead." @@ -28,7 +30,7 @@ for url in ${urls} ; do echo " \"${url}\": # NoQA: E501" echo ' r"""' - curl --silent "${API_URL_real}/${url}" | jq --monochrome-output . + curl --location --silent "${API_URL_real}/${url}" | jq --monochrome-output . echo ' """, # NoQA: E501' done echo '}' diff --git a/swh/web/client/tests/test_web_api_client.py b/swh/web/client/tests/test_web_api_client.py --- a/swh/web/client/tests/test_web_api_client.py +++ b/swh/web/client/tests/test_web_api_client.py @@ -199,3 +199,18 @@ web_api_client.authenticate(refresh_token) assert e.match(repr(oidc_error_response)) + + +def test_get_visits(web_api_client, web_api_mock): + obj = web_api_client.visits('https://github.com/NixOS/nixpkgs', + last_visit=50, + per_page=10) + visits = [v for v in obj] + assert len(visits) == 20 + + timestamp = parse_date('2018-07-31 04:34:23.298931+00:00') + assert visits[0]['date'] == timestamp + + assert visits[0]["snapshot"] is None + snapshot_pid = 'swh:1:snp:456550ea74af4e2eecaa406629efaaf0b9b5f976' + assert visits[7]["snapshot"] == parse_pid(snapshot_pid)