Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/bitbucket/tests/test_lister.py
| # Copyright (C) 2017-2021 The Software Heritage developers | # Copyright (C) 2017-2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from datetime import datetime | from datetime import datetime | ||||
| import json | import json | ||||
| import os | import os | ||||
| ▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | ): | ||||
| assert lister.updated | assert lister.updated | ||||
| lister_state = lister.get_state_from_scheduler() | lister_state = lister.get_state_from_scheduler() | ||||
| last_repo_cdate = lister_state.last_repo_cdate.isoformat() | last_repo_cdate = lister_state.last_repo_cdate.isoformat() | ||||
| assert hasattr(lister_state, "last_repo_cdate") | assert hasattr(lister_state, "last_repo_cdate") | ||||
| assert last_repo_cdate == bb_api_repositories_page2["values"][-1]["created_on"] | assert last_repo_cdate == bb_api_repositories_page2["values"][-1]["created_on"] | ||||
| # Second listing, restarting from last state | # Second listing, restarting from last state | ||||
| lister.session.get = mocker.spy(lister.session, "get") | lister.session.request = mocker.spy(lister.session, "request") | ||||
| lister.run() | lister.run() | ||||
| url_params = lister.url_params | url_params = lister.url_params | ||||
| url_params["after"] = last_repo_cdate | url_params["after"] = last_repo_cdate | ||||
| lister.session.get.assert_called_once_with(lister.API_URL, params=url_params) | lister.session.request.assert_called_once_with( | ||||
| "GET", lister.API_URL, params=url_params | |||||
| ) | |||||
| all_origins = ( | all_origins = ( | ||||
| bb_api_repositories_page1["values"] + bb_api_repositories_page2["values"] | bb_api_repositories_page1["values"] + bb_api_repositories_page2["values"] | ||||
| ) | ) | ||||
| _check_listed_origins(lister.get_origins_from_page(all_origins), scheduler_origins) | _check_listed_origins(lister.get_origins_from_page(all_origins), scheduler_origins) | ||||
| Show All 13 Lines | requests_mock.get( | ||||
| {"json": None, "status_code": 429}, | {"json": None, "status_code": 429}, | ||||
| {"json": None, "status_code": 429}, | {"json": None, "status_code": 429}, | ||||
| {"json": bb_api_repositories_page2, "status_code": 200}, | {"json": bb_api_repositories_page2, "status_code": 200}, | ||||
| ], | ], | ||||
| ) | ) | ||||
| lister = BitbucketLister(scheduler=swh_scheduler, page_size=10) | lister = BitbucketLister(scheduler=swh_scheduler, page_size=10) | ||||
| mocker.patch.object(lister.page_request.retry, "sleep") | mocker.patch.object(lister.http_request.retry, "sleep") | ||||
| stats = lister.run() | stats = lister.run() | ||||
| assert stats.pages == 2 | assert stats.pages == 2 | ||||
| assert stats.origins == 20 | assert stats.origins == 20 | ||||
| assert len(swh_scheduler.get_listed_origins(lister.lister_obj.id).results) == 20 | assert len(swh_scheduler.get_listed_origins(lister.lister_obj.id).results) == 20 | ||||
| ▲ Show 20 Lines • Show All 61 Lines • Show Last 20 Lines | |||||