Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/tests/test_scheduler.py
| Show First 20 Lines • Show All 758 Lines • ▼ Show 20 Lines | def _grab_next_visits_setup(self, swh_scheduler, listed_origins_by_type): | ||||
| recorded_origins = swh_scheduler.record_listed_origins(origins) | recorded_origins = swh_scheduler.record_listed_origins(origins) | ||||
| return visit_type, recorded_origins | return visit_type, recorded_origins | ||||
| def _check_grab_next_visit(self, swh_scheduler, visit_type, policy, expected): | def _check_grab_next_visit(self, swh_scheduler, visit_type, policy, expected): | ||||
| """Calls grab_next_visits with the passed policy, and check that all | """Calls grab_next_visits with the passed policy, and check that all | ||||
| the origins returned are the expected ones (in the same order), and | the origins returned are the expected ones (in the same order), and | ||||
| that no extra origins are returned. Also checks the origin visits have | that no extra origins are returned. Also checks the origin visits have | ||||
| been marked as scheduled.""" | been marked as scheduled, and are only re-scheduled a week later""" | ||||
| assert len(expected) != 0 | assert len(expected) != 0 | ||||
| before = utcnow() | before = utcnow() | ||||
| ret = swh_scheduler.grab_next_visits( | ret = swh_scheduler.grab_next_visits( | ||||
| visit_type=visit_type, | visit_type=visit_type, | ||||
| # Request one more than expected to check that no extra origin is returned | # Request one more than expected to check that no extra origin is returned | ||||
| count=len(expected) + 1, | count=len(expected) + 1, | ||||
| policy=policy, | policy=policy, | ||||
| ) | ) | ||||
| after = utcnow() | after = utcnow() | ||||
| assert ret == expected | assert ret == expected | ||||
| visit_stats_list = swh_scheduler.origin_visit_stats_get( | visit_stats_list = swh_scheduler.origin_visit_stats_get( | ||||
| [(origin.url, origin.visit_type) for origin in expected] | [(origin.url, origin.visit_type) for origin in expected] | ||||
| ) | ) | ||||
| assert len(visit_stats_list) == len(expected) | assert len(visit_stats_list) == len(expected) | ||||
| for visit_stats in visit_stats_list: | for visit_stats in visit_stats_list: | ||||
| # Check that last_scheduled got updated | # Check that last_scheduled got updated | ||||
| assert before <= visit_stats.last_scheduled <= after | assert before <= visit_stats.last_scheduled <= after | ||||
| # They should not be scheduled again | |||||
| ret = swh_scheduler.grab_next_visits( | |||||
| visit_type=visit_type, count=len(expected) + 1, policy=policy, | |||||
| ) | |||||
| assert ret == [], "grab_next_visits returned already-scheduled origins" | |||||
| # But a week later, they should | |||||
| ret = swh_scheduler.grab_next_visits( | |||||
| visit_type=visit_type, | |||||
| count=len(expected) + 1, | |||||
| policy=policy, | |||||
| timestamp=after + datetime.timedelta(days=7), | |||||
| ) | |||||
| # We need to sort them because their 'last_scheduled' field is updated to | |||||
| # exactly the same value, so the order is not deterministic | |||||
| assert sorted(ret) == sorted( | |||||
| expected | |||||
| ), "grab_next_visits didn't reschedule visits after a week" | |||||
| def test_grab_next_visits_oldest_scheduled_first( | def test_grab_next_visits_oldest_scheduled_first( | ||||
| self, swh_scheduler, listed_origins_by_type, | self, swh_scheduler, listed_origins_by_type, | ||||
| ): | ): | ||||
| visit_type, origins = self._grab_next_visits_setup( | visit_type, origins = self._grab_next_visits_setup( | ||||
| swh_scheduler, listed_origins_by_type | swh_scheduler, listed_origins_by_type | ||||
| ) | ) | ||||
| # Give all origins but one a last_scheduled date | # Give all origins but one a last_scheduled date | ||||
| ▲ Show 20 Lines • Show All 421 Lines • Show Last 20 Lines | |||||