Page MenuHomeSoftware Heritage
Differential D1742

Make the backfiller include visit ids.
ClosedPublic
Actions

Authored by vlorentz on Jul 17 2019, 3:11 PM.

Details

Reviewers
ardumont
Group Reviewers
Reviewers
Commits
rDJNLa70dce845662: Make the backfiller include visit ids.
Summary

They are needed by the replayer.

Diff Detail

Repository
rDJNL Journal infrastructure
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathSize
swh/
journal/
2 lines
tests/
4 lines

Diff 5865

View Options

swh/journal/backfill.py

Show First 20 Lines • Show All 321 Lines • ▼ Show 20 Lines ) -> List[ListedOrigin]:
"""Get at most the `count` next origins that need to be visited with """Get at most the `count` next origins that need to be visited with
the `visit_type` loader according to the given scheduling `policy`. the `visit_type` loader according to the given scheduling `policy`.
This will mark the origins as "being visited" in the listed_origins This will mark the origins as "being visited" in the listed_origins
table, to avoid scheduling multiple visits to the same origin. table, to avoid scheduling multiple visits to the same origin.
""" """
origin_select_cols = ", ".join(ListedOrigin.select_columns()) origin_select_cols = ", ".join(ListedOrigin.select_columns())
# TODO: filter on last_scheduled "too recent" to avoid always
# re-scheduling the same tasks.
ardumontUnsubmitted
Not Done
Inline Actions

what does 'enabled' mean here?

I gather that ends up in the query like "WHERE enabled AND visit_type=%s"
but i don't know what that means exactly.

ardumont: what does 'enabled' mean here? I gather that ends up in the query like "WHERE enabled AND…
vlorentzAuthorUnsubmitted
Done
Inline Actions

Whether this origin has been seen during the last listing, and visits should be scheduled

vlorentz: Whether this origin has been seen during the last listing, and visits should be scheduled
olasdUnsubmitted
Not Done
Inline Actions

I don't think the enabled field is ever updated currently. But we will, eventually.

Obviously this would deserve a comment rather than being snuck in.

olasd: I don't think the `enabled` field is ever updated currently. But we will, eventually.
where_clauses = [
"enabled", # "NOT enabled" = the lister said the origin no longer exists
"visit_type = %s",
]
if policy == "oldest_scheduled_first": if policy == "oldest_scheduled_first":
query = f""" order_by = "origin_visit_stats.last_scheduled NULLS FIRST"
WITH selected_origins AS ( else:
raise UnknownPolicy(f"Unknown scheduling policy {policy}")
select_query = f"""
SELECT SELECT
{origin_select_cols} {origin_select_cols}
FROM FROM
listed_origins listed_origins
LEFT JOIN LEFT JOIN
origin_visit_stats USING (url, visit_type) origin_visit_stats USING (url, visit_type)
WHERE WHERE
visit_type = %s {" AND ".join(where_clauses)}
ORDER BY ORDER BY
origin_visit_stats.last_scheduled NULLS FIRST {order_by}
LIMIT %s LIMIT %s
"""
query = f"""
WITH selected_origins AS (
{select_query}
), ),
update_stats AS ( update_stats AS (
INSERT INTO INSERT INTO
origin_visit_stats ( origin_visit_stats (
url, visit_type, last_scheduled url, visit_type, last_scheduled
) )
SELECT SELECT
url, visit_type, now() url, visit_type, now()
FROM FROM
selected_origins selected_origins
ON CONFLICT (url, visit_type) DO UPDATE ON CONFLICT (url, visit_type) DO UPDATE
SET last_scheduled = GREATEST(origin_visit_stats.last_scheduled, EXCLUDED.last_scheduled) SET last_scheduled = GREATEST(
origin_visit_stats.last_scheduled,
EXCLUDED.last_scheduled
)
) )
SELECT SELECT
* *
FROM FROM
selected_origins selected_origins
""" # noqa """
cur.execute(query, (visit_type, count))
cur.execute(query, (visit_type, count))
return [ListedOrigin(**d) for d in cur] return [ListedOrigin(**d) for d in cur]
else:
raise UnknownPolicy(f"Unknown scheduling policy {policy}")
task_create_keys = [ task_create_keys = [
"type", "type",
"arguments", "arguments",
"next_run", "next_run",
"policy", "policy",
"status", "status",
"retries_left", "retries_left",
▲ Show 20 Lines • Show All 567 Lines • Show Last 20 Lines
View Options

swh/journal/tests/test_backfill.py

Show First 20 Lines • Show All 321 Lines • ▼ Show 20 Lines ) -> List[ListedOrigin]:
"""Get at most the `count` next origins that need to be visited with """Get at most the `count` next origins that need to be visited with
the `visit_type` loader according to the given scheduling `policy`. the `visit_type` loader according to the given scheduling `policy`.
This will mark the origins as "being visited" in the listed_origins This will mark the origins as "being visited" in the listed_origins
table, to avoid scheduling multiple visits to the same origin. table, to avoid scheduling multiple visits to the same origin.
""" """
origin_select_cols = ", ".join(ListedOrigin.select_columns()) origin_select_cols = ", ".join(ListedOrigin.select_columns())
# TODO: filter on last_scheduled "too recent" to avoid always
# re-scheduling the same tasks.
ardumontUnsubmitted
Not Done
Inline Actions

what does 'enabled' mean here?

I gather that ends up in the query like "WHERE enabled AND visit_type=%s"
but i don't know what that means exactly.

ardumont: what does 'enabled' mean here? I gather that ends up in the query like "WHERE enabled AND…
vlorentzAuthorUnsubmitted
Done
Inline Actions

Whether this origin has been seen during the last listing, and visits should be scheduled

vlorentz: Whether this origin has been seen during the last listing, and visits should be scheduled
olasdUnsubmitted
Not Done
Inline Actions

I don't think the enabled field is ever updated currently. But we will, eventually.

Obviously this would deserve a comment rather than being snuck in.

olasd: I don't think the `enabled` field is ever updated currently. But we will, eventually.
where_clauses = [
"enabled", # "NOT enabled" = the lister said the origin no longer exists
"visit_type = %s",
]
if policy == "oldest_scheduled_first": if policy == "oldest_scheduled_first":
query = f""" order_by = "origin_visit_stats.last_scheduled NULLS FIRST"
WITH selected_origins AS ( else:
raise UnknownPolicy(f"Unknown scheduling policy {policy}")
select_query = f"""
SELECT SELECT
{origin_select_cols} {origin_select_cols}
FROM FROM
listed_origins listed_origins
LEFT JOIN LEFT JOIN
origin_visit_stats USING (url, visit_type) origin_visit_stats USING (url, visit_type)
WHERE WHERE
visit_type = %s {" AND ".join(where_clauses)}
ORDER BY ORDER BY
origin_visit_stats.last_scheduled NULLS FIRST {order_by}
LIMIT %s LIMIT %s
"""
query = f"""
WITH selected_origins AS (
{select_query}
), ),
update_stats AS ( update_stats AS (
INSERT INTO INSERT INTO
origin_visit_stats ( origin_visit_stats (
url, visit_type, last_scheduled url, visit_type, last_scheduled
) )
SELECT SELECT
url, visit_type, now() url, visit_type, now()
FROM FROM
selected_origins selected_origins
ON CONFLICT (url, visit_type) DO UPDATE ON CONFLICT (url, visit_type) DO UPDATE
SET last_scheduled = GREATEST(origin_visit_stats.last_scheduled, EXCLUDED.last_scheduled) SET last_scheduled = GREATEST(
origin_visit_stats.last_scheduled,
EXCLUDED.last_scheduled
)
) )
SELECT SELECT
* *
FROM FROM
selected_origins selected_origins
""" # noqa """
cur.execute(query, (visit_type, count))
cur.execute(query, (visit_type, count))
return [ListedOrigin(**d) for d in cur] return [ListedOrigin(**d) for d in cur]
else:
raise UnknownPolicy(f"Unknown scheduling policy {policy}")
task_create_keys = [ task_create_keys = [
"type", "type",
"arguments", "arguments",
"next_run", "next_run",
"policy", "policy",
"status", "status",
"retries_left", "retries_left",
▲ Show 20 Lines • Show All 567 Lines • Show Last 20 Lines
About · Developer information · Legal