Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7124266
D4890.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
1 KB
Subscribers
None
D4890.diff
View Options
diff --git a/swh/scheduler/journal_client.py b/swh/scheduler/journal_client.py
--- a/swh/scheduler/journal_client.py
+++ b/swh/scheduler/journal_client.py
@@ -48,10 +48,18 @@
}, f"Got unexpected {', '.join(set(messages) - set([msg_type]))} message types"
assert msg_type in messages, f"Expected {msg_type} messages"
- origin_visit_stats: Dict[Tuple[str, str], Dict] = {}
- for msg_dict in messages[msg_type]:
- if msg_dict["status"] in ("created", "ongoing"):
- continue
+ interesting_messages = [
+ msg for msg in messages[msg_type] if msg["status"] not in ("created", "ongoing")
+ ]
+
+ origin_visit_stats: Dict[Tuple[str, str], Dict] = {
+ (visit_stats.url, visit_stats.visit_type): attr.asdict(visit_stats)
+ for visit_stats in scheduler.origin_visit_stats_get(
+ list(set((vs["origin"], vs["type"]) for vs in interesting_messages))
+ )
+ }
+
+ for msg_dict in interesting_messages:
origin = msg_dict["origin"]
visit_type = msg_dict["type"]
empty_object = {
@@ -65,11 +73,7 @@
}
pk = origin, visit_type
if pk not in origin_visit_stats:
- visit_stats = scheduler.origin_visit_stats_get([pk])
- origin_visit_stats[pk] = (
- attr.asdict(visit_stats[0]) if visit_stats else empty_object
- )
-
+ origin_visit_stats[pk] = empty_object
visit_stats_d = origin_visit_stats[pk]
if msg_dict["status"] == "not_found":
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Dec 21 2024, 4:05 AM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3221976
Attached To
D4890: journal_client: Read visit_stats entries by batch out of the loop
Event Timeline
Log In to Comment