Differential D5978 Diff 21544 swh/scheduler/journal_client.py

Changeset View

Standalone View

View Options

swh/scheduler/journal_client.py

Show First 20 Lines • Show All 171 Lines • ▼ Show 20 Lines for msg_dict in interesting_messages:

if pk not in origin_visit_stats: if pk not in origin_visit_stats:

origin_visit_stats[pk] = { origin_visit_stats[pk] = {

**empty_object, **empty_object,

"url": origin, "url": origin,

"visit_type": visit_type, "visit_type": visit_type,

} }

visit_stats_d = origin_visit_stats[pk] visit_stats_d = origin_visit_stats[pk]

anlambertUnsubmitted

Done

recent

anlambert: recent

ardumontAuthorUnsubmitted

Done

lol i rebased a diff from david but i kept the typo, nice or what!

ardumont: lol i rebased a diff from david but i kept the typo, nice or what!

# get the name of the most recent event we got

date_keys = [

f"last_{k}" for k in ("eventful", "uneventful", "failed", "notfound")

]

event_dates = dict(

(v, k) for k, v in visit_stats_d.items() if k in date_keys and v is not None

)

if not event_dates:

maxdate = None

last_event = None

else:

maxdate = max_date(*event_dates.keys())

last_event = event_dates[maxdate]

increment_successive_visits = False

vlorentzUnsubmitted

Done

last_visit_status, eventful = get_last_status(msg_dict, visit_stats_d)

- increment_successive_visits = False

# Update the position offset according to the visit status,

# if we had already visited this origin before.

if visit_stats_d.get("last_visit"):

# Update the next position offset according to the existing value and the

# eventfulness of the visit.

increment = -2 if eventful else 1

visit_stats_d["next_position_offset"] = max(

0, visit_stats_d["next_position_offset"] + increment

)

# increment the counter when last_visit_status is the same

increment_successive_visits = (

last_visit_status == visit_stats_d["last_visit_status"]

)

+ else:

+ increment_successive_visits = False

# Record current visit date as highest known date (we've rejected out of order

I find this more readable

vlorentz: I find this more readable

if msg_dict["status"] == "not_found": if msg_dict["status"] == "not_found":

visit_stats_d["last_notfound"] = max_date( visit_stats_d["last_notfound"] = max_date(

msg_dict["date"], visit_stats_d.get("last_notfound") msg_dict["date"], visit_stats_d.get("last_notfound")

) )

update_next_position_offset(visit_stats_d, 1) # visit less often update_next_position_offset(visit_stats_d, 1) # visit less often

if last_event == "last_notfound":

increment_successive_visits = True

elif msg_dict["status"] == "failed" or msg_dict["snapshot"] is None: elif msg_dict["status"] == "failed" or msg_dict["snapshot"] is None:

visit_stats_d["last_failed"] = max_date( visit_stats_d["last_failed"] = max_date(

msg_dict["date"], visit_stats_d.get("last_failed") msg_dict["date"], visit_stats_d.get("last_failed")

) )

update_next_position_offset(visit_stats_d, 1) # visit less often update_next_position_offset(visit_stats_d, 1) # visit less often

if last_event == "last_failed":

increment_successive_visits = True

else: # visit with snapshot, something happened else: # visit with snapshot, something happened

if visit_stats_d["last_snapshot"] is None: if visit_stats_d["last_snapshot"] is None:

# first time visit with snapshot, we keep relevant information # first time visit with snapshot, we keep relevant information

visit_stats_d["last_eventful"] = msg_dict["date"] visit_stats_d["last_eventful"] = msg_dict["date"]

visit_stats_d["last_snapshot"] = msg_dict["snapshot"] visit_stats_d["last_snapshot"] = msg_dict["snapshot"]

else: else:

# last_snapshot is set, so an eventful visit should have previously been # last_snapshot is set, so an eventful visit should have previously been

# recorded # recorded

Show All 10 Lines for msg_dict in interesting_messages:

): ):

# out of order message so ignored # out of order message so ignored

continue continue

# new eventful visit (new snapshot) # new eventful visit (new snapshot)

visit_stats_d["last_eventful"] = current_status_date visit_stats_d["last_eventful"] = current_status_date

visit_stats_d["last_snapshot"] = msg_dict["snapshot"] visit_stats_d["last_snapshot"] = msg_dict["snapshot"]

# Visit this origin more often in the future # Visit this origin more often in the future

update_next_position_offset(visit_stats_d, -2) update_next_position_offset(visit_stats_d, -2)

if last_event == "last_eventful":

increment_successive_visits = True

else: else:

# same snapshot as before # same snapshot as before

if ( if (

latest_recorded_visit_date latest_recorded_visit_date

and current_status_date < latest_recorded_visit_date and current_status_date < latest_recorded_visit_date

): ):

# we receive an old message which is an earlier "eventful" event # we receive an old message which is an earlier "eventful" event

# than what we had, we consider the last_eventful event as # than what we had, we consider the last_eventful event as

# actually an uneventful event. # actually an uneventful event.

# The last uneventful visit remains the most recent: # The last uneventful visit remains the most recent:

# max, previously computed # max, previously computed

visit_stats_d["last_uneventful"] = latest_recorded_visit_date visit_stats_d["last_uneventful"] = latest_recorded_visit_date

# The eventful visit remains the oldest one: min # The eventful visit remains the oldest one: min

visit_stats_d["last_eventful"] = min( visit_stats_d["last_eventful"] = min(

visit_stats_d["last_eventful"], current_status_date visit_stats_d["last_eventful"], current_status_date

) )

# Visit this origin less often in the future # Visit this origin less often in the future

update_next_position_offset(visit_stats_d, 1) update_next_position_offset(visit_stats_d, 1)

# there is no way we can do anything but reset the

# successive_visits here...

elif ( elif (

latest_recorded_visit_date latest_recorded_visit_date

and current_status_date == latest_recorded_visit_date and current_status_date == latest_recorded_visit_date

): ):

# A duplicated message must be ignored to avoid # A duplicated message must be ignored to avoid

# populating the last_uneventful message # populating the last_uneventful message

continue continue

else: else:

# uneventful event # uneventful event

visit_stats_d["last_uneventful"] = current_status_date visit_stats_d["last_uneventful"] = current_status_date

# Visit this origin less often in the future # Visit this origin less often in the future

update_next_position_offset(visit_stats_d, 1) update_next_position_offset(visit_stats_d, 1)

if last_event == "last_uneventful":

increment_successive_visits = True

# Update the next visit queue position (which will be used solely for origin # Update the next visit queue position (which will be used solely for origin

# without any last_update, cf. the dedicated scheduling policy # without any last_update, cf. the dedicated scheduling policy

# "origins_without_last_update") # "origins_without_last_update")

visit_stats_d["next_visit_queue_position"] = next_visit_queue_position( visit_stats_d["next_visit_queue_position"] = next_visit_queue_position(

queue_position_per_visit_type, visit_stats_d queue_position_per_visit_type, visit_stats_d

) )

if increment_successive_visits:

visit_stats_d["successive_visits"] += 1

else:

vlorentzUnsubmitted

Done

if increment_successive_visits == False resets the counter, it's probably not the best variable name.

What about same_visit_status?

vlorentz: if `increment_successive_visits == False` resets the counter, it's probably not the best…

visit_stats_d["successive_visits"] = 1

scheduler.origin_visit_stats_upsert( scheduler.origin_visit_stats_upsert(

OriginVisitStats(**ovs) for ovs in origin_visit_stats.values() OriginVisitStats(**ovs) for ovs in origin_visit_stats.values()

) )