Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/storage.py
Show First 20 Lines • Show All 1,087 Lines • ▼ Show 20 Lines | |||||
@db_transaction() | @db_transaction() | ||||
def origin_count( | def origin_count( | ||||
self, url_pattern, regexp=False, with_visit=False, db=None, cur=None | self, url_pattern, regexp=False, with_visit=False, db=None, cur=None | ||||
): | ): | ||||
return db.origin_count(url_pattern, regexp, with_visit, cur) | return db.origin_count(url_pattern, regexp, with_visit, cur) | ||||
@timed | @timed | ||||
@db_transaction() | @db_transaction() | ||||
def origin_add(self, origins: Iterable[Origin], db=None, cur=None) -> List[Dict]: | def origin_add( | ||||
origins = list(origins) | self, origins: Iterable[Origin], db=None, cur=None | ||||
for origin in origins: | ) -> Dict[str, int]: | ||||
self.origin_add_one(origin, db=db, cur=cur) | urls = [o.url for o in origins] | ||||
known_origins = set(url for (url,) in db.origin_get_by_url(urls, cur)) | |||||
return [o.to_dict() for o in origins] | # use lists here to keep origins sorted; some tests depend on this | ||||
to_add = [url for url in urls if url not in known_origins] | |||||
self.journal_writer.origin_add([Origin(url=url) for url in to_add]) | |||||
added = 0 | |||||
for url in to_add: | |||||
if db.origin_add(url, cur): | |||||
added += 1 | |||||
return {"origin:add": added} | |||||
@timed | @timed | ||||
@db_transaction() | @db_transaction() | ||||
def origin_add_one(self, origin: Origin, db=None, cur=None) -> str: | def origin_add_one(self, origin: Origin, db=None, cur=None) -> str: | ||||
origin_row = list(db.origin_get_by_url([origin.url], cur))[0] | stats = self.origin_add([origin]) | ||||
origin_url = dict(zip(db.origin_cols, origin_row))["url"] | if stats.get("origin:add", 0): | ||||
if origin_url: | |||||
return origin_url | |||||
self.journal_writer.origin_add([origin]) | |||||
url = db.origin_add(origin.url, cur) | |||||
send_metric("origin:add", count=1, method_name="origin_add_one") | send_metric("origin:add", count=1, method_name="origin_add_one") | ||||
return url | return origin.url | ||||
@db_transaction(statement_timeout=500) | @db_transaction(statement_timeout=500) | ||||
def stat_counters(self, db=None, cur=None): | def stat_counters(self, db=None, cur=None): | ||||
return {k: v for (k, v) in db.stat_counters()} | return {k: v for (k, v) in db.stat_counters()} | ||||
@db_transaction() | @db_transaction() | ||||
def refresh_stat_counters(self, db=None, cur=None): | def refresh_stat_counters(self, db=None, cur=None): | ||||
keys = [ | keys = [ | ||||
▲ Show 20 Lines • Show All 183 Lines • Show Last 20 Lines |