Changeset View
Changeset View
Standalone View
Standalone View
swh/fuse/cache.py
Show First 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | async def set(self, swhid: SWHID, metadata: Any) -> None: | ||||
swhid_date = "" | swhid_date = "" | ||||
if swhid.object_type == REVISION: | if swhid.object_type == REVISION: | ||||
date = dateutil.parser.parse(metadata["date"]) | date = dateutil.parser.parse(metadata["date"]) | ||||
swhid_date = RevisionHistoryShardByDate.DATE_FMT.format( | swhid_date = RevisionHistoryShardByDate.DATE_FMT.format( | ||||
year=date.year, month=date.month, day=date.day | year=date.year, month=date.month, day=date.day | ||||
) | ) | ||||
await self.conn.execute( | await self.conn.execute( | ||||
"insert into metadata_cache values (?, ?, ?)", | "insert or ignore into metadata_cache values (?, ?, ?)", | ||||
(str(swhid), json.dumps(metadata), swhid_date), | (str(swhid), json.dumps(metadata), swhid_date), | ||||
) | ) | ||||
await self.conn.commit() | await self.conn.commit() | ||||
async def set_visits(self, url_encoded: str, visits: List[Dict[str, Any]]) -> None: | async def set_visits(self, url_encoded: str, visits: List[Dict[str, Any]]) -> None: | ||||
await self.conn.execute( | await self.conn.execute( | ||||
"insert or replace into visits_cache values (?, ?, ?)", | "insert or replace into visits_cache values (?, ?, ?)", | ||||
(url_encoded, json.dumps(visits), datetime.now()), | (url_encoded, json.dumps(visits), datetime.now()), | ||||
▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | HISTORY_REC_QUERY = """ | ||||
-- Do not keep the root node since it is not an ancestor | -- Do not keep the root node since it is not an ancestor | ||||
select * from dfs limit -1 offset 1 | select * from dfs limit -1 offset 1 | ||||
""" | """ | ||||
async def get(self, swhid: SWHID) -> Optional[List[SWHID]]: | async def get(self, swhid: SWHID) -> Optional[List[SWHID]]: | ||||
cursor = await self.conn.execute(self.HISTORY_REC_QUERY, (str(swhid),),) | cursor = await self.conn.execute(self.HISTORY_REC_QUERY, (str(swhid),),) | ||||
cache = await cursor.fetchall() | cache = await cursor.fetchall() | ||||
if not cache: | if not cache: | ||||
return None | return [] | ||||
history = [] | history = [] | ||||
for row in cache: | for row in cache: | ||||
parent = row[0] | parent = row[0] | ||||
try: | try: | ||||
history.append(parse_swhid(parent)) | history.append(parse_swhid(parent)) | ||||
except ValidationError: | except ValidationError: | ||||
logging.warning("Cannot parse object from history cache: %s", parent) | logging.warning("Cannot parse object from history cache: %s", parent) | ||||
return history | return history | ||||
Show All 17 Lines | ) -> List[Tuple[SWHID, str]]: | ||||
for row in cache: | for row in cache: | ||||
parent, date = row[0], row[1] | parent, date = row[0], row[1] | ||||
try: | try: | ||||
history.append((parse_swhid(parent), date)) | history.append((parse_swhid(parent), date)) | ||||
except ValidationError: | except ValidationError: | ||||
logging.warning("Cannot parse object from history cache: %s", parent) | logging.warning("Cannot parse object from history cache: %s", parent) | ||||
return history | return history | ||||
async def set(self, history: str) -> None: | async def set(self, edges: List[str]) -> None: | ||||
history = history.strip() | if edges: | ||||
if history: | |||||
edges = [edge.split(" ") for edge in history.split("\n")] | |||||
await self.conn.executemany( | await self.conn.executemany( | ||||
"insert or ignore into history_graph values (?, ?)", edges | "insert or ignore into history_graph values (?, ?)", edges | ||||
) | ) | ||||
await self.conn.commit() | await self.conn.commit() | ||||
class DirEntryCache: | class DirEntryCache: | ||||
""" The direntry cache map inode representing directories to the entries | """ The direntry cache map inode representing directories to the entries | ||||
▲ Show 20 Lines • Show All 74 Lines • Show Last 20 Lines |