Changeset View
Changeset View
Standalone View
Standalone View
swh/provenance/postgresql/archive.py
| Show All 17 Lines | |||||||||||
| class ArchivePostgreSQL: | class ArchivePostgreSQL: | ||||||||||
| def __init__(self, conn: psycopg2.extensions.connection) -> None: | def __init__(self, conn: psycopg2.extensions.connection) -> None: | ||||||||||
| self.storage = get_storage( | self.storage = get_storage( | ||||||||||
| "postgresql", db=conn.dsn, objstorage={"cls": "memory"} | "postgresql", db=conn.dsn, objstorage={"cls": "memory"} | ||||||||||
| ) | ) | ||||||||||
| self.conn = conn | self.conn = conn | ||||||||||
| def directory_ls(self, id: Sha1Git, minsize: int = 0) -> Iterable[Dict[str, Any]]: | def directory_ls(self, id: Sha1Git, minsize: int = 0) -> Iterable[Dict[str, Any]]: | ||||||||||
| entries = self._directory_ls(id, minsize=minsize) | yield from self._directory_ls(id, minsize=minsize) | ||||||||||
| yield from entries | |||||||||||
| @lru_cache(maxsize=100000) | @lru_cache(maxsize=100000) | ||||||||||
| @statsd.timed(metric=ARCHIVE_DURATION_METRIC, tags={"method": "directory_ls"}) | @statsd.timed(metric=ARCHIVE_DURATION_METRIC, tags={"method": "directory_ls"}) | ||||||||||
| def _directory_ls(self, id: Sha1Git, minsize: int = 0) -> List[Dict[str, Any]]: | def _directory_ls(self, id: Sha1Git, minsize: int = 0) -> List[Dict[str, Any]]: | ||||||||||
| with self.conn.cursor() as cursor: | with self.conn.cursor() as cursor: | ||||||||||
| if minsize > 0: | if minsize > 0: | ||||||||||
| cursor.execute( | cursor.execute( | ||||||||||
| """ | """ | ||||||||||
| ▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | def _directory_ls(self, id: Sha1Git, minsize: int = 0) -> List[Dict[str, Any]]: | ||||||||||
| LEFT JOIN directory_entry_file e ON ls_f.entry_id=e.id) | LEFT JOIN directory_entry_file e ON ls_f.entry_id=e.id) | ||||||||||
| """, | """, | ||||||||||
| (id,), | (id,), | ||||||||||
| ) | ) | ||||||||||
| return [ | return [ | ||||||||||
| {"type": row[0], "target": row[1], "name": row[2]} for row in cursor | {"type": row[0], "target": row[1], "name": row[2]} for row in cursor | ||||||||||
| ] | ] | ||||||||||
| def revision_get_parents(self, id: Sha1Git) -> Iterable[Sha1Git]: | |||||||||||
| yield from self._revision_get_parents(id) | |||||||||||
ardumontUnsubmitted Done Inline Actions
ardumont: | |||||||||||
| @lru_cache(maxsize=100000) | |||||||||||
Done Inline ActionsI have no idea what a sensible number would be here ;) ardumont: I have no idea what a sensible number would be here ;) | |||||||||||
Done Inline ActionsMe neither, but this one has been working good so far... Maybe there is a way to send stats about hits/misses to StatsD to have better visualization and do some proper experimentation? Thanks for the review! aeviso: Me neither, but this one has been working good so far... Maybe there is a way to send stats… | |||||||||||
| @statsd.timed( | @statsd.timed( | ||||||||||
| metric=ARCHIVE_DURATION_METRIC, tags={"method": "revision_get_parents"} | metric=ARCHIVE_DURATION_METRIC, tags={"method": "revision_get_parents"} | ||||||||||
| ) | ) | ||||||||||
| def revision_get_parents(self, id: Sha1Git) -> Iterable[Sha1Git]: | def _revision_get_parents(self, id: Sha1Git) -> List[Sha1Git]: | ||||||||||
| with self.conn.cursor() as cursor: | with self.conn.cursor() as cursor: | ||||||||||
| cursor.execute( | cursor.execute( | ||||||||||
| """ | """ | ||||||||||
| SELECT RH.parent_id::bytea | SELECT RH.parent_id::bytea | ||||||||||
| FROM revision_history AS RH | FROM revision_history AS RH | ||||||||||
| WHERE RH.id=%s | WHERE RH.id=%s | ||||||||||
| ORDER BY RH.parent_rank | ORDER BY RH.parent_rank | ||||||||||
| """, | """, | ||||||||||
| (id,), | (id,), | ||||||||||
| ) | ) | ||||||||||
| yield from (row[0] for row in cursor) | return [row[0] for row in cursor] | ||||||||||
| @statsd.timed(metric=ARCHIVE_DURATION_METRIC, tags={"method": "snapshot_get_heads"}) | @statsd.timed(metric=ARCHIVE_DURATION_METRIC, tags={"method": "snapshot_get_heads"}) | ||||||||||
| def snapshot_get_heads(self, id: Sha1Git) -> Iterable[Sha1Git]: | def snapshot_get_heads(self, id: Sha1Git) -> Iterable[Sha1Git]: | ||||||||||
| with self.conn.cursor() as cursor: | with self.conn.cursor() as cursor: | ||||||||||
| cursor.execute( | cursor.execute( | ||||||||||
| """ | """ | ||||||||||
| WITH | WITH | ||||||||||
| snaps AS (SELECT object_id FROM snapshot WHERE snapshot.id=%s), | snaps AS (SELECT object_id FROM snapshot WHERE snapshot.id=%s), | ||||||||||
| Show All 28 Lines | |||||||||||