diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py --- a/swh/storage/cassandra/storage.py +++ b/swh/storage/cassandra/storage.py @@ -260,39 +260,6 @@ [{"sha1_git": c for c in contents}], key_hash="sha1_git" ) - def content_metadata_add( - self, - id: str, - context: Dict[str, Union[str, bytes, int]], - discovery_date: datetime.datetime, - authority: Dict[str, Any], - fetcher: Dict[str, Any], - format: str, - metadata: bytes, - ) -> None: - self._object_metadata_add( - "content", - id, - discovery_date, - authority, - fetcher, - format, - metadata, - context, - ) - - def content_metadata_get( - self, - id: str, - authority: Dict[str, str], - after: Optional[datetime.datetime] = None, - page_token: Optional[bytes] = None, - limit: int = 1000, - ) -> Dict[str, Any]: - return self._object_metadata_get( - "content", id, authority, after, page_token, limit, - ) - def content_get_random(self): return self._cql_runner.content_get_random().sha1_git @@ -1051,6 +1018,39 @@ def refresh_stat_counters(self): pass + def content_metadata_add( + self, + id: str, + context: Dict[str, Union[str, bytes, int]], + discovery_date: datetime.datetime, + authority: Dict[str, Any], + fetcher: Dict[str, Any], + format: str, + metadata: bytes, + ) -> None: + self._object_metadata_add( + "content", + id, + discovery_date, + authority, + fetcher, + format, + metadata, + context, + ) + + def content_metadata_get( + self, + id: str, + authority: Dict[str, str], + after: Optional[datetime.datetime] = None, + page_token: Optional[bytes] = None, + limit: int = 1000, + ) -> Dict[str, Any]: + return self._object_metadata_get( + "content", id, authority, after, page_token, limit, + ) + def origin_metadata_add( self, origin_url: str, diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -1037,6 +1037,18 @@ context, ) + def content_metadata_get( + self, + id: str, + authority: Dict[str, str], + after: Optional[datetime.datetime] = None, + page_token: Optional[bytes] = None, + limit: int = 1000, + ) -> Dict[str, Any]: + return self._object_metadata_get( + "content", id, authority, after, page_token, limit + ) + def origin_metadata_add( self, origin_url: str, @@ -1064,6 +1076,26 @@ context, ) + def origin_metadata_get( + self, + origin_url: str, + authority: Dict[str, str], + after: Optional[datetime.datetime] = None, + page_token: Optional[bytes] = None, + limit: int = 1000, + ) -> Dict[str, Any]: + if not isinstance(origin_url, str): + raise TypeError("origin_url must be str, not %r" % (origin_url,)) + + res = self._object_metadata_get( + "origin", origin_url, authority, after, page_token, limit + ) + res["results"] = copy.deepcopy(res["results"]) + for result in res["results"]: + result["origin_url"] = result.pop("id") + + return res + def _object_metadata_add( self, object_type: str, @@ -1112,38 +1144,6 @@ else: object_metadata_list.add(object_metadata) - def content_metadata_get( - self, - id: str, - authority: Dict[str, str], - after: Optional[datetime.datetime] = None, - page_token: Optional[bytes] = None, - limit: int = 1000, - ) -> Dict[str, Any]: - return self._object_metadata_get( - "content", id, authority, after, page_token, limit - ) - - def origin_metadata_get( - self, - origin_url: str, - authority: Dict[str, str], - after: Optional[datetime.datetime] = None, - page_token: Optional[bytes] = None, - limit: int = 1000, - ) -> Dict[str, Any]: - if not isinstance(origin_url, str): - raise TypeError("origin_url must be str, not %r" % (origin_url,)) - - res = self._object_metadata_get( - "origin", origin_url, authority, after, page_token, limit - ) - res["results"] = copy.deepcopy(res["results"]) - for result in res["results"]: - result["origin_url"] = result.pop("id") - - return res - def _object_metadata_get( self, object_type: str, diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -364,49 +364,6 @@ ) return [dict(zip(db.content_find_cols, content)) for content in contents] - @timed - @db_transaction() - def content_metadata_add( - self, - id: str, - context: Dict[str, Union[str, bytes, int]], - discovery_date: datetime.datetime, - authority: Dict[str, Any], - fetcher: Dict[str, Any], - format: str, - metadata: bytes, - db=None, - cur=None, - ) -> None: - self._object_metadata_add( - "content", - id, - context, - discovery_date, - authority, - fetcher, - format, - metadata, - db, - cur, - ) - - @timed - @db_transaction() - def content_metadata_get( - self, - id: str, - authority: Dict[str, str], - after: Optional[datetime.datetime] = None, - page_token: Optional[bytes] = None, - limit: int = 1000, - db=None, - cur=None, - ) -> Dict[str, Any]: - return self._object_metadata_get( - "content", id, authority, after, page_token, limit, db, cur - ) - @timed @db_transaction() def content_get_random(self, db=None, cur=None): @@ -1196,6 +1153,49 @@ for key in keys: cur.execute("select * from swh_update_counter(%s)", (key,)) + @timed + @db_transaction() + def content_metadata_add( + self, + id: str, + context: Dict[str, Union[str, bytes, int]], + discovery_date: datetime.datetime, + authority: Dict[str, Any], + fetcher: Dict[str, Any], + format: str, + metadata: bytes, + db=None, + cur=None, + ) -> None: + self._object_metadata_add( + "content", + id, + context, + discovery_date, + authority, + fetcher, + format, + metadata, + db, + cur, + ) + + @timed + @db_transaction() + def content_metadata_get( + self, + id: str, + authority: Dict[str, str], + after: Optional[datetime.datetime] = None, + page_token: Optional[bytes] = None, + limit: int = 1000, + db=None, + cur=None, + ) -> Dict[str, Any]: + return self._object_metadata_get( + "content", id, authority, after, page_token, limit, db, cur + ) + @timed @db_transaction() def origin_metadata_add( @@ -1224,6 +1224,28 @@ cur, ) + @timed + @db_transaction(statement_timeout=500) + def origin_metadata_get( + self, + origin_url: str, + authority: Dict[str, str], + after: Optional[datetime.datetime] = None, + page_token: Optional[bytes] = None, + limit: int = 1000, + db=None, + cur=None, + ) -> Dict[str, Any]: + result = self._object_metadata_get( + "origin", origin_url, authority, after, page_token, limit, db, cur + ) + + for res in result["results"]: + res.pop("id") + res["origin_url"] = origin_url + + return result + def _object_metadata_add( self, object_type: str, @@ -1264,28 +1286,6 @@ method_name=f"{object_type}_metadata_add", ) - @timed - @db_transaction(statement_timeout=500) - def origin_metadata_get( - self, - origin_url: str, - authority: Dict[str, str], - after: Optional[datetime.datetime] = None, - page_token: Optional[bytes] = None, - limit: int = 1000, - db=None, - cur=None, - ) -> Dict[str, Any]: - result = self._object_metadata_get( - "origin", origin_url, authority, after, page_token, limit, db, cur - ) - - for res in result["results"]: - res.pop("id") - res["origin_url"] = origin_url - - return result - def _object_metadata_get( self, object_type: str,