				-- SWH DB schema upgrade
				-- from_version: 141
				-- to_version: 142
				-- description: Remove origin.type

				insert into dbversion(version, release, description)
				values(142, now(), 'Work In Progress');

				alter table origin drop column type;

View Options

swh/storage/db.py

Show First 20 Lines • Show All 598 Lines • ▼ Show 20 Lines	def object_find_by_sha1_git(self, ids, cur=None):
((id,) for id in ids)		((id,) for id in ids)
)		)

def stat_counters(self, cur=None):		def stat_counters(self, cur=None):
cur = self._cursor(cur)		cur = self._cursor(cur)
cur.execute('SELECT * FROM swh_stat_counters()')		cur.execute('SELECT * FROM swh_stat_counters()')
yield from cur		yield from cur

def origin_add(self, type, url, cur=None):		def origin_add(self, url, cur=None):
"""Insert a new origin and return the new identifier."""		"""Insert a new origin and return the new identifier."""
insert = """INSERT INTO origin (type, url) values (%s, %s)		insert = """INSERT INTO origin (url) values (%s)
RETURNING id"""		RETURNING id"""

cur.execute(insert, (type, url))		cur.execute(insert, (url,))
return cur.fetchone()[0]		return cur.fetchone()[0]

origin_cols = ['id', 'type', 'url']		origin_cols = ['id', 'url']

def origin_get_by_url(self, origins, cur=None):		def origin_get_by_url(self, origins, cur=None):
"""Retrieve origin `(id, type, url)` from urls if found."""		"""Retrieve origin `(id, type, url)` from urls if found."""
cur = self._cursor(cur)		cur = self._cursor(cur)

query = """SELECT %s FROM (VALUES %%s) as t(url)		query = """SELECT %s FROM (VALUES %%s) as t(url)
LEFT JOIN origin ON t.url = origin.url		LEFT JOIN origin ON t.url = origin.url
""" % ','.join('origin.' + col for col in self.origin_cols)		""" % ','.join('origin.' + col for col in self.origin_cols)
▲ Show 20 Lines • Show All 243 Lines • Show Last 20 Lines

View Options

swh/storage/in_memory.py

Show First 20 Lines • Show All 1,008 Lines • ▼ Show 20 Lines	class Storage:

def _convert_origin(self, t):		def _convert_origin(self, t):
if t is None:		if t is None:
return None		return None
(origin_id, origin) = t		(origin_id, origin) = t
origin = origin.to_dict()		origin = origin.to_dict()
if ENABLE_ORIGIN_IDS:		if ENABLE_ORIGIN_IDS:
origin['id'] = origin_id		origin['id'] = origin_id

		if 'type' in origin:
		del origin['type']

return origin		return origin

def origin_get(self, origins):		def origin_get(self, origins):
"""Return origins, either all identified by their ids or all		"""Return origins, either all identified by their ids or all
identified by tuples (type, url).		identified by urls.

If the url is given and the type is omitted, one of the origins with
that url is returned.

Args:		Args:
origin: a list of dictionaries representing the individual		origin: a list of dictionaries representing the individual
origins to find.		origins to find.
These dicts have either the key url (and optionally type):		These dicts have either the key url (and optionally type):

- type (FIXME: enum TBD): the origin type ('git', 'wget', ...)
- url (bytes): the url the origin points to		- url (bytes): the url the origin points to

or the id:		or the id:

- id (int): the origin's identifier		- id (int): the origin's identifier

Returns:		Returns:
dict: the origin dictionary with the keys:		dict: the origin dictionary with the keys:

- id: origin's id		- id: origin's id
- type: origin's type
- url: origin's url		- url: origin's url

Raises:		Raises:
ValueError: if the keys does not match (url and type) nor id.		ValueError: if the keys does not match (url and type) nor id.

"""		"""
if isinstance(origins, dict):		if isinstance(origins, dict):
# Old API		# Old API
▲ Show 20 Lines • Show All 113 Lines • ▼ Show 20 Lines

def origin_add(self, origins):		def origin_add(self, origins):
"""Add origins to the storage		"""Add origins to the storage

Args:		Args:
origins: list of dictionaries representing the individual origins,		origins: list of dictionaries representing the individual origins,
with the following keys:		with the following keys:

- type: the origin type ('git', 'svn', 'deb', ...)
- url (bytes): the url the origin points to		- url (bytes): the url the origin points to

Returns:		Returns:
list: given origins as dict updated with their id		list: given origins as dict updated with their id

"""		"""
origins = copy.deepcopy(origins)		origins = copy.deepcopy(origins)
for origin in origins:		for origin in origins:
if ENABLE_ORIGIN_IDS:		if ENABLE_ORIGIN_IDS:
origin['id'] = self.origin_add_one(origin)		origin['id'] = self.origin_add_one(origin)
else:		else:
self.origin_add_one(origin)		self.origin_add_one(origin)
return origins		return origins

def origin_add_one(self, origin):		def origin_add_one(self, origin):
"""Add origin to the storage		"""Add origin to the storage

Args:		Args:
origin: dictionary representing the individual origin to add. This		origin: dictionary representing the individual origin to add. This
dict has the following keys:		dict has the following keys:

- type (FIXME: enum TBD): the origin type ('git', 'wget', ...)
- url (bytes): the url the origin points to		- url (bytes): the url the origin points to

Returns:		Returns:
the id of the added origin, or of the identical one that already		the id of the added origin, or of the identical one that already
exists.		exists.

"""		"""
origin = Origin.from_dict(origin)		origin = Origin.from_dict(origin)

if origin.url in self._origins:		if origin.url in self._origins:
if ENABLE_ORIGIN_IDS:		if ENABLE_ORIGIN_IDS:
(origin_id, _) = self._origins[origin.url]		(origin_id, _) = self._origins[origin.url]
else:		else:
if self.journal_writer:		if self.journal_writer:
self.journal_writer.write_addition('origin', origin)		self.journal_writer.write_addition('origin', origin)
if ENABLE_ORIGIN_IDS:		if ENABLE_ORIGIN_IDS:
# origin ids are in the range [1, +inf[		# origin ids are in the range [1, +inf[
▲ Show 20 Lines • Show All 516 Lines • Show Last 20 Lines