Page MenuHomeSoftware Heritage
Differential D714

mimetype: Migrate to indexed data as text
ClosedPublic
Actions

Authored by ardumont on Nov 27 2018, 1:23 PM.

Details

Reviewers
vlorentz
Group Reviewers
Reviewers
Commits
rDCIDX6cbee6588b87: mimetype: Migrate to indexed data as text
rDCIDXe2defde1b5eb: schema: Align schema version with latest upgrade script
Summary

Note: migration currently running in production

Related T818

Test Plan

tox

Diff Detail

Repository
rDCIDX Metadata indexer
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathSize
sql/
upgrades/
3 lines
swh/
indexer/
sql/
2 lines

Diff 2268

View Options

sql/upgrades/117.sql

Show First 20 Lines • Show All 277 Lines • ▼ Show 20 Lines def send_tool(self, tool):
tool['name'], tool['version'], tool['configuration']), tool['name'], tool['version'], tool['configuration']),
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'tool', 'swh_content_type': 'tool',
'swh_num': 1, 'swh_num': 1,
'swh_id': log_id 'swh_id': log_id
}) })
tools = list(self.storage.tool_add([tool])) tools = self.storage.tool_add([tool])
vlorentzUnsubmitted
Not Done
Inline Actions

Why this change? I don't think it will play well with a non-remote storage, because they return iterators.

vlorentz: Why this change? I don't think it will play well with a non-remote storage, because they return…
ardumontAuthorUnsubmitted
Done
Inline Actions

See D1321 which transformed the generator into a list to unify with other _add endpoints.
That simplifies the metrics extraction (counting length).

ardumont: See D1321 which transformed the generator into a list to unify with other `_add` endpoints.
tool_id = tools[0]['id'] tool_id = tools[0]['id']
self.log.debug( self.log.debug(
'Done creating tool with name %s version %s and configuration %s' % ( # noqa 'Done creating tool with name %s version %s and configuration %s' % ( # noqa
tool['name'], tool['version'], tool['configuration']), tool['name'], tool['version'], tool['configuration']),
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'tool', 'swh_content_type': 'tool',
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines def send_contents(self, content_list):
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d contents" % num_contents, self.log.debug("Sending %d contents" % num_contents,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'content', 'swh_content_type': 'content',
'swh_num': num_contents, 'swh_num': num_contents,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.content_add(content_list) result = self.storage.content_add(content_list)
self.counters['contents'] += num_contents self.counters['contents'] += result['content:add']
self.log.debug("Done sending %d contents" % num_contents, self.log.debug("Done sending %d contents" % num_contents,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'content', 'swh_content_type': 'content',
'swh_num': num_contents, 'swh_num': num_contents,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_directories(self, directory_list): def send_directories(self, directory_list):
"""Actually send properly formatted directories to the database. """Actually send properly formatted directories to the database.
""" """
num_directories = len(directory_list) num_directories = len(directory_list)
if num_directories > 0: if num_directories > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d directories" % num_directories, self.log.debug("Sending %d directories" % num_directories,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'directory', 'swh_content_type': 'directory',
'swh_num': num_directories, 'swh_num': num_directories,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.directory_add(directory_list) result = self.storage.directory_add(directory_list)
self.counters['directories'] += num_directories self.counters['directories'] += result['directory:add']
self.log.debug("Done sending %d directories" % num_directories, self.log.debug("Done sending %d directories" % num_directories,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'directory', 'swh_content_type': 'directory',
'swh_num': num_directories, 'swh_num_new': num_directories,
olasdUnsubmitted
Not Done
Inline Actions

I'm not sure this was intended?

olasd: I'm not sure this was intended?
ardumontAuthorUnsubmitted
Done
Inline Actions

no, it was not ;)

Your 'weird' feeling about the counter discrepancy was shared and i started something.
But then i stopped myself because it's out of scope (first migration) and did not want to go there.

That's a left over!
Thanks for pointing it out.

ardumont: no, it was not ;) Your 'weird' feeling about the counter discrepancy was shared and i started…
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_revisions(self, revision_list): def send_revisions(self, revision_list):
"""Actually send properly formatted revisions to the database. """Actually send properly formatted revisions to the database.
""" """
num_revisions = len(revision_list) num_revisions = len(revision_list)
if num_revisions > 0: if num_revisions > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d revisions" % num_revisions, self.log.debug("Sending %d revisions" % num_revisions,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'revision', 'swh_content_type': 'revision',
'swh_num': num_revisions, 'swh_num': num_revisions,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.revision_add(revision_list) result = self.storage.revision_add(revision_list)
self.counters['revisions'] += num_revisions self.counters['revisions'] += result['revision:add']
self.log.debug("Done sending %d revisions" % num_revisions, self.log.debug("Done sending %d revisions" % num_revisions,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'revision', 'swh_content_type': 'revision',
'swh_num': num_revisions, 'swh_num': num_revisions,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_releases(self, release_list): def send_releases(self, release_list):
"""Actually send properly formatted releases to the database. """Actually send properly formatted releases to the database.
""" """
num_releases = len(release_list) num_releases = len(release_list)
if num_releases > 0: if num_releases > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d releases" % num_releases, self.log.debug("Sending %d releases" % num_releases,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'release', 'swh_content_type': 'release',
'swh_num': num_releases, 'swh_num': num_releases,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.release_add(release_list) result = self.storage.release_add(release_list)
self.counters['releases'] += num_releases self.counters['releases'] += result['release:add']
self.log.debug("Done sending %d releases" % num_releases, self.log.debug("Done sending %d releases" % num_releases,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'release', 'swh_content_type': 'release',
'swh_num': num_releases, 'swh_num': num_releases,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_snapshot(self, snapshot): def send_snapshot(self, snapshot):
self.storage.snapshot_add(self.origin_id, self.visit, snapshot) self.storage.snapshot_add([snapshot])
olasdUnsubmitted
Done
Inline Actions

That should be snapshot['id']

olasd: That should be `snapshot['id']`
olasdUnsubmitted
Done
Inline Actions

also needs an explicit keyword argument

olasd: also needs an explicit keyword argument
self.storage.origin_visit_update(
self.origin_id, self.visit, snapshot=snapshot['id'])
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def filter_missing_contents(self, contents): def filter_missing_contents(self, contents):
"""Return only the contents missing from swh""" """Return only the contents missing from swh"""
max_content_size = self.config['content_size_limit'] max_content_size = self.config['content_size_limit']
contents_per_key = {} contents_per_key = {}
content_key = 'blake2s256' content_key = 'blake2s256'
▲ Show 20 Lines • Show All 514 Lines • Show Last 20 Lines
View Options

swh/indexer/sql/30-swh-schema.sql

Show First 20 Lines • Show All 277 Lines • ▼ Show 20 Lines def send_tool(self, tool):
tool['name'], tool['version'], tool['configuration']), tool['name'], tool['version'], tool['configuration']),
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'tool', 'swh_content_type': 'tool',
'swh_num': 1, 'swh_num': 1,
'swh_id': log_id 'swh_id': log_id
}) })
tools = list(self.storage.tool_add([tool])) tools = self.storage.tool_add([tool])
vlorentzUnsubmitted
Not Done
Inline Actions

Why this change? I don't think it will play well with a non-remote storage, because they return iterators.

vlorentz: Why this change? I don't think it will play well with a non-remote storage, because they return…
ardumontAuthorUnsubmitted
Done
Inline Actions

See D1321 which transformed the generator into a list to unify with other _add endpoints.
That simplifies the metrics extraction (counting length).

ardumont: See D1321 which transformed the generator into a list to unify with other `_add` endpoints.
tool_id = tools[0]['id'] tool_id = tools[0]['id']
self.log.debug( self.log.debug(
'Done creating tool with name %s version %s and configuration %s' % ( # noqa 'Done creating tool with name %s version %s and configuration %s' % ( # noqa
tool['name'], tool['version'], tool['configuration']), tool['name'], tool['version'], tool['configuration']),
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'tool', 'swh_content_type': 'tool',
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines def send_contents(self, content_list):
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d contents" % num_contents, self.log.debug("Sending %d contents" % num_contents,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'content', 'swh_content_type': 'content',
'swh_num': num_contents, 'swh_num': num_contents,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.content_add(content_list) result = self.storage.content_add(content_list)
self.counters['contents'] += num_contents self.counters['contents'] += result['content:add']
self.log.debug("Done sending %d contents" % num_contents, self.log.debug("Done sending %d contents" % num_contents,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'content', 'swh_content_type': 'content',
'swh_num': num_contents, 'swh_num': num_contents,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_directories(self, directory_list): def send_directories(self, directory_list):
"""Actually send properly formatted directories to the database. """Actually send properly formatted directories to the database.
""" """
num_directories = len(directory_list) num_directories = len(directory_list)
if num_directories > 0: if num_directories > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d directories" % num_directories, self.log.debug("Sending %d directories" % num_directories,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'directory', 'swh_content_type': 'directory',
'swh_num': num_directories, 'swh_num': num_directories,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.directory_add(directory_list) result = self.storage.directory_add(directory_list)
self.counters['directories'] += num_directories self.counters['directories'] += result['directory:add']
self.log.debug("Done sending %d directories" % num_directories, self.log.debug("Done sending %d directories" % num_directories,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'directory', 'swh_content_type': 'directory',
'swh_num': num_directories, 'swh_num_new': num_directories,
olasdUnsubmitted
Not Done
Inline Actions

I'm not sure this was intended?

olasd: I'm not sure this was intended?
ardumontAuthorUnsubmitted
Done
Inline Actions

no, it was not ;)

Your 'weird' feeling about the counter discrepancy was shared and i started something.
But then i stopped myself because it's out of scope (first migration) and did not want to go there.

That's a left over!
Thanks for pointing it out.

ardumont: no, it was not ;) Your 'weird' feeling about the counter discrepancy was shared and i started…
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_revisions(self, revision_list): def send_revisions(self, revision_list):
"""Actually send properly formatted revisions to the database. """Actually send properly formatted revisions to the database.
""" """
num_revisions = len(revision_list) num_revisions = len(revision_list)
if num_revisions > 0: if num_revisions > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d revisions" % num_revisions, self.log.debug("Sending %d revisions" % num_revisions,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'revision', 'swh_content_type': 'revision',
'swh_num': num_revisions, 'swh_num': num_revisions,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.revision_add(revision_list) result = self.storage.revision_add(revision_list)
self.counters['revisions'] += num_revisions self.counters['revisions'] += result['revision:add']
self.log.debug("Done sending %d revisions" % num_revisions, self.log.debug("Done sending %d revisions" % num_revisions,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'revision', 'swh_content_type': 'revision',
'swh_num': num_revisions, 'swh_num': num_revisions,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_releases(self, release_list): def send_releases(self, release_list):
"""Actually send properly formatted releases to the database. """Actually send properly formatted releases to the database.
""" """
num_releases = len(release_list) num_releases = len(release_list)
if num_releases > 0: if num_releases > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d releases" % num_releases, self.log.debug("Sending %d releases" % num_releases,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'release', 'swh_content_type': 'release',
'swh_num': num_releases, 'swh_num': num_releases,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.release_add(release_list) result = self.storage.release_add(release_list)
self.counters['releases'] += num_releases self.counters['releases'] += result['release:add']
self.log.debug("Done sending %d releases" % num_releases, self.log.debug("Done sending %d releases" % num_releases,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'release', 'swh_content_type': 'release',
'swh_num': num_releases, 'swh_num': num_releases,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_snapshot(self, snapshot): def send_snapshot(self, snapshot):
self.storage.snapshot_add(self.origin_id, self.visit, snapshot) self.storage.snapshot_add([snapshot])
olasdUnsubmitted
Done
Inline Actions

That should be snapshot['id']

olasd: That should be `snapshot['id']`
olasdUnsubmitted
Done
Inline Actions

also needs an explicit keyword argument

olasd: also needs an explicit keyword argument
self.storage.origin_visit_update(
self.origin_id, self.visit, snapshot=snapshot['id'])
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def filter_missing_contents(self, contents): def filter_missing_contents(self, contents):
"""Return only the contents missing from swh""" """Return only the contents missing from swh"""
max_content_size = self.config['content_size_limit'] max_content_size = self.config['content_size_limit']
contents_per_key = {} contents_per_key = {}
content_key = 'blake2s256' content_key = 'blake2s256'
▲ Show 20 Lines • Show All 514 Lines • Show Last 20 Lines
About · Developer information · Legal