Page MenuHomeSoftware Heritage
Differential D8465

Move uvicorn and starlette to the main requirement file
ClosedPublic
Actions

Authored by jayeshv on Sep 13 2022, 3:14 PM.

Details

Reviewers
ardumont
Group Reviewers
Reviewers
Maniphest Tasks
T4532: GraphQL: staging - Deploy version v0.0.4
Commits
rDGQLf2f222267ff4: Move uvicorn and starlette to the main requirement file
Summary

uvicorn is a dependency for UvicornWorker which is used with gunicorn.
Moving uvicorn and starlette to the requirement file

Related to T4532

Diff Detail

Repository
rDGQL GraphQL API
Branch
asgi-bug
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 31502
Build 49280: Phabricator diff pipeline on jenkinsJenkins console · Jenkins
Build 49279: arc lint + arc unit

Event Timeline

jayeshv created this revision.Sep 13 2022, 3:14 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptSep 13 2022, 3:14 PM
ardumont published this revision for review.Sep 13 2022, 3:15 PM
ardumont edited the summary of this revision. (Show Details)
ardumont added a task: T4532: GraphQL: staging - Deploy version v0.0.4.
ardumont added a subscriber: ardumont.

\o/

ardumont accepted this revision.Sep 13 2022, 3:16 PM
This revision is now accepted and ready to land.Sep 13 2022, 3:16 PM
This revision was landed with ongoing or failed builds.Sep 13 2022, 3:18 PM
Closed by commit rDGQLf2f222267ff4: Move uvicorn and starlette to the main requirement file (authored by jayeshv). · Explain Why
This revision was automatically updated to reflect the committed changes.
jayeshv added a commit: rDGQLf2f222267ff4: Move uvicorn and starlette to the main requirement file.
swh-public-ci added a comment.Sep 13 2022, 3:19 PM

Build is green

Patch application report for D8465 (id=30495)

Rebasing onto 7e4eacb71f...

Current branch diff-target is up to date.
Changes applied before test
commit f2f222267ff493740efcc1af0edefeed06f24fc3
Author: Jayesh Velayudhan <jayesh@softwareheritage.org>
Date:   Tue Sep 13 15:07:49 2022 +0200

    Move uvicorn and starlette to the main requirement file
    
    uvicorn is a dependency for UvicornWorker which is used with gunicorn.
    Moving uvicorn and starlette to the requirement file

See https://jenkins.softwareheritage.org/job/DGQL/job/tests-on-diff/174/ for more details.

Harbormaster completed remote builds in B31502: Diff 30495.Sep 13 2022, 3:19 PM
ardumont mentioned this in T4532: GraphQL: staging - Deploy version v0.0.4.Sep 13 2022, 3:35 PM

Revision Contents
Changeset List

Diff 30495

View Options

requirements-dev.txt

Show First 20 Lines • Show All 277 Lines • ▼ Show 20 Lines def send_tool(self, tool):
tool['name'], tool['version'], tool['configuration']), tool['name'], tool['version'], tool['configuration']),
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'tool', 'swh_content_type': 'tool',
'swh_num': 1, 'swh_num': 1,
'swh_id': log_id 'swh_id': log_id
}) })
tools = list(self.storage.tool_add([tool])) tools = self.storage.tool_add([tool])
vlorentzUnsubmitted
Not Done
Inline Actions

Why this change? I don't think it will play well with a non-remote storage, because they return iterators.

vlorentz: Why this change? I don't think it will play well with a non-remote storage, because they return…
ardumontAuthorUnsubmitted
Done
Inline Actions

See D1321 which transformed the generator into a list to unify with other _add endpoints.
That simplifies the metrics extraction (counting length).

ardumont: See D1321 which transformed the generator into a list to unify with other `_add` endpoints.
tool_id = tools[0]['id'] tool_id = tools[0]['id']
self.log.debug( self.log.debug(
'Done creating tool with name %s version %s and configuration %s' % ( # noqa 'Done creating tool with name %s version %s and configuration %s' % ( # noqa
tool['name'], tool['version'], tool['configuration']), tool['name'], tool['version'], tool['configuration']),
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'tool', 'swh_content_type': 'tool',
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines def send_contents(self, content_list):
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d contents" % num_contents, self.log.debug("Sending %d contents" % num_contents,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'content', 'swh_content_type': 'content',
'swh_num': num_contents, 'swh_num': num_contents,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.content_add(content_list) result = self.storage.content_add(content_list)
self.counters['contents'] += num_contents self.counters['contents'] += result['content:add']
self.log.debug("Done sending %d contents" % num_contents, self.log.debug("Done sending %d contents" % num_contents,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'content', 'swh_content_type': 'content',
'swh_num': num_contents, 'swh_num': num_contents,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_directories(self, directory_list): def send_directories(self, directory_list):
"""Actually send properly formatted directories to the database. """Actually send properly formatted directories to the database.
""" """
num_directories = len(directory_list) num_directories = len(directory_list)
if num_directories > 0: if num_directories > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d directories" % num_directories, self.log.debug("Sending %d directories" % num_directories,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'directory', 'swh_content_type': 'directory',
'swh_num': num_directories, 'swh_num': num_directories,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.directory_add(directory_list) result = self.storage.directory_add(directory_list)
self.counters['directories'] += num_directories self.counters['directories'] += result['directory:add']
self.log.debug("Done sending %d directories" % num_directories, self.log.debug("Done sending %d directories" % num_directories,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'directory', 'swh_content_type': 'directory',
'swh_num': num_directories, 'swh_num_new': num_directories,
olasdUnsubmitted
Not Done
Inline Actions

I'm not sure this was intended?

olasd: I'm not sure this was intended?
ardumontAuthorUnsubmitted
Done
Inline Actions

no, it was not ;)

Your 'weird' feeling about the counter discrepancy was shared and i started something.
But then i stopped myself because it's out of scope (first migration) and did not want to go there.

That's a left over!
Thanks for pointing it out.

ardumont: no, it was not ;) Your 'weird' feeling about the counter discrepancy was shared and i started…
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_revisions(self, revision_list): def send_revisions(self, revision_list):
"""Actually send properly formatted revisions to the database. """Actually send properly formatted revisions to the database.
""" """
num_revisions = len(revision_list) num_revisions = len(revision_list)
if num_revisions > 0: if num_revisions > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d revisions" % num_revisions, self.log.debug("Sending %d revisions" % num_revisions,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'revision', 'swh_content_type': 'revision',
'swh_num': num_revisions, 'swh_num': num_revisions,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.revision_add(revision_list) result = self.storage.revision_add(revision_list)
self.counters['revisions'] += num_revisions self.counters['revisions'] += result['revision:add']
self.log.debug("Done sending %d revisions" % num_revisions, self.log.debug("Done sending %d revisions" % num_revisions,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'revision', 'swh_content_type': 'revision',
'swh_num': num_revisions, 'swh_num': num_revisions,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_releases(self, release_list): def send_releases(self, release_list):
"""Actually send properly formatted releases to the database. """Actually send properly formatted releases to the database.
""" """
num_releases = len(release_list) num_releases = len(release_list)
if num_releases > 0: if num_releases > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d releases" % num_releases, self.log.debug("Sending %d releases" % num_releases,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'release', 'swh_content_type': 'release',
'swh_num': num_releases, 'swh_num': num_releases,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.release_add(release_list) result = self.storage.release_add(release_list)
self.counters['releases'] += num_releases self.counters['releases'] += result['release:add']
self.log.debug("Done sending %d releases" % num_releases, self.log.debug("Done sending %d releases" % num_releases,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'release', 'swh_content_type': 'release',
'swh_num': num_releases, 'swh_num': num_releases,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_snapshot(self, snapshot): def send_snapshot(self, snapshot):
self.storage.snapshot_add(self.origin_id, self.visit, snapshot) self.storage.snapshot_add([snapshot])
olasdUnsubmitted
Done
Inline Actions

That should be snapshot['id']

olasd: That should be `snapshot['id']`
olasdUnsubmitted
Done
Inline Actions

also needs an explicit keyword argument

olasd: also needs an explicit keyword argument
self.storage.origin_visit_update(
self.origin_id, self.visit, snapshot=snapshot['id'])
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def filter_missing_contents(self, contents): def filter_missing_contents(self, contents):
"""Return only the contents missing from swh""" """Return only the contents missing from swh"""
max_content_size = self.config['content_size_limit'] max_content_size = self.config['content_size_limit']
contents_per_key = {} contents_per_key = {}
content_key = 'blake2s256' content_key = 'blake2s256'
▲ Show 20 Lines • Show All 514 Lines • Show Last 20 Lines
View Options

requirements.txt

Show First 20 Lines • Show All 277 Lines • ▼ Show 20 Lines def send_tool(self, tool):
tool['name'], tool['version'], tool['configuration']), tool['name'], tool['version'], tool['configuration']),
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'tool', 'swh_content_type': 'tool',
'swh_num': 1, 'swh_num': 1,
'swh_id': log_id 'swh_id': log_id
}) })
tools = list(self.storage.tool_add([tool])) tools = self.storage.tool_add([tool])
vlorentzUnsubmitted
Not Done
Inline Actions

Why this change? I don't think it will play well with a non-remote storage, because they return iterators.

vlorentz: Why this change? I don't think it will play well with a non-remote storage, because they return…
ardumontAuthorUnsubmitted
Done
Inline Actions

See D1321 which transformed the generator into a list to unify with other _add endpoints.
That simplifies the metrics extraction (counting length).

ardumont: See D1321 which transformed the generator into a list to unify with other `_add` endpoints.
tool_id = tools[0]['id'] tool_id = tools[0]['id']
self.log.debug( self.log.debug(
'Done creating tool with name %s version %s and configuration %s' % ( # noqa 'Done creating tool with name %s version %s and configuration %s' % ( # noqa
tool['name'], tool['version'], tool['configuration']), tool['name'], tool['version'], tool['configuration']),
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'tool', 'swh_content_type': 'tool',
▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines def send_contents(self, content_list):
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d contents" % num_contents, self.log.debug("Sending %d contents" % num_contents,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'content', 'swh_content_type': 'content',
'swh_num': num_contents, 'swh_num': num_contents,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.content_add(content_list) result = self.storage.content_add(content_list)
self.counters['contents'] += num_contents self.counters['contents'] += result['content:add']
self.log.debug("Done sending %d contents" % num_contents, self.log.debug("Done sending %d contents" % num_contents,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'content', 'swh_content_type': 'content',
'swh_num': num_contents, 'swh_num': num_contents,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_directories(self, directory_list): def send_directories(self, directory_list):
"""Actually send properly formatted directories to the database. """Actually send properly formatted directories to the database.
""" """
num_directories = len(directory_list) num_directories = len(directory_list)
if num_directories > 0: if num_directories > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d directories" % num_directories, self.log.debug("Sending %d directories" % num_directories,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'directory', 'swh_content_type': 'directory',
'swh_num': num_directories, 'swh_num': num_directories,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.directory_add(directory_list) result = self.storage.directory_add(directory_list)
self.counters['directories'] += num_directories self.counters['directories'] += result['directory:add']
self.log.debug("Done sending %d directories" % num_directories, self.log.debug("Done sending %d directories" % num_directories,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'directory', 'swh_content_type': 'directory',
'swh_num': num_directories, 'swh_num_new': num_directories,
olasdUnsubmitted
Not Done
Inline Actions

I'm not sure this was intended?

olasd: I'm not sure this was intended?
ardumontAuthorUnsubmitted
Done
Inline Actions

no, it was not ;)

Your 'weird' feeling about the counter discrepancy was shared and i started something.
But then i stopped myself because it's out of scope (first migration) and did not want to go there.

That's a left over!
Thanks for pointing it out.

ardumont: no, it was not ;) Your 'weird' feeling about the counter discrepancy was shared and i started…
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_revisions(self, revision_list): def send_revisions(self, revision_list):
"""Actually send properly formatted revisions to the database. """Actually send properly formatted revisions to the database.
""" """
num_revisions = len(revision_list) num_revisions = len(revision_list)
if num_revisions > 0: if num_revisions > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d revisions" % num_revisions, self.log.debug("Sending %d revisions" % num_revisions,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'revision', 'swh_content_type': 'revision',
'swh_num': num_revisions, 'swh_num': num_revisions,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.revision_add(revision_list) result = self.storage.revision_add(revision_list)
self.counters['revisions'] += num_revisions self.counters['revisions'] += result['revision:add']
self.log.debug("Done sending %d revisions" % num_revisions, self.log.debug("Done sending %d revisions" % num_revisions,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'revision', 'swh_content_type': 'revision',
'swh_num': num_revisions, 'swh_num': num_revisions,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_releases(self, release_list): def send_releases(self, release_list):
"""Actually send properly formatted releases to the database. """Actually send properly formatted releases to the database.
""" """
num_releases = len(release_list) num_releases = len(release_list)
if num_releases > 0: if num_releases > 0:
log_id = str(uuid.uuid4()) log_id = str(uuid.uuid4())
self.log.debug("Sending %d releases" % num_releases, self.log.debug("Sending %d releases" % num_releases,
extra={ extra={
'swh_type': 'storage_send_start', 'swh_type': 'storage_send_start',
'swh_content_type': 'release', 'swh_content_type': 'release',
'swh_num': num_releases, 'swh_num': num_releases,
'swh_id': log_id, 'swh_id': log_id,
}) })
self.storage.release_add(release_list) result = self.storage.release_add(release_list)
self.counters['releases'] += num_releases self.counters['releases'] += result['release:add']
self.log.debug("Done sending %d releases" % num_releases, self.log.debug("Done sending %d releases" % num_releases,
extra={ extra={
'swh_type': 'storage_send_end', 'swh_type': 'storage_send_end',
'swh_content_type': 'release', 'swh_content_type': 'release',
'swh_num': num_releases, 'swh_num': num_releases,
'swh_id': log_id, 'swh_id': log_id,
}) })
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def send_snapshot(self, snapshot): def send_snapshot(self, snapshot):
self.storage.snapshot_add(self.origin_id, self.visit, snapshot) self.storage.snapshot_add([snapshot])
olasdUnsubmitted
Done
Inline Actions

That should be snapshot['id']

olasd: That should be `snapshot['id']`
olasdUnsubmitted
Done
Inline Actions

also needs an explicit keyword argument

olasd: also needs an explicit keyword argument
self.storage.origin_visit_update(
self.origin_id, self.visit, snapshot=snapshot['id'])
@retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3)
def filter_missing_contents(self, contents): def filter_missing_contents(self, contents):
"""Return only the contents missing from swh""" """Return only the contents missing from swh"""
max_content_size = self.config['content_size_limit'] max_content_size = self.config['content_size_limit']
contents_per_key = {} contents_per_key = {}
content_key = 'blake2s256' content_key = 'blake2s256'
▲ Show 20 Lines • Show All 514 Lines • Show Last 20 Lines
About · Developer information · Legal