Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7123279
D2123.id7110.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
14 KB
Subscribers
None
D2123.id7110.diff
View Options
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@
version.txt
swh/lister/_version.py
.tox/
+.mypy_cache/
diff --git a/MANIFEST.in b/MANIFEST.in
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -7,3 +7,4 @@
include swh/lister/cran/list_all_packages.R
recursive-include swh/lister/*/tests/ *.json *.html *.txt *.* *
recursive-include swh/lister/cgit/tests/data/ *.* *
+recursive-include swh py.typed
diff --git a/mypy.ini b/mypy.ini
new file mode 100644
--- /dev/null
+++ b/mypy.ini
@@ -0,0 +1,36 @@
+[mypy]
+namespace_packages = True
+warn_unused_ignores = True
+
+
+# 3rd party libraries without stubs (yet)
+
+[mypy-bs4.*]
+ignore_missing_imports = True
+
+[mypy-celery.*]
+ignore_missing_imports = True
+
+[mypy-debian.*]
+ignore_missing_imports = True
+
+[mypy-iso8601.*]
+ignore_missing_imports = True
+
+[mypy-pkg_resources.*]
+ignore_missing_imports = True
+
+[mypy-pytest.*]
+ignore_missing_imports = True
+
+[mypy-requests_mock.*]
+ignore_missing_imports = True
+
+[mypy-testing.postgresql.*]
+ignore_missing_imports = True
+
+[mypy-urllib3.util.*]
+ignore_missing_imports = True
+
+[mypy-xmltodict.*]
+ignore_missing_imports = True
diff --git a/swh/__init__.py b/swh/__init__.py
--- a/swh/__init__.py
+++ b/swh/__init__.py
@@ -1 +1,4 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+from pkgutil import extend_path
+from typing import Iterable
+
+__path__ = extend_path(__path__, __name__) # type: Iterable[str]
diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py
--- a/swh/lister/bitbucket/lister.py
+++ b/swh/lister/bitbucket/lister.py
@@ -6,6 +6,7 @@
import iso8601
from datetime import datetime, timezone
+from typing import Any
from urllib import parse
from swh.lister.bitbucket.models import BitBucketModel
@@ -21,7 +22,7 @@
LISTER_NAME = 'bitbucket'
DEFAULT_URL = 'https://api.bitbucket.org/2.0'
instance = 'bitbucket'
- default_min_bound = datetime.fromtimestamp(0, timezone.utc)
+ default_min_bound = datetime.fromtimestamp(0, timezone.utc) # type: Any
def __init__(self, url=None, override_config=None, per_page=100):
super().__init__(url=url, override_config=override_config)
diff --git a/swh/lister/bitbucket/tests/test_bb_lister.py b/swh/lister/bitbucket/tests/test_bb_lister.py
--- a/swh/lister/bitbucket/tests/test_bb_lister.py
+++ b/swh/lister/bitbucket/tests/test_bb_lister.py
@@ -6,7 +6,6 @@
import unittest
from datetime import timedelta
-
from urllib.parse import unquote
import iso8601
@@ -16,7 +15,7 @@
from swh.lister.core.tests.test_lister import HttpListerTester
-def convert_type(req_index):
+def _convert_type(req_index):
"""Convert the req_index to its right type according to the model's
"indexable" column.
@@ -30,17 +29,17 @@
lister_subdir = 'bitbucket'
good_api_response_file = 'api_response.json'
bad_api_response_file = 'api_empty_response.json'
- first_index = convert_type('2008-07-12T07:44:01.476818+00:00')
- last_index = convert_type('2008-07-19T06:16:43.044743+00:00')
+ first_index = _convert_type('2008-07-12T07:44:01.476818+00:00')
+ last_index = _convert_type('2008-07-19T06:16:43.044743+00:00')
entries_per_page = 10
- convert_type = staticmethod(convert_type)
+ convert_type = _convert_type
def request_index(self, request):
"""(Override) This is needed to emulate the listing bootstrap
when no min_bound is provided to run
"""
m = self.test_re.search(request.path_url)
- idx = convert_type(m.group(1))
+ idx = _convert_type(m.group(1))
if idx == self.Lister.default_min_bound:
idx = self.first_index
return idx
diff --git a/swh/lister/core/abstractattribute.py b/swh/lister/core/abstractattribute.py
--- a/swh/lister/core/abstractattribute.py
+++ b/swh/lister/core/abstractattribute.py
@@ -16,7 +16,8 @@
import abc
class ClassContainingAnAbstractAttribute(abc.ABC):
- foo = AbstractAttribute('descriptive docstring for foo')
+ foo: Union[AbstractAttribute, Any] = \
+ AbstractAttribute('docstring for foo')
"""
__isabstractmethod__ = True
diff --git a/swh/lister/core/lister_base.py b/swh/lister/core/lister_base.py
--- a/swh/lister/core/lister_base.py
+++ b/swh/lister/core/lister_base.py
@@ -13,6 +13,7 @@
from sqlalchemy import create_engine, func
from sqlalchemy.orm import sessionmaker
+from typing import Any, Type, Union
from swh.core import config
from swh.scheduler import get_scheduler, utils
@@ -64,10 +65,12 @@
def is_within_bounds
"""
- MODEL = AbstractAttribute('Subclass type (not instance)'
- ' of swh.lister.core.models.ModelBase'
- ' customized for a specific service.')
- LISTER_NAME = AbstractAttribute("Lister's name")
+ MODEL = AbstractAttribute(
+ 'Subclass type (not instance) of swh.lister.core.models.ModelBase '
+ 'customized for a specific service.'
+ ) # type: Union[AbstractAttribute, Type[Any]]
+ LISTER_NAME = AbstractAttribute(
+ "Lister's name") # type: Union[AbstractAttribute, str]
def transport_request(self, identifier):
"""Given a target endpoint identifier to query, try once to request it.
diff --git a/swh/lister/core/lister_transports.py b/swh/lister/core/lister_transports.py
--- a/swh/lister/core/lister_transports.py
+++ b/swh/lister/core/lister_transports.py
@@ -12,6 +12,8 @@
import requests
import xmltodict
+from typing import Optional, Union
+
try:
from swh.lister._version import __version__
except ImportError:
@@ -29,14 +31,14 @@
To be used in conjunction with ListerBase or a subclass of it.
"""
- DEFAULT_URL = None
- PATH_TEMPLATE = AbstractAttribute('string containing a python string'
- ' format pattern that produces the API'
- ' endpoint path for listing stored'
- ' repositories when given an index.'
- ' eg. "/repositories?after=%s".'
- 'To be implemented in the API-specific'
- ' class inheriting this.')
+ DEFAULT_URL = None # type: Optional[str]
+ PATH_TEMPLATE = \
+ AbstractAttribute(
+ 'string containing a python string format pattern that produces'
+ ' the API endpoint path for listing stored repositories when given'
+ ' an index, e.g., "/repositories?after=%s". To be implemented in'
+ ' the API-specific class inheriting this.'
+ ) # type: Union[AbstractAttribute, Optional[str]]
EXPECTED_STATUS_CODES = (200, 429, 403, 404)
@@ -214,8 +216,9 @@
To be used in conjunction with ListerBase or a subclass of it.
"""
- PAGE = AbstractAttribute("The server api's unique page to retrieve and "
- "parse for information")
+ PAGE = AbstractAttribute(
+ "URL of the API's unique page to retrieve and parse "
+ "for information") # type: Union[AbstractAttribute, str]
PATH_TEMPLATE = None # we do not use it
def __init__(self, url=None):
diff --git a/swh/lister/core/models.py b/swh/lister/core/models.py
--- a/swh/lister/core/models.py
+++ b/swh/lister/core/models.py
@@ -8,6 +8,7 @@
from sqlalchemy import Column, DateTime, Integer, String
from sqlalchemy.ext.declarative import DeclarativeMeta
+from typing import Type, Union
from .abstractattribute import AbstractAttribute
@@ -24,9 +25,12 @@
class ModelBase(SQLBase, metaclass=ABCSQLMeta):
"""a common repository"""
__abstract__ = True
- __tablename__ = AbstractAttribute
+ __tablename__ = \
+ AbstractAttribute # type: Union[Type[AbstractAttribute], str]
- uid = AbstractAttribute('Column(<uid_type>, primary_key=True)')
+ uid = AbstractAttribute(
+ 'Column(<uid_type>, primary_key=True)'
+ ) # type: Union[AbstractAttribute, Column]
name = Column(String, index=True)
full_name = Column(String, index=True)
@@ -45,11 +49,14 @@
class IndexingModelBase(ModelBase, metaclass=ABCSQLMeta):
__abstract__ = True
- __tablename__ = AbstractAttribute
+ __tablename__ = \
+ AbstractAttribute # type: Union[Type[AbstractAttribute], str]
# The value used for sorting, segmenting, or api query paging,
# because uids aren't always sequential.
- indexable = AbstractAttribute('Column(<indexable_type>, index=True)')
+ indexable = AbstractAttribute(
+ 'Column(<indexable_type>, index=True)'
+ ) # type: Union[AbstractAttribute, Column]
def initialize(db_engine, drop_tables=False, **kwargs):
diff --git a/swh/lister/core/tests/test_abstractattribute.py b/swh/lister/core/tests/test_abstractattribute.py
--- a/swh/lister/core/tests/test_abstractattribute.py
+++ b/swh/lister/core/tests/test_abstractattribute.py
@@ -5,13 +5,15 @@
import abc
import unittest
+from typing import Any
+
from swh.lister.core.abstractattribute import AbstractAttribute
class BaseClass(abc.ABC):
- v1 = AbstractAttribute
- v2 = AbstractAttribute()
- v3 = AbstractAttribute('changed docstring')
+ v1 = AbstractAttribute # type: Any
+ v2 = AbstractAttribute() # type: Any
+ v3 = AbstractAttribute('changed docstring') # type: Any
v4 = 'qux'
diff --git a/swh/lister/core/tests/test_lister.py b/swh/lister/core/tests/test_lister.py
--- a/swh/lister/core/tests/test_lister.py
+++ b/swh/lister/core/tests/test_lister.py
@@ -10,6 +10,7 @@
import requests_mock
from sqlalchemy import create_engine
+from typing import Any, Callable, Optional, Pattern, Type, Union
from swh.lister.core.abstractattribute import AbstractAttribute
from swh.lister.tests.test_utils import init_db
@@ -28,9 +29,12 @@
to customize for a specific listing service.
"""
- Lister = AbstractAttribute('The lister class to test')
- lister_subdir = AbstractAttribute('bitbucket, github, etc.')
- good_api_response_file = AbstractAttribute('Example good response body')
+ Lister = AbstractAttribute(
+ 'Lister class to test') # type: Union[AbstractAttribute, Type[Any]]
+ lister_subdir = AbstractAttribute(
+ 'bitbucket, github, etc.') # type: Union[AbstractAttribute, str]
+ good_api_response_file = AbstractAttribute(
+ 'Example good response body') # type: Union[AbstractAttribute, str]
LISTER_NAME = 'fake-lister'
# May need to override this if the headers are used for something
@@ -157,13 +161,21 @@
to customize for a specific listing service.
"""
- last_index = AbstractAttribute('Last index in good_api_response')
- first_index = AbstractAttribute('First index in good_api_response')
- bad_api_response_file = AbstractAttribute('Example bad response body')
- entries_per_page = AbstractAttribute('Number of results in good response')
- test_re = AbstractAttribute('Compiled regex matching the server url. Must'
- ' capture the index value.')
- convert_type = str
+ last_index = AbstractAttribute(
+ 'Last index '
+ 'in good_api_response') # type: Union[AbstractAttribute, int]
+ first_index = AbstractAttribute(
+ 'First index in '
+ ' good_api_response') # type: Union[AbstractAttribute, Optional[int]]
+ bad_api_response_file = AbstractAttribute(
+ 'Example bad response body') # type: Union[AbstractAttribute, str]
+ entries_per_page = AbstractAttribute(
+ 'Number of results in '
+ 'good response') # type: Union[AbstractAttribute, int]
+ test_re = AbstractAttribute(
+ 'Compiled regex matching the server url. Must capture the '
+ 'index value.') # type: Union[AbstractAttribute, Pattern]
+ convert_type = str # type: Callable[..., Any]
"""static method used to convert the "request_index" to its right type (for
indexing listers for example, this is in accordance with the model's
"indexable" column).
@@ -343,9 +355,12 @@
to customize for a specific listing service.
"""
- entries = AbstractAttribute('Number of results in good response')
- PAGE = AbstractAttribute("The server api's unique page to retrieve and "
- "parse for information")
+ entries = AbstractAttribute(
+ 'Number of results '
+ 'in good response') # type: Union[AbstractAttribute, int]
+ PAGE = AbstractAttribute(
+ "URL of the server api's unique page to retrieve and "
+ "parse for information") # type: Union[AbstractAttribute, str]
def get_fl(self, override_config=None):
"""Retrieve an instance of fake lister (fl).
diff --git a/swh/lister/cran/lister.py b/swh/lister/cran/lister.py
--- a/swh/lister/cran/lister.py
+++ b/swh/lister/cran/lister.py
@@ -8,7 +8,7 @@
import subprocess
from collections import defaultdict
-from typing import List, Dict
+from typing import Any, Dict, List, Mapping
from swh.lister.cran.models import CRANModel
@@ -23,7 +23,7 @@
MODEL = CRANModel
LISTER_NAME = 'cran'
instance = 'cran'
- descriptions = defaultdict(dict)
+ descriptions = defaultdict(dict) # type: Mapping[str, Mapping[Any, Any]]
def task_dict(self, origin_type, origin_url, **kwargs):
"""Return task format dict
diff --git a/swh/lister/github/lister.py b/swh/lister/github/lister.py
--- a/swh/lister/github/lister.py
+++ b/swh/lister/github/lister.py
@@ -5,6 +5,8 @@
import re
import time
+from typing import Any
+
from swh.lister.core.indexing_lister import IndexingHttpLister
from swh.lister.github.models import GitHubModel
@@ -16,7 +18,7 @@
API_URL_INDEX_RE = re.compile(r'^.*/repositories\?since=(\d+)')
LISTER_NAME = 'github'
instance = 'github' # There is only 1 instance of such lister
- default_min_bound = 0
+ default_min_bound = 0 # type: Any
def get_model_from_repo(self, repo):
return {
diff --git a/swh/lister/phabricator/lister.py b/swh/lister/phabricator/lister.py
--- a/swh/lister/phabricator/lister.py
+++ b/swh/lister/phabricator/lister.py
@@ -20,7 +20,8 @@
class PhabricatorLister(IndexingHttpLister):
PATH_TEMPLATE = '?order=oldest&attachments[uris]=1&after=%s'
- DEFAULT_URL = 'https://forge.softwareheritage.org/api/diffusion.repository.search' # noqa
+ DEFAULT_URL = \
+ 'https://forge.softwareheritage.org/api/diffusion.repository.search'
MODEL = PhabricatorModel
LISTER_NAME = 'phabricator'
diff --git a/swh/lister/py.typed b/swh/lister/py.typed
new file mode 100644
--- /dev/null
+++ b/swh/lister/py.typed
@@ -0,0 +1 @@
+# Marker file for PEP 561.
diff --git a/tox.ini b/tox.ini
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist=flake8,py3
+envlist=flake8,mypy,py3
[testenv:py3]
deps =
@@ -17,3 +17,11 @@
flake8
commands =
{envpython} -m flake8
+
+[testenv:mypy]
+skip_install = true
+deps =
+ .[testing]
+ mypy
+commands =
+ mypy swh
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Wed, Dec 18, 11:02 AM (9 h, 51 m ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3226930
Attached To
D2123: typing: minimal changes to make a no-op mypy run pass
Event Timeline
Log In to Comment