diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ dist/ version.txt .tox +.mypy_cache/ diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include requirements-swh.txt include version.txt include README.md +recursive-include swh py.typed diff --git a/mypy.ini b/mypy.ini new file mode 100644 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,18 @@ +[mypy] +namespace_packages = True +warn_unused_ignores = True + + +# 3rd party libraries without stubs (yet) + +[mypy-elasticsearch.*] +ignore_missing_imports = True + +[mypy-msgpack.*] +ignore_missing_imports = True + +[mypy-pkg_resources.*] +ignore_missing_imports = True + +[mypy-pytest.*] +ignore_missing_imports = True diff --git a/swh/__init__.py b/swh/__init__.py --- a/swh/__init__.py +++ b/swh/__init__.py @@ -1 +1,4 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) +from pkgutil import extend_path +from typing import Iterable + +__path__ = extend_path(__path__, __name__) # type: Iterable[str] diff --git a/swh/search/cli.py b/swh/search/cli.py --- a/swh/search/cli.py +++ b/swh/search/cli.py @@ -73,7 +73,7 @@ @cli.command('rpc-serve') -@click.argument('config-path', required=1) +@click.argument('config-path', required=True) @click.option('--host', default='0.0.0.0', help="Host to run the server") @click.option('--port', default=5010, type=click.INT, help="Binding port of the server") diff --git a/swh/search/elasticsearch.py b/swh/search/elasticsearch.py --- a/swh/search/elasticsearch.py +++ b/swh/search/elasticsearch.py @@ -4,7 +4,7 @@ # See top-level LICENSE file for more information import base64 -from typing import Iterable, Dict, List, Iterator +from typing import Any, Iterable, Dict, List, Iterator, Optional from elasticsearch import Elasticsearch from elasticsearch.helpers import bulk, scan @@ -131,7 +131,7 @@ list of dictionaries with key: * `url`: URL of a matching origin """ - query_clauses = [] + query_clauses = [] # type: List[Dict[str, Any]] if url_pattern: query_clauses.append({ @@ -185,9 +185,11 @@ } if scroll_token: # TODO: use ElasticSearch's scroll API? - scroll_token = msgpack.loads(base64.b64decode(scroll_token)) + scroll_token_content = msgpack.loads( + base64.b64decode(scroll_token)) body['search_after'] = \ - [scroll_token[b'score'], scroll_token[b'id'].decode('ascii')] + [scroll_token_content[b'score'], + scroll_token_content[b'id'].decode('ascii')] res = self._backend.search( index='origin', @@ -199,12 +201,12 @@ if len(hits) == count: last_hit = hits[-1] - next_scroll_token = { + next_scroll_token_content = { b'score': last_hit['_score'], b'id': last_hit['_id'], } next_scroll_token = base64.b64encode(msgpack.dumps( - next_scroll_token)) + next_scroll_token_content)) # type: Optional[bytes] else: next_scroll_token = None diff --git a/swh/search/in_memory.py b/swh/search/in_memory.py --- a/swh/search/in_memory.py +++ b/swh/search/in_memory.py @@ -7,7 +7,7 @@ from collections import defaultdict import itertools import re -from typing import Iterable, Dict +from typing import Any, Dict, Iterable, Iterator, List, Optional import msgpack @@ -40,8 +40,8 @@ del self._origin_ids def initialize(self) -> None: - self._origins = defaultdict(dict) - self._origin_ids = [] + self._origins = defaultdict(dict) # type: Dict[str, Dict[str, Any]] + self._origin_ids = [] # type: List[str] _url_splitter = re.compile(r'\W') @@ -64,7 +64,7 @@ with_visit: bool = False, scroll_token: str = None, count: int = 50 ) -> Dict[str, object]: - matches = (self._origins[id_] for id_ in self._origin_ids) + matches = (self._origins[id_] for id_ in self._origin_ids) # type: Iterator[Dict[str, Any]] if url_pattern: tokens = set(self._url_splitter.split(url_pattern)) @@ -96,8 +96,9 @@ matches = filter(lambda o: o.get('has_visits'), matches) if scroll_token: - scroll_token = msgpack.loads(base64.b64decode(scroll_token)) - start_at_index = scroll_token[b'start_at_index'] + scroll_token_content = msgpack.loads( + base64.b64decode(scroll_token)) + start_at_index = scroll_token_content[b'start_at_index'] else: start_at_index = 0 @@ -105,11 +106,11 @@ matches, start_at_index, start_at_index+count)) if len(hits) == count: - next_scroll_token = { + next_scroll_token_content = { b'start_at_index': start_at_index+count, } next_scroll_token = base64.b64encode(msgpack.dumps( - next_scroll_token)) + next_scroll_token_content)) # type: Optional[bytes] else: next_scroll_token = None diff --git a/swh/search/py.typed b/swh/search/py.typed new file mode 100644 --- /dev/null +++ b/swh/search/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. diff --git a/tox.ini b/tox.ini --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=flake8,py3 +envlist=flake8,mypy,py3 [testenv:py3] deps = @@ -16,3 +16,11 @@ flake8 commands = {envpython} -m flake8 + +[testenv:mypy] +skip_install = true +deps = + .[testing] + mypy +commands = + mypy swh