diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -43,15 +43,13 @@ ### Preparation steps -1. git clone under $GHLISTER_ROOT (of your choosing) +1. git clone under $SWH_ENVIRONMENT_HOME/swh-lister (of your choosing) 2. mkdir ~/.config/swh/ ~/.cache/swh/lister/github.com/ 3. create configuration file ~/.config/swh/lister-github.com.yml 4. Bootstrap the db instance schema -``` sh -$ createdb lister-github -$ python3 -m swh.lister.cli --db-url postgres:///lister-github github --createdb -``` + $ createdb lister-github + $ python3 -m swh.lister.cli --db-url postgres:///lister-github github --createdb ### Configuration file sample @@ -61,11 +59,15 @@ credentials: [] cache_responses: True cache_dir: /home/zack/.cache/swh/lister/github.com - - storage: - cls: remote + storage: # to avoid having to run yet another service + cls: local args: - url: http://localhost:5002/ + db: service=swh-dev + objstorage: + cls: pathslicing + args: + root: /home/storage/swh-storage/ + slicing: 0:1/1:5 ### Run @@ -79,3 +81,50 @@ DEBUG:urllib3.connectionpool:https://api.github.com:443 "GET /repositories?since=364 HTTP/1.1" 200 None DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): localhost DEBUG:urllib3.connectionpool:http://localhost:5002 "POST /origin/add HTTP/1.1" 200 1 + + +## lister-gitlab + +### preparation steps + +1. git clone under $SWH_ENVIRONMENT_HOME/swh-lister (of your choosing) +2. mkdir ~/.config/swh/ ~/.cache/swh/lister/gitlab/ +3. create configuration file ~/.config/swh/lister-gitlab.yml +4. Bootstrap the db instance schema + + $ createdb lister-gitlab + $ python3 -m swh.lister.cli --db-url postgres:///lister-gitlab gitlab --createdb + +### Configuration file sample + + $ cat ~/.config/swh/lister-gitlab.yml + # see http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls + lister_db_url: postgres:///lister-gitlab + credentials: [] + cache_responses: True + cache_dir: /home/zack/.cache/swh/lister/gitlab + storage: + cls: local + args: + db: service=swh-dev + objstorage: + cls: pathslicing + args: + root: /home/storage/swh-storage/ + slicing: 0:1/1:5 + +### Run + + $ python3 + Python 3.6.6 (default, Jun 27 2018, 14:44:17) + [GCC 8.1.0] on linux + Type "help", "copyright", "credits" or "license" for more information. + >>> from swh.lister.gitlab.tasks import RangeGitLabLister; RangeGitLabLister().run_task(1, 2, + lister_name='salsa.debian.org', api_baseurl='https://salsa.debian.org/api/v4') + >>> from swh.lister.gitlab.tasks import RangeGitLabLister; RangeGitLabLister().run_task(1, 2, + lister_name='gitlab.freedesktop.org', api_baseurl='https://gitlab.freedesktop.org/api/v4') + >>> from swh.lister.gitlab.tasks import RangeGitLabLister; RangeGitLabLister().run_task(1, 2, + lister_name='gitlab.gnome.org', api_baseurl='https://gitlab.gnome.org/api/v4') + >>> from swh.lister.gitlab.tasks import RangeGitLabLister; RangeGitLabLister().run_task(1, 2, + lister_name='gitlab.inria.fr', api_baseurl='https://gitlab.inria.fr/api/v4') + >>>