Page MenuHomeSoftware Heritage
Paste P529

migrate swh-lister tasks (in scheduler db) following the lister normalization
ActivePublic
Actions

Authored by ardumont on Sep 11 2019, 3:54 PM.
-- gitlab (renamed key "api_baseurl" to "url")
update task set arguments='{"args": [], "kwargs": {"instance": "inria", "url": "https://gitlab.inria.fr/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'inria' and type in ('list-gitlab-full', 'list-gitlab-incremental');
update task set arguments='{"args": [], "kwargs": {"instance": "framagit", "url": "https://framagit.org/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'framagit' and type in ('list-gitlab-full', 'list-gitlab-incremental');
update task set arguments='{"args": [], "kwargs": {"instance": "riseup", "url": "https://0xacab.org/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'riseup' and type in ('list-gitlab-full', 'list-gitlab-incremental');
update task set arguments='{"args": [], "kwargs": {"instance": "gitlab", "url": "https://gitlab.com/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'gitlab' and type in ('list-gitlab-full', 'list-gitlab-incremental');
update task set arguments='{"args": [], "kwargs": {"instance": "freedesktop", "url": "https://gitlab.freedesktop.org/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'freedesktop' and type in ('list-gitlab-full', 'list-gitlab-incremental');
update task set arguments='{"args": [], "kwargs": {"instance": "ow2", "url": "https://gitlab.ow2.org/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'ow2' and type in ('list-gitlab-full', 'list-gitlab-incremental');
update task set arguments='{"args": [], "kwargs": {"instance": "debian", "url": "https://salsa.debian.org/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'debian' and type in ('list-gitlab-full', 'list-gitlab-incremental');
update task set arguments='{"args": [], "kwargs": {"instance": "gite.lirmm", "url": "https://gite.lirmm.fr/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'gite.lirmm' and type in ('list-gitlab-full', 'list-gitlab-incremental');
update task set arguments='{"args": [], "kwargs": {"instance": "common-lisp", "url": "https://gitlab.common-lisp.net/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'common-lisp' and type in ('list-gitlab-full', 'list-gitlab-incremental');
update task set arguments='{"args": [], "kwargs": {"instance": "gnome", "url": "https://gitlab.gnome.org/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'gnome' and type in ('list-gitlab-full', 'list-gitlab-incremental');
-- phabricator (renamed the key "forge_url" to "url")
update task set arguments='{"args": [], "kwargs": {"instance": "llvm", "url": "https://reviews.llvm.org/api/diffusion.repository.search"}}' where arguments#>>'{kwargs,instance}' = 'llvm' and type='list-phabricator-full';
update task set arguments='{"args": [], "kwargs": {"instance": "wikimedia", "url": "https://phabricator.wikimedia.org/api/diffusion.repository.search"}}' where arguments#>>'{kwargs,instance}' = 'wikimedia' and type='list-phabricator-full';
update task set arguments='{"args": [], "kwargs": {"instance": "blender", "url": "https://developer.blender.org/api/diffusion.repository.search"}}' where arguments#>>'{kwargs,instance}' = 'blender' and type='list-phabricator-full';
update task set arguments='{"args": [], "kwargs": {"instance": "kde", "url": "https://phabricator.kde.org/api/diffusion.repository.search"}}' where arguments#>>'{kwargs,instance}' = 'kde' and type='list-phabricator-full';
update task set arguments='{"args": [], "kwargs": {"instance": "swh", "url": "https://forge.softwareheritage.org/api/diffusion.repository.search"}}' where arguments#>>'{kwargs,instance}' = 'swh' and type='list-phabricator-full';
-- cgit ("prefix_url" key dropped, kept only the "forge_url" and renamed it "url")
update task set arguments='{"args": [], "kwargs": {"instance": "fedora", "url": "https://fedorapeople.org/cgit/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'fedora';
update task set arguments='{"args": [], "kwargs": {"instance": "git.gnu.org.ua", "url": "http://git.gnu.org.ua/cgit/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'git.gnu.org.ua';
update task set arguments='{"args": [], "kwargs": {"instance": "yoctoproject", "url": "http://git.yoctoproject.org/clean/cgit.cgi/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'yoctoproject';
update task set arguments='{"args": [], "kwargs": {"instance": "kde", "url": "https://cgit.kde.org/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'kde';
update task set arguments='{"args": [], "kwargs": {"instance": "hdiff.luite", "url": "http://hdiff.luite.com/cgit/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'hdiff.luite';
update task set arguments='{"args": [], "kwargs": {"instance": "tor", "url": "https://gitweb.torproject.org/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'tor';
update task set arguments='{"args": [], "kwargs": {"instance": "alpinelinux", "url": "https://git.alpinelinux.org/", "instance": "alpinelinux"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'alpinelinux';
update task set arguments='{"args": [], "kwargs": {"instance": "qt.io", "url": "https://code.qt.io/cgit/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'qt.io';
update task set arguments='{"args": [], "kwargs": {"instance": "baserock", "url": "https://git.baserock.org/cgit/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'baserock';
update task set arguments='{"args": [], "kwargs": {"instance": "happyassassin", "url": "https://www.happyassassin.net/cgit/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'happyassassin';
update task set arguments='{"args": [], "kwargs": {"instance": "git-kernel", "url": "https://git.kernel.org/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'git-kernel';
update task set arguments='{"args": [], "kwargs": {"instance": "openembedded", "url": "https://git.openembedded.org/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'openembedded';
update task set arguments='{"args": [], "kwargs": {"instance": "zx2c4", "url": "https://git.zx2c4.com/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'zx2c4';
update task set arguments='{"args": [], "kwargs": {"instance": "gnu-savannah", "url": "https://git.savannah.gnu.org/cgit/"}}' where type='list-cgit' and arguments#>>'{kwargs,instance}' = 'gnu-savannah';

Event Timeline

ardumont created this paste.Sep 11 2019, 3:54 PM
ardumont added a comment.Sep 11 2019, 3:56 PM

Not perfect for gitlab statements, see 2nd tryout to specify the type which is better:

softwareheritage-scheduler=> explain update task set arguments='{"args": [], "kwargs": {"instance": "inria", "url": "https://gitlab.inria.fr/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'inria';
                                  QUERY PLAN
-------------------------------------------------------------------------------
 Update on task  (cost=0.00..10290411.16 rows=894495 width=105)
   ->  Seq Scan on task  (cost=0.00..10290411.16 rows=894495 width=105)
         Filter: ((arguments #>> '{kwargs,instance}'::text[]) = 'inria'::text)
(3 rows)

softwareheritage-scheduler=> explain update task set arguments='{"args": [], "kwargs": {"instance": "inria", "url": "https://gitlab.inria.fr/api/v4"}}' where arguments#>>'{kwargs,instance}' = 'inria' and type in ('list-gitlab-full', 'list-
gitlab-incremental');
                                       QUERY PLAN
-----------------------------------------------------------------------------------------
 Update on task  (cost=0.57..8.79 rows=1 width=105)
   ->  Index Scan using temporary_task_types on task  (cost=0.57..8.79 rows=1 width=105)
         Index Cond: (type = ANY ('{list-gitlab-full,list-gitlab-incremental}'::text[]))
         Filter: ((arguments #>> '{kwargs,instance}'::text[]) = 'inria'::text)
(4 rows)
ardumont added a comment.Sep 11 2019, 3:57 PM

Also, current data is:

            type            |                                                                           arguments
----------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "inria", "api_baseurl": "https://gitlab.inria.fr/api/v4"}}
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "framagit", "api_baseurl": "https://framagit.org/api/v4"}}
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "riseup", "api_baseurl": "https://0xacab.org/api/v4"}}
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "gitlab", "api_baseurl": "https://gitlab.com/api/v4"}}
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "freedesktop", "api_baseurl": "https://gitlab.freedesktop.org/api/v4"}}
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "ow2", "api_baseurl": "https://gitlab.ow2.org/api/v4"}}
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "debian", "api_baseurl": "https://salsa.debian.org/api/v4"}}
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "gite.lirmm", "api_baseurl": "https://gite.lirmm.fr/api/v4"}}
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "common-lisp", "api_baseurl": "https://gitlab.common-lisp.net/api/v4"}}
 list-gitlab-full           | {"args": [], "kwargs": {"instance": "gnome", "api_baseurl": "https://gitlab.gnome.org/api/v4"}}
 list-gitlab-incremental    | {"args": [], "kwargs": {"instance": "inria", "api_baseurl": "https://gitlab.inria.fr/api/v4"}}
 list-gitlab-incremental    | {"args": [], "kwargs": {"instance": "framagit", "api_baseurl": "https://framagit.org/api/v4"}}
 list-gitlab-incremental    | {"args": [], "kwargs": {"instance": "riseup", "api_baseurl": "https://0xacab.org/api/v4"}}
 list-gitlab-incremental    | {"args": [], "kwargs": {"instance": "gitlab", "api_baseurl": "https://gitlab.com/api/v4"}}
 list-phabricator-full      | {"args": [], "kwargs": {"instance": "llvm", "forge_url": "https://reviews.llvm.org"}}
 list-phabricator-full      | {"args": [], "kwargs": {"instance": "wikimedia", "forge_url": "https://phabricator.wikimedia.org"}}
 list-phabricator-full      | {"args": [], "kwargs": {"instance": "blender", "forge_url": "https://developer.blender.org"}}
 list-phabricator-full      | {"args": [], "kwargs": {"instance": "kde", "forge_url": "https://phabricator.kde.org"}}
 list-phabricator-full      | {"args": [], "kwargs": {"instance": "swh", "forge_url": "https://forge.softwareheritage.org"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://fedorapeople.org/cgit/", "instance": "fedora"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "http://git.gnu.org.ua/cgit/", "instance": "git.gnu.org.ua", "url_prefix": "http://git.gnu.org.ua/repo/"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "http://git.yoctoproject.org/clean/cgit.cgi/", "instance": "yoctoproject", "url_prefix": "https://git.yoctoproject.org/git/"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://cgit.kde.org/", "instance": "kde", "url_prefix": "https://anongit.kde.org/"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "http://hdiff.luite.com/cgit/", "instance": "hdiff.luite"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://gitweb.torproject.org/", "instance": "tor", "url_prefix": "https://git.torproject.org/"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://git.alpinelinux.org/", "instance": "alpinelinux"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://code.qt.io/cgit/", "instance": "qt.io", "url_prefix": "http://code.qt.io/"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://git.baserock.org/cgit/", "instance": "baserock", "url_prefix": "https://git.baserock.org/git/"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://www.happyassassin.net/cgit/", "instance": "happyassassin"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://git.kernel.org/", "instance": "git-kernel"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://git.openembedded.org/", "instance": "openembedded"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://git.zx2c4.com/", "instance": "zx2c4"}}
 list-cgit                  | {"args": [], "kwargs": {"url": "https://git.savannah.gnu.org/cgit/", "instance": "gnu-savannah", "url_prefix": "http://git.savannah.gnu.org/git/"}}
--
 list-debian-distribution   | {"args": ["Debian-Security"], "kwargs": {}}
 list-debian-distribution   | {"args": ["Debian"], "kwargs": {}}
 list-github-full           | {"args": [], "kwargs": {}}
 list-pypi                  | {"args": [], "kwargs": {}}
 list-github-incremental    | {"args": [], "kwargs": {}}
 list-npm-full              | {"args": [], "kwargs": {}}
 list-bitbucket-incremental | {"args": [], "kwargs": {}}
 list-gnu-full              | {"args": [], "kwargs": {}}
(41 rows)
ardumont edited the content of this paste. (Show Details)Sep 11 2019, 4:00 PM
ardumont changed the title of this paste from migrate swh-lister tasks (in scheduler db) to migrate swh-lister tasks (in scheduler db) following the lister normalization.
ardumont updated the paste's language from autodetect to sql.
ardumont edited the content of this paste. (Show Details)
ardumont added a project: Lister.EditedSep 11 2019, 4:02 PM

Related D1950
Related D1929
Related T1861

ardumont edited the content of this paste. (Show Details)Sep 12 2019, 3:54 PM
About · Developer information · Legal