```
(setf forgerie-phabricator:*project-assignment-overrides*
'((:KEY 14 :NAME "Git cloner" :ACTION :ASSIGN :REPOSITORY "swh-cloner-git")
(:KEY 15 :NAME "Storage manager" :ACTION :ASSIGN :REPOSITORY "swh-storage")
(:KEY 16 :NAME "Core & foundations" :ACTION :ASSIGN :REPOSITORY "swh-core")
(:KEY 17 :NAME "Git loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-git")
(:KEY 18 :NAME "Directory loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-core")
(:KEY 19 :NAME "Development environment" :ACTION :ASSIGN :REPOSITORY "swh-environment")
(:KEY 20 :NAME "Web app" :ACTION :ASSIGN :REPOSITORY "swh-web")
(:KEY 21 :NAME "GitHub lister" :ACTION :ASSIGN :REPOSITORY "swh-lister")
(:KEY 22 :NAME "CGit lister" :ACTION :ASSIGN :REPOSITORY "swh-lister")
(:KEY 23 :NAME "Documentation" :ACTION :ASSIGN :REPOSITORY "swh-docs")
(:KEY 24 :NAME "Website" :ACTION :ASSIGN :REPOSITORY "www.softwareheritage.org-gandi")
(:KEY 25 :NAME "Debian loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-core")
(:KEY 26 :NAME "Tarball loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-core")
(:KEY 28 :NAME "Data Model" :ACTION :ASSIGN :REPOSITORY "swh-model")
(:KEY 34 :NAME "Antelink loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-antelink")
(:KEY 36 :NAME "Scheduling utilities" :ACTION :ASSIGN :REPOSITORY "swh-scheduler")
(:KEY 37 :NAME "SVN Loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-svn")
(:KEY 38 :NAME "Puppet recipes" :ACTION :ASSIGN :REPOSITORY "puppet-swh-site")
(:KEY 41 :NAME "System administration" :ACTION :ASSIGN :REPOSITORY "sysadm-environment")
(:KEY 42 :NAME "Core Loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-core")
(:KEY 43 :NAME "Fetcher Googlecode" :ACTION :ASSIGN :REPOSITORY "swh-fetcher-googlecode")
(:KEY 58 :NAME "Object storage" :ACTION :ASSIGN :REPOSITORY "swh-objstorage")
(:KEY 59 :NAME "Journal" :ACTION :ASSIGN :REPOSITORY "swh-journal")
(:KEY 61 :NAME "Indexer" :ACTION :ASSIGN :REPOSITORY "swh-indexer")
(:KEY 66 :NAME "Mercurial loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-mercurial")
(:KEY 67 :NAME "Bitbucket lister" :ACTION :ASSIGN :REPOSITORY "swh-lister")
(:KEY 76 :NAME "Vault" :ACTION :ASSIGN :REPOSITORY "swh-vault")
(:KEY 77 :NAME "SWORD deposit" :ACTION :ASSIGN :REPOSITORY "swh-deposit")
(:KEY 78 :NAME "Deposit" :ACTION :ASSIGN :REPOSITORY "swh-deposit")
(:KEY 79 :NAME "Software Heritage Archiver" :ACTION :ASSIGN :REPOSITORY "swh-objstorage")
(:KEY 88 :NAME "PyPI loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-core")
(:KEY 90 :NAME "Continuous Integration" :ACTION :ASSIGN :REPOSITORY "swh-jenkins-jobs")
(:KEY 91 :NAME "sandbox" :ACTION :ASSIGN :REPOSITORY "swh-sandbox")
(:KEY 111 :NAME "Npm Lister" :ACTION :ASSIGN :REPOSITORY "swh-lister")
(:KEY 116 :NAME "Docker environment" :ACTION :ASSIGN :REPOSITORY "swh-environment")
(:KEY 121 :NAME "Npm loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-core")
(:KEY 124 :NAME "SWH command line interface" :ACTION :ASSIGN :REPOSITORY "swh-core")
(:KEY 125 :NAME "Graph service" :ACTION :ASSIGN :REPOSITORY "swh-graph")
(:KEY 128 :NAME "Datasets" :ACTION :ASSIGN :REPOSITORY "swh-dataset")
(:KEY 129 :NAME "Lister" :ACTION :ASSIGN :REPOSITORY "swh-lister")
(:KEY 133 :NAME "CodeMeta Generator" :ACTION :ASSIGN :REPOSITORY "codemeta-generator")
(:KEY 134 :NAME "Archive search" :ACTION :ASSIGN :REPOSITORY "swh-search")
(:KEY 142 :NAME "Code scanner" :ACTION :ASSIGN :REPOSITORY "swh-scanner")
(:KEY 143 :NAME "Web client" :ACTION :ASSIGN :REPOSITORY "swh-web-client")
(:KEY 145 :NAME "Package Loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-core")
(:KEY 150 :NAME "Software Heritage filesystem" :ACTION :ASSIGN :REPOSITORY "swh-fuse")
(:KEY 152 :NAME "Nixguix loader" :ACTION :ASSIGN :REPOSITORY "swh-loader-core")))
(setf forgerie-phabricator:*repository-overrides*
'(
(:KEY 9 :ACTION :SKIP :SHORTNAME "git-loader-test-data")
(:KEY 15 :ACTION :UPDATE :DATA (:name "Credentials passwords keys etc"))
(:KEY 40 :ACTION :SKIP :SHORTNAME "org-slides")
(:KEY 42 :ACTION :SKIP :SHORTNAME "pg-hash")
(:KEY 48 :ACTION :SKIP :SHORTNAME "swh-statistics")
(:KEY 58 :ACTION :UPDATE :DATA (:name "Subversion loader"))
(:KEY 60 :ACTION :UPDATE :DATA (:name "Generic VCS and Package Loader"))
(:KEY 62 :ACTION :SKIP :SHORTNAME "FFEventFeed")
(:KEY 65 :ACTION :SKIP :SHORTNAME "staging")
(:KEY 67 :ACTION :SKIP :SHORTNAME "gsdgh")
(:KEY 73 :ACTION :SKIP :SHORTNAME "python-subvertpy")
(:KEY 89 :ACTION :UPDATE :DATA (:name "Manage etc resolv.conf with puppet"))
(:KEY 90 :ACTION :UPDATE :DATA (:name "Icinga 2 Puppet Module"))
(:KEY 102 :ACTION :UPDATE :DATA (:name "Web user interface - Django"))
(:KEY 105 :ACTION :UPDATE :DATA (:name "Manage entries in the Debian debconf database with Puppet"))
(:KEY 109 :ACTION :UPDATE :DATA (:name "Puppet NGINX management module"))
(:KEY 111 :ACTION :SKIP :SHORTNAME "test_pger")
(:KEY 114 :ACTION :SKIP :SHORTNAME "repos-git_1") ;; found through mysql
(:KEY 130 :ACTION :UPDATE :DATA (:name "Hello World"))
(:KEY 152 :ACTION :SKIP :SHORTNAME "swh-memory-graph")
(:KEY 160 :ACTION :SKIP :SHORTNAME "ci-environment") ;; empty
(:KEY 161 :ACTION :UPDATE :DATA (:name "Private Puppet Data - censored"))
(:KEY 195 :ACTION :UPDATE :DATA (:name "Cassandra and replayer deploment"))
(:KEY 202 :ACTION :UPDATE :DATA (:name "CodeMeta Generator - mirror"))
(:KEY 204 :ACTION :SKIP :SHORTNAME "swhapguide") ;; swhap guidelines
(:KEY 233 :ACTION :SKIP :SHORTNAME "test")
(:KEY 236 :ACTION :SKIP :SHORTNAME "python3-django-test-migrations")
(:KEY 239 :ACTION :SKIP :SHORTNAME "scrabble")
(:KEY 241 :ACTION :SKIP :SHORTNAME "python-pure-sasl")
(:KEY 245 :ACTION :SKIP :SHORTNAME "training-mob-programming")
(:KEY 249 :ACTION :SKIP :SHORTNAME "python3-py4j")
(:KEY 265 :ACTION :SKIP :SHORTNAME "django-rename-app")
))
(defun migrated-from (&key type id)
(let* ((type-str
(cond
((string= type "paste") "P")
((string= type "diff") "D")
((string= type "task") "T")))
(full-id (format nil "~A~A" type-str id)))
(format nil "~%~%----~%*Migrated from ~A ([view on Phabricator](https://forge.softwareheritage.org/~A))*"
full-id full-id)))
;; (migrated-from :type "paste" :id 1)
;; (migrated-from :type "diff" :id 2)
;; (migrated-from :type "task" :id 3)
; A function that takes an argument of a forgerie-core:merge-request and
; returns a string that will be appended to the description of created merge requests.
;
; Useful to create backlinks to the previous system, or addition migration information
(setf forgerie-gitlab:*merge-request-suffix*
(lambda (mr)
(migrated-from :type "diff" :id (forgerie-core:merge-request-id mr))))
; A function that takes an argument of a forgerie-core:ticket and
; returns a string that will be appended to the description of created tickets (issues).
;
; Useful to create backlinks to the previous system, or addition migration information
(setf forgerie-gitlab:*ticket-suffix*
(lambda (ticket)
(migrated-from :type "task" :id (forgerie-core:ticket-id ticket))))
(setf forgerie-gitlab:*snippet-suffix*
(lambda (snippet)
(migrated-from :type "paste" :id (forgerie-core:snippet-id snippet))))
;; (funcall forgerie-gitlab:*snippet-suffix* (car *snippets*))
;; (funcall forgerie-gitlab:*ticket-suffix* (car *tickets*))
;; (funcall forgerie-gitlab:*merge-request-suffix* (car *merge-requests*))
(format nil "~A~A" "blah" (funcall forgerie-gitlab:*snippet-suffix* (car *snippets*)))
;; Namespace structure
;; - Group: Software Heritage
;; - repos: python projects in a flat namespace
;; - Group: Infrastructure
;; - Group: Jenkins
;; - repos: jenkins-related repos
;; - Group: Puppet
;; - repos: our own puppet repos (puppet-swh-*)
;; - Group: puppet dependencies
;; - Group: Provisioning
;; - Group: git-annex
;; - Group: Teams
;; - Group: staff
;; - Group: HR
;; - Group: Research
;; - repos: papers
(setf forgerie-gitlab:*group-structure*
(labels ((nested (parent subgroup)
`(:name ,(getf subgroup :name)
:path ,(getf subgroup :path)
:parent ,(format nil "~@[~A/~]~A" (getf parent :parent) (getf parent :path))
,@(if (getf subgroup :subgroups)
`(:subgroups ,(getf subgroup :subgroups)))))
(dropsubgroups (entry)
`(:name ,(getf entry :name)
:path ,(getf entry :path)
,@(if (getf entry :parent)
`(:parent ,(getf entry :parent)))))
(flatten (structure)
(loop for parent in structure
nconcing (cons
(dropsubgroups parent)
(flatten
(mapcar
(lambda (subgroup) (nested parent subgroup))
(getf parent :subgroups)))))))
(flatten
'((:name "Software Heritage" :path "modules"
:subgroups
((:name "Fixture" :path "fixtures")
(:name "Experiments" :path "experiments")))
(:name "Infrastructure" :path "infra"
:subgroups
((:name "CI CD" :path "ci-cd"
:subgroups
((:name "Debian packages" :path "debs")))
(:name "Puppet" :path "puppet"
:subgroups
((:name "Dependencies" :path "3rdparty")))
(:name "Websites" :path "websites")
(:name "Annex" :path "annex")
(:name "Experiments" :path "experiments")))
(:name "Teams" :path "teams"
:subgroups
((:name "Staff" :path "staff")
(:name "System Administrators" :path "sysadmin")
(:name "Interns" :path "interns")
(:name "Management" :path "management")
))
(:name "Research" :path "research"
:subgroups
((:name "Papers" :path "papers")
(:name "Experiments" :path "experiments")))))))
(defun from-cache (wdir dir id)
"Read cache information from *working-directory*/<DIR>/<ID>"
(let ((cache-path (format nil "~A/~A/~A" wdir dir id)))
(when (probe-fiple cache-path)
(with-open-file (stream cache-path)
(read stream)))))
;; (let ((filepath "/var/tmp/forgerie/phabricator/everything/repositories"))
;; (when (probe-file filepath)
;; (with-open-file (str filepath)
;; (read str))))
(setf *repositories-cache*
(from-cache "/var/tmp/migrate-gitlab/forgerie/phabricator" "everything" "repositories"))
;; (setf *projects-cache*
;; (from-cache "/var/tmp/migrate-gitlab/forgerie/phabricator" "everything" "projects"))
;; (forgerie-core:vc-repository-slug (first *repositories-cache*)) ; => "puppet-environment"
;; (getf (first *repositories-cache*) :name)
(defun mapping-repo (repo)
(let*
((repo-slug (forgerie-core:vc-repository-slug repo))
(repo-tags
(remove-duplicates
(apply #'append
(mapcar
#'forgerie-core:project-tags
(append
(forgerie-core:vc-repository-primary-projects repo)
(forgerie-core:vc-repository-projects repo))))
:test #'string=)))
(cond
;; puppet repositories
((str:starts-with? "puppet-swh-" repo-slug) "infra/puppet")
((string= "puppet-environment" repo-slug) "infra/puppet")
((str:starts-with? "puppet-" repo-slug) "infra/puppet/3rdparty")
;; sysadmin stuff
((string= "swh-mirror" repo-slug) "infra")
((string= "swh-sysadmin" repo-slug) "infra")
((string= "iFWCFG" repo-slug) "infra") ;; opnsense
((string= "limnoria-plugins" repo-slug) "infra")
;; sysadm experiment
((string= "storage-benchmark-deployment" repo-slug) "infra/experiments")
((string= "cassandra-replayer-deployment" repo-slug) "infra/experiments")
;; websites and related
((string= "swh-keycloak-theme" repo-slug) "infra/websites")
((string= "www.softwareheritage.org-gandi" repo-slug) "infra/websites")
((string= "tools-mediawiki" repo-slug) "infra/websites")
;; ci
((string= "swh-jenkins" repo-slug) "infra/ci-cd")
((string= "swh-charts" repo-slug) "infra")
((string= "forgerie" repo-slug) "infra")
;; management
((string= "management" repo-slug) "teams/management")
((string= "team-management" repo-slug) "teams/management")
((string= "opdocs" repo-slug) "teams/management")
((string= "strategy" repo-slug) "teams/management")
((string= "gsoc-management" repo-slug) "teams/management")
((string= "swh-strategic-plan" repo-slug) "teams/management")
;; swh modules
((string= "swh-environment" repo-slug) "modules")
((string= "snippets" repo-slug) "modules")
((string= "swh-docs" repo-slug) "modules")
((string= "swh-loader-deposit" repo-slug) "modules")
((string= "swh-mirror-forge" repo-slug) "modules")
((string= "swh-storage-cassandra" repo-slug) "modules")
((string= "swh-docker-dev" repo-slug) "modules")
((string= "swh-web-django" repo-slug) "modules")
;; swh modules fixtures
((string= "jesuisgpl" repo-slug) "modules/fixtures")
((string= "helloworld" repo-slug) "modules/fixtures")
((string= "swh-storage-testdata" repo-slug) "modules/fixtures")
((string= "maven-index-exporter" repo-slug) "modules/fixtures")
((string= "codemeta-generator" repo-slug) "modules/fixtures")
;; swh modules experiments
((string= "pristine-zip" repo-slug) "modules/experiments")
((string= "mirror-tryout" repo-slug) "modules/experiments")
((string= "swh-db-audit" repo-slug) "modules/experiments")
((string= "listandsavegforge" repo-slug) "modules/experiments")
;; debian package dependencies
((str:starts-with? "python3-" repo-slug) "infra/ci-cd/debs")
((string= "universal-ctags" repo-slug) "infra/ci-cd/debs")
((string= "fossology-nomossa" repo-slug) "infra/ci-cd/debs")
((string= "python-cassandra-driver" repo-slug) "infra/ci-cd/debs")
((string= "django-webpack-loader" repo-slug) "infra/ci-cd/debs")
((string= "djangorestframework-xml" repo-slug) "infra/ci-cd/debs")
((string= "python-elasticsearch" repo-slug) "infra/ci-cd/debs")
((string= "monitoring-plugins-systemd" repo-slug) "infra/ci-cd/debs")
((string= "flask-api-debian-packaging" repo-slug) "infra/ci-cd/debs")
((string= "flask-testing-debian-packaging" repo-slug) "infra/ci-cd/debs")
;; research papers
((str:starts-with? "paper-" repo-slug) "research/papers")
((str:starts-with? "paper_" repo-slug) "research/papers")
((string= "emse-topology" repo-slug) "research/papers")
((string= "swh-topology-rr" repo-slug) "research/papers")
((string= "swhbib" repo-slug) "research/papers")
;; research experiments
;; git-annex
((str:starts-with? "annex-" repo-slug) "infra/annex")
;; slides
((string= "slides" repo-slug) "teams/staff")
((string= "swh-style" repo-slug) "teams/staff")
;; user related
((string= "seirl-data" repo-slug) "seirl")
;; internship
((string= "internship-lang-detection" repo-slug) "teams/interns")
((string= "metadata" repo-slug) "teams/interns")
;; default
((find "packagers" repo-tags :test #'string=) "infra/ci-cd/debs")
((find "continuous_integration" repo-tags :test #'string=) "infra/ci-cd")
((find "language-puppet" repo-tags :test #'string=) "infra/puppet")
((find "language-python" repo-tags :test #'string=) "modules")
((find "system_administrators" repo-tags :test #'string=) "infra")
((find "system_administration" repo-tags :test #'string=) "infra")
(t nil))))
;; slides, org-slides in team "speakers"?
;; how to deal
(setf forgerie-gitlab:*namespace-for-repo* 'mapping-repo)
(setf *print-miser-width* nil)
(setf *print-length* nil)
(setf *print-right-margin* 200)
(let* ((repos *repositories-cache*)
;; transform into a list of repositories to skip
(ignored-repos (remove-if-not #'identity
(mapcar (lambda (e) (let ((short (getf e :SHORTNAME)))
(when short short)))
forgerie-phabricator:*repository-overrides*)))
;; filter out repositories present in ignored-repos
(filtered-repos
(remove-if
(lambda (repo)
(let ((slug (forgerie-core:vc-repository-slug repo)))
(find slug ignored-repos :test #'string=)))
repos))
;; compute and sort the mapping for each repository
(tuples (sort
(mapcar
(lambda (repo)
(list (mapping-repo repo) (forgerie-core:vc-repository-slug repo) (forgerie-core:vc-repository-name repo)))
filtered-repos)
#'string-lessp :key #'first)))
;; and print it
(mapc 'pprint tuples)
nil)
(setf forgerie-phabricator:*included-repositories* nil
;; '("training-mob-programming" ;; private repository (well staff tagged)
;; "puppet-swh-site"
;; "swh-sysadmin-provisioning"
;; "puppet-environment"
;; ;; extra
;; "puppet-swh-postfix"
;; "getsentry-onpremise"
;; "puppet-puppet-grafana"
;; "puppet-theforeman-puppet"
;; "puppet-camptocamp-systemd"
;; "puppet-elastic-elastic_stack"
;; "puppet-puppetlabs-postgresql"
;; "puppet-puppetlabs-apache"
;; "puppet-icinga-icinga2"
;; "puppet-ssm-hitch"
;; "puppet-puppetlabs-puppetdb"
;; "puppet-puppet-letsencrypt"
;; "puppet-elastic-elasticsearch"
;; "puppet-openstack-ceph"
;; "puppet-puppet-nginx"
;; "puppet-icinga-icingaweb2"
;; "puppet-puppet-unattended_upgrades"
;; "puppet-puppet-cassandra"
;; "puppet-treydock-keycloak"
;; "puppet-puppet-archive"
;; "puppet-deric-zookeeper"
;; "puppet-puppet-redis"
;; "puppet-puppetlabs-mysql"
;; "puppet-puppet-extlib"
;; "puppet-puppetlabs-java_ks"
;; "puppet-puppetlabs-apt"
;; "puppet-puppetlabs-java"
;; "puppet-saz-memcached"
;; "puppet-puppetlabs-vcsrepo"
;; "puppet-puppetlabs-inifile"
;; "puppet-puppetlabs-stdlib"
;; "puppet-puppetlabs-concat"
;; "puppet-puppetlabs-docker"
;; "puppet-puppet-rabbitmq"
;; "puppet-puppetlabs-ntp"
;; "puppet-puppet-php"
;; "puppet-stm-debconf"
;; "puppet-saz-ssh"
;; "puppet-saz-sudo"
;; "puppet-saz-resolv_conf"
;; "puppet-saz-locales"
;; "puppet-puppet-kafka"
;; "puppet-swh-mediawiki"
;; "puppet-saz-timezone"
;; "puppet-puppetlabs-translate"
;; "puppet-swh-gunicorn"
;; "swh-grafana-dashboards"
;; "puppet-trepasi-debnet"
;; "puppet-ssm-munin"
;; "puppet-claranet-varnish"
;; "puppet-swh-uwsgi"
;; "puppet-swh-systemd"
;; "puppet-swh-dar"
;; "puppet-covermymeds-pgbouncer"
;; "puppet-swh-profile"
;; "puppet-swh-role"
;; "puppet-inkblot-bind"
;; "puppet-wikimedia-zookeeper"
;; "puppet-puppet-prometheus"
;; "puppet-thias-php"
;; "puppet-richardc-datacat"
;; "puppet-mosen-cups"
;; "puppet-wikimedia-kafka"
;; "puppet-arioch-redis"
;; "puppet-ripienaar-module_data"
;; "puppet-rlenglet-debconf_package"
;; "puppet-swh-private-data-censored"
;; "swh-jenkins-jobs"
;; "swh-jenkins-dockerfiles"
;; "swh-ansible"
;; )
)
```