diff --git a/DESIGN.md b/DESIGN.md index 5d25c53..22bb440 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,13 +1,13 @@ This is just a dumping ground about design of project. This should get updated at some point into a nice form, but for now it's just a list of things as I think of them. * Imports come from database. Don't use api to import information, except as reference. For instance, if the username from the api is different than in the database, figure out why and adjust. * Export go to api first, and then follow up with database edits. The API for any given system should set up objects correctly. Then, after initial, edit database directly. For instance, the user created date is probably set to `time.now()` when creating through the api, but we may want the user creation date to be set to what it was in the previous system. -* Each forge gets it's own component, and implements some top level generic methods. +* Each forge gets its own component, and implements some top level generic methods. diff --git a/README.md b/README.md index 693805a..53d80d7 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,46 @@ # Forgerie Forgerie is an open source tool for converting software projects from one collaborative development forge to another (e.g., Phabricator -> GitLab). See the [DESIGN.md](DESIGN.md) for design decisions. # Install quicklisp Forgerie requires quicklisp on sbcl. Generally you can get sbcl through your distribution. Once installed, you then need to install quicklsip. Follow the installation instructions at [quicklisp.org](https://www.quicklisp.org/beta/) # Running ## Configure To configure: ``` -$ cp config/config.lisp.tmpl config/config.lisp +$ cp config/config.lisp.tmpl config/config.lisp $ $EDITOR config/config.lisp ``` A lot of these configuration options will lead you to need to take other actions, such as making sure the database for phabricator is local to the machine. You'll also need to ensure that keys are set up on various machines that will need to be sshed to. ## Run the script Run `bin/run` # Postmortem -After run, if `forgerie-core:*log-mapping-errors*` is turned out, running +After run, if `forgerie-core:*log-mapping-errors*` is turned on, running `bin/postmortem` will dump out a log of all the errors that happened during the run # Generating config.lisp.tmpl There's a helper script `bin/build-config-tmpl` to generate `config/config.lisp.tmpl` from inline documentation. diff --git a/bin/build-config-tmpl b/bin/build-config-tmpl index e821376..9193664 100755 --- a/bin/build-config-tmpl +++ b/bin/build-config-tmpl @@ -1,66 +1,66 @@ #!/usr/bin/env -S /usr/bin/sbcl --script (let ((quicklisp-init (merge-pathnames "quicklisp/setup.lisp" (user-homedir-pathname)))) (when (probe-file quicklisp-init) (load quicklisp-init))) (ql:quickload :forgerie) (with-open-file (str "config/config.lisp.tmpl" :direction :output :if-exists :supersede) (format str -";;;; This template file is generated by bin/build-config-templ and should not be edited manuall. +";;;; This template file is generated by bin/build-config-templ and should not be edited manually. ;;;; ;;;; Install by copying to config.lisp~%~%") (mapcar (lambda (section) (format str ";;;;;;;;;;;;;;;;;;;~%;;; ~A~%;;;;;;;;;;;;;;;;;;;~%~%" (car section)) (mapcar (lambda (variable-def) (let ((variable-name (if (listp variable-def) (car variable-def) variable-def)) (default (if (listp variable-def) (cadr variable-def) "nil"))) - (format str + (format str "~{;~A~%~}(setf ~(~A~):~(~A~) ~A)~%~%" (mapcar (lambda (line) (if (< 0 (length line)) (cl-ppcre:regex-replace-all "^ ?" line " ") "")) (cl-ppcre:split "\\n" (documentation variable-name 'variable))) (package-name (symbol-package variable-name)) (symbol-name variable-name) default ))) (cdr section))) '(("system" (sb-impl::*default-external-format* ":UTF-8") (drakma:*drakma-default-external-format* "'UTF-8")) ("core" forgerie-core:*debug* forgerie-core:*log-mapping-errors* forgerie-core:*working-directory*) ("phabricator" forgerie-phabricator:*database-password* forgerie-phabricator:*database-username* forgerie-phabricator:*revisions-to-skip* forgerie-phabricator:*phabricator-location* forgerie-phabricator:*project-assignment-overrides* forgerie-phabricator:*repository-overrides* forgerie-phabricator:*pastes-to-skip* forgerie-phabricator:*user-overrides* forgerie-phabricator:*storage-location* forgerie-phabricator:*confidential-space-phids* forgerie-phabricator:*included-repositories* forgerie-phabricator:*staging-repository* forgerie-phabricator:*email-address-sanitizer*) ("gitlab" forgerie-gitlab:*private-token* forgerie-gitlab:*server-address* forgerie-gitlab:*default-project* forgerie-gitlab:*ssh-public-key* forgerie-gitlab:*default-group* forgerie-gitlab:*rails-console-ssh-args* forgerie-gitlab:*merge-request-suffix* forgerie-gitlab:*ticket-suffix* forgerie-gitlab:*limit-to-active-users*)))) diff --git a/config/config.lisp.tmpl b/config/config.lisp.tmpl index f46102f..07b0cc6 100644 --- a/config/config.lisp.tmpl +++ b/config/config.lisp.tmpl @@ -1,172 +1,171 @@ -;;;; This template file is generated by bin/build-config-templ and should not be edited manuall. +;;;; This template file is generated by bin/build-config-templ and should not be edited manually. ;;;; ;;;; Install by copying to config.lisp ;;;;;;;;;;;;;;;;;;; ;;; system ;;;;;;;;;;;;;;;;;;; (setf sb-ext:*default-external-format* :UTF-8) ; The default value for the external format keyword arguments of ; HTTP-REQUEST. The value of this variable will be interpreted by ; FLEXI-STREAMS. The initial value is the keyword :LATIN-1. ; (Note that Drakma binds *DEFAULT-EOL-STYLE* to :LF). (setf drakma:*drakma-default-external-format* 'UTF-8) ;;;;;;;;;;;;;;;;;;; ;;; core ;;;;;;;;;;;;;;;;;;; ; Set to T to enable output of trace level messaging, such ; as http requests, git commands, and database queries (setf forgerie-core:*debug* nil) ; Boolean which indicates whether mapping errors should be logged. Defaults ; to T to create a mapping errors file that can be later used to output ; what errors the run ran into. (setf forgerie-core:*log-mapping-errors* nil) ; The working directory for the run. Core files will be put into ; 'core' underneath this, and then each forgerie will base on this. (setf forgerie-core:*working-directory* nil) ;;;;;;;;;;;;;;;;;;; ;;; phabricator ;;;;;;;;;;;;;;;;;;; ; Password to access the mysql database. If NIL, will not enter password (setf forgerie-phabricator:*database-password* nil) ; Username to access the database. If NIL, will use the current user (setf forgerie-phabricator:*database-username* nil) ; List of revisions to skip. For instance, if they cause errors due to weird git ; history. They need to be handled manually. The list is of database ids. (setf forgerie-phabricator:*revisions-to-skip* nil) ; The HTTP location of the phabricator instance. This is only used for ; differentials that cannot be understood via the database. The raw diffs ; for these are pulled from the instance, and so they need to be accessible ; from the script. (setf forgerie-phabricator:*phabricator-location* nil) ; A list of plists of override commands for projects. Each item in the list is of the form ; '(:key KEY :repository SLUG) ; ; Where the KEY is the database id of the project, and the SLUG is which repository that this ; project should be a primary project of. Then the project will be removed from all other -; repositories it mgiht be assigned to. The ramification of this is that tasks that are +; repositories it might be assigned to. The ramification of this is that tasks that are ; part of this project, for instance, will be assigned to the repository in various forgeries ; that link project and repository. (setf forgerie-phabricator:*project-assignment-overrides* nil) ; A list of plists for overriding certain features of projects. The plists are of the form ; '(:key KEY :action ACTION) ; ; Where KEY is the id of the database. ACTION can be either :skip or :update. ; - :skip, the repository will be skipped (useful for things like the staging repository) ; - :update, will be require a further item :DATA which is a plist of overrides corresponding ; to database fields for the repository table. Useful when renaming items, or specifying ; slugs. (setf forgerie-phabricator:*repository-overrides* nil) ; Pastes that can't be migrated, and will need to be handled manually. This is a list of ; database ids. They need to be handled manually. (setf forgerie-phabricator:*pastes-to-skip* nil) ; A list of plists for overriding certain features of users. The plists are of the form ; '(:key KEY :action ACTION) ; ; Where KEY is the id of the database. ACTION can be only :update. ; - :update, will be require a further item :DATA which is a plist of overrides corresponding ; to database fields for the user table. Useful when renaming user names, or specifying ; other aspects of the user. (setf forgerie-phabricator:*user-overrides* nil) ; The path on the local file system for the local storage of files. The phabricator_file ; database can refer to local storage items, which are stored here (other options being ; that the file is stored in the database). (setf forgerie-phabricator:*storage-location* nil) ; List of spaces that should be marked as confidential on the export. (setf forgerie-phabricator:*confidential-space-phids* nil) ; When doing only a partial import, use to list which slugs to be imported. ; ; This means that any tasks will be not mappable to a repository, and may end up in the ; default project of the exporter, so when using this, you'll want to disable that feature ; in the exporter of choice (setf forgerie-phabricator:*included-repositories* nil) ; PHID of the staging repository, if used. If NIL, commits for differentials will ; not be extracted from staging. Used if set up with arcanist. (setf forgerie-phabricator:*staging-repository* nil) ; A function that takes a string representing an email address, and then returns ; one that should be used in its place. Used in testing mode to ensure that ; users aren't getting random emails from the export forgerie, as an extra precaution. (setf forgerie-phabricator:*email-address-sanitizer* nil) ;;;;;;;;;;;;;;;;;;; ;;; gitlab ;;;;;;;;;;;;;;;;;;; ; The private token with which to access the gitlab instance. Needs to be ; set up either from within gitlab, or via a script that uses the rails console ; directory (setf forgerie-gitlab:*private-token* nil) ; The fully qualitifed server address for the gitlab instance, including the scheme to ; use, e.g. https://gitlab.yourdomain.tld (setf forgerie-gitlab:*server-address* nil) ; A plist of the form '(:name NAME :slug SLUG) for the default project ; in which things like snippets, tickets that can't be assigned to a project, ; and other misc items go. NAME is the proper name of the project, with ; SLUG being the url slug to access it. (setf forgerie-gitlab:*default-project* nil) ; The public key that should be installed for the running user so that git commands ; work correctly. (setf forgerie-gitlab:*ssh-public-key* nil) ; A plist of the form '(:name NAME :path SLUG) that defines the group in which ; all created projects will be placed. NAME is the proper name for the group, ; while SLUG is the url slug. If NIL, the projects will all be created at the ; top level. (setf forgerie-gitlab:*default-group* nil) ; A tuple of the form '(HOST COMMAND) that informs the gitlab forgerie ; how to run rails commands over ssh. It will always use SSH, even if ; set up to run on localhost, so keys must be installed to ssh to localhost. ; ; An example for a server using docker might be: ; ; '("ssh.gitlab.yourdomain.tld" "docker exec -i gitlab /opt/gitlab/bin/gitlab-rails c") ; ; A useful thing to do is to run ssh on the server for non git purposes on port 2222, and then ; set up your .ssh/config to have the following: ; ; Host ssh.gitlab.yourdomain.tld ; User ; Port 2222 ; IdentityFile ~/.ssh/your_identity_file (setf forgerie-gitlab:*rails-console-ssh-args* nil) ; A function that takes an argument of a forgerie-core:merge-request and ; returns a string that will be appended to the description of created merge requests. ; ; Useful to create backlinks to the previous system, or addition migration information (setf forgerie-gitlab:*merge-request-suffix* nil) ; A function that takes an argument of a forgerie-core:ticket and ; returns a string that will be appended to the description of created tickets (issues). ; ; Useful to create backlinks to the previous system, or addition migration information (setf forgerie-gitlab:*ticket-suffix* nil) ; If non nil, will only add users to the gitlab instance if they are active in the ; items also coming over for processing. Useful when doing piecemeal conversions. (setf forgerie-gitlab:*limit-to-active-users* nil) -