diff --git a/common/modules/deposit.org b/common/modules/deposit.org index a02f4b5..a7b2efe 100644 --- a/common/modules/deposit.org +++ b/common/modules/deposit.org @@ -1,183 +1,183 @@ #+COLUMNS: %40ITEM %10BEAMER_env(Env) %9BEAMER_envargs(Env Args) %10BEAMER_act(Act) %4BEAMER_col(Col) %10BEAMER_extra(Extra) %8BEAMER_opt(Opt) #+INCLUDE: "prelude.org" :minlevel 1 # Requirements: # #+LATEX_HEADER_EXTRA: \usepackage{tikz} #+LATEX_HEADER_EXTRA: \usetikzlibrary{arrows,shapes} #+LATEX_HEADER_EXTRA: \definecolor{swh-orange}{RGB}{254,205,27} #+LATEX_HEADER_EXTRA: \definecolor{swh-red}{RGB}{226,0,38} #+LATEX_HEADER_EXTRA: \definecolor{swh-green}{RGB}{77,181,174} * Deposit :PROPERTIES: :CUSTOM_ID: main :END: ** Pushing source code to Software Heritage :PROPERTIES: :CUSTOM_ID: overview :END: **** Deposit service - complement regular (pull) crawling of forges and distributions - restricted access (i.e., not a warez dumpster!) - early prototype at: *\url{https://deposit.softwareheritage.org/}* **** Tech bits - *SWORD* 2.0 compliant server, for digital repositories interoperability - RESTful API for deposit and monitoring, with CLI wrapper ** Prepare a deposit :PROPERTIES: :CUSTOM_ID: prepare :END: *** Prepare source code tarball #+BEGIN_SRC $ tar caf software.tar.gz /path/to/software/ #+END_SRC #+BEAMER: \pause *** Associate metadata #+BEAMER: \scriptsize #+BEGIN_SRC $ cat > software.tar.gz.metadata.xml Je suis GPL https://forge.softwareheritage.org/source/jesuisgpl/ Stefano Zacchiroli Maintainer ^D #+END_SRC ** Send a deposit :PROPERTIES: :CUSTOM_ID: send :END: *** #+BEGIN_SRC $ swh-deposit --username 'name' --password 'pass' \ --archive software.tar.gz #+END_SRC #+BEAMER: \pause #+BEGIN_SRC { 'deposit_id': '11', 'deposit_status': 'deposited', 'deposit_date': 'Jan. 30, 2018, 9:37 a.m.' } #+END_SRC ** Multipart deposit :PROPERTIES: :CUSTOM_ID: multipart :END: *** #+BEAMER: \scriptsize #+BEGIN_SRC $ swh-deposit --username 'name' --password 'secret' \ --partial \ --archive the-bulk-of-it.tar.gz #+END_SRC #+BEAMER: \pause #+BEGIN_SRC { 'deposit_id': '11', 'deposit_status': 'partial', 'deposit_date': 'Jan. 30, 2018, 9:37 a.m.' } #+END_SRC #+BEAMER: \pause *** #+BEAMER: \scriptsize #+BEGIN_SRC $ swh-deposit --username 'name' --password 'secret' \ --partial \ # omit this to finalize --deposit-id 11 --archive addendum.tar.gz #+END_SRC #+BEAMER: \pause *** #+BEAMER: \scriptsize #+BEGIN_SRC $ swh-deposit --username 'name' --password 'secret' \ --deposit-id 11 --archive last-touch.tar.gz #+END_SRC ** Replace previous archive and/or metadata :PROPERTIES: :CUSTOM_ID: override :END: *** #+BEAMER: \scriptsize #+BEGIN_SRC $ swh-deposit --username 'name' --password 'secret' \ --deposit-id 11 \ --archive updated-software.tar.gz \ --replace #+END_SRC #+BEAMER: \pause #+BEAMER: \scriptsize #+BEGIN_SRC { 'deposit_id': '11', 'deposit_status': 'deposited', 'deposit_date': 'Jan. 30, 2018, 9:37 a.m.' } #+END_SRC ** Ingestion status :PROPERTIES: :CUSTOM_ID: status :END: \tikzstyle{fail} = [draw, thin, fill=swh-red!80!swh-orange, minimum height=1.5em] \tikzstyle{processing} = [draw, thin, fill=swh-orange!100, minimum height=1.5em] \tikzstyle{success} = [draw, thin, fill=swh-green!80!swh-orange, minimum height=1.5em] \begin{figure} \begin{tikzpicture}[node distance=1.8cm, auto,>=latex', thick] % We need to set at bounding box first. Otherwise the diagram % will change position for each frame. %\path[use as bounding box] (-1,0) rectangle (10,-2); \path[->]<1-> node[processing] (partial) {partial} node[success, right of=partial] (deposited) {deposited} (partial) edge node {} (deposited) node[processing, right of=deposited] (checking) {} (deposited) edge node {} (checking) node[success, right of=checking] (verified) {verified} node[fail, below of=checking] (rejected) {rejected} (checking) edge node {} (verified) edge node[swap] {} (rejected) node[processing, right of=verified] (loading) {} node[success, right of=loading] (done) {done} node[fail, below of=loading] (failed) {failed} (verified) edge node {} (loading) (loading) edge node {} (failed) (loading) edge node {} (done); \end{tikzpicture} \end{figure} #+BEAMER: \pause *** #+BEAMER: \footnotesize #+BEGIN_SRC $ swh-deposit --username 'name' --pass 'secret' \ --deposit-id '11' --status #+END_SRC #+BEAMER: \pause #+BEGIN_SRC { 'deposit_id': 11, 'deposit_status': 'done', 'deposit_status_detail': The deposit has been successfully loaded into the Software Heritage archive', - 'deposit_swh_id': 'swh:1:rev:608757ea9bd8494d729732cc9a414948c160bd3c' + 'deposit_swh_id': 'swh:1:rev:a86747d201ab8f8657d145df4376676d5e47cf9f' } #+END_SRC ** Access a deposit :PROPERTIES: :CUSTOM_ID: access :END: After ingestion a deposit becomes an integral, permanent part of the Software Heritage archive. - it has a *persistent identifier* - - e.g., =swh:1:rev:608757ea9bd8494d729732cc9a414948c160bd3c= + - e.g., =swh:1:rev:a86747d201ab8f8657d145df4376676d5e47cf9f= - it can be *browsed* online at https://archive.softwareheritage.org/ - e.g., - https://archive.softwareheritage.org/browse/swh:1:rev:608757ea9bd8494d729732cc9a414948c160bd3c + https://archive.softwareheritage.org/browse/swh:1:rev:a86747d201ab8f8657d145df4376676d5e47cf9f - it can be bulk *downloaded* using the Software Heritage Vault diff --git a/common/modules/vault.org b/common/modules/vault.org index fbb7313..f032599 100644 --- a/common/modules/vault.org +++ b/common/modules/vault.org @@ -1,110 +1,110 @@ #+COLUMNS: %40ITEM %10BEAMER_env(Env) %9BEAMER_envargs(Env Args) %10BEAMER_act(Act) %4BEAMER_col(Col) %10BEAMER_extra(Extra) %8BEAMER_opt(Opt) #+INCLUDE: "prelude.org" :minlevel 1 # Requirements: # #+LATEX_HEADER_EXTRA: \usepackage{tikz} #+LATEX_HEADER_EXTRA: \usetikzlibrary{arrows,shapes} #+LATEX_HEADER_EXTRA: \definecolor{swh-orange}{RGB}{254,205,27} #+LATEX_HEADER_EXTRA: \definecolor{swh-red}{RGB}{226,0,38} #+LATEX_HEADER_EXTRA: \definecolor{swh-green}{RGB}{77,181,174} * Vault :PROPERTIES: :CUSTOM_ID: main :END: ** Bulk download *TODO* ** Vault walkthrough *** Software identifier to request download #+BEAMER: \footnotesize -The swh-id *swh:1:rev:608757ea9bd8494d729732cc9a414948c160bd3c* +The swh-id *swh:1:rev:a86747d201ab8f8657d145df4376676d5e47cf9f* is composed of: - the object type *swh:1:rev* - and the sha1 hash as the object identifier \\ - *608757ea9bd8494d729732cc9a414948c160bd3c* + *a86747d201ab8f8657d145df4376676d5e47cf9f* We will use the object identifier to create a bundle to download #+BEAMER: \pause *** Requesting download with swh-id #+BEAMER: \tiny #+BEGIN_SRC sh -$ curl -X POST /api/1/vault/revision/608757ea.../gitfast +$ curl -X POST /api/1/vault/revision/a86747d2.../gitfast #+END_SRC #+BEAMER: \pause *** Email notification #+BEAMER: \footnotesize optionally, an email POST parameter containing an e-mail to notify when the bundle cooking has ended. ** Vault walkthrough *** Checking progress #+BEAMER: \tiny #+BEGIN_SRC sh -$ curl /api/1/vault/revision/608757ea.../gitfast +$ curl /api/1/vault/revision/a86747d2.../gitfast #+END_SRC # can we cook objects that aren't revisions? #+BEAMER: \pause *** Response #+BEAMER: \tiny #+BEGIN_SRC json { - 'fetch_url': '/api/1/vault/revision/608757ea.../gitfast/raw/', + 'fetch_url': '/api/1/vault/revision/a86747d2.../gitfast/raw/', 'progress_message': None, 'status': 'pending', 'id': 4, - 'obj_id': '608757ea9bd8494d729732cc9a414948c160bd3c', + 'obj_id': 'a86747d201ab8f8657d145df4376676d5e47cf9f', 'obj_type': 'revision_gitfast' } #+END_SRC #+BEAMER: \pause *** What's your status? #+BEAMER: \small \tikzstyle{fail} = [draw, thin, fill=swh-red!80!swh-orange, minimum height=1.5em] \tikzstyle{processing} = [draw, thin, fill=swh-orange!100, minimum height=1.5em] \tikzstyle{success} = [draw, thin, fill=swh-green!80!swh-orange, minimum height=1.5em] \begin{figure} \begin{tikzpicture}[node distance=1.8cm, auto,>=latex', thick] % We need to set at bounding box first. Otherwise the diagram % will change position for each frame. %\path[use as bounding box] (-1,0) rectangle (10,-2); \path[->]<1-> node[processing] (new) {new} node[processing, right of=new] (pending) {pending} node[processing, right of=pending] (loading) {} node[success, right of=loading] (done) {done} node[fail, below of=loading] (failed) {failed} (new) edge node {} (pending) (pending) edge node {} (loading) (loading) edge node {} (failed) (loading) edge node {} (done); \end{tikzpicture} \end{figure} ** Vault walkthrough *** Checking progress #+BEAMER: \tiny #+BEGIN_SRC sh -$ curl /api/1/vault/revision/608757ea.../gitfast +$ curl /api/1/vault/revision/a86747d2.../gitfast #+END_SRC #+BEAMER: \pause *** Response #+BEAMER: \tiny #+BEGIN_SRC json { - 'fetch_url': '/api/1/vault/revision/608757ea.../gitfast/raw/', + 'fetch_url': '/api/1/vault/revision/a86747d2.../gitfast/raw/', 'progress_message': None, 'status': 'done', 'id': 4, - 'obj_id': '608757ea9bd8494d729732cc9a414948c160bd3c', + 'obj_id': 'a86747d201ab8f8657d145df4376676d5e47cf9f', 'obj_type': 'revision_gitfast' } #+END_SRC #+BEAMER: \pause *** Download available when status is marked /done/ #+BEAMER: \tiny #+BEGIN_SRC sh -$ curl /api/1/vault/revision/608757ea.../gitfast/raw/ \ +$ curl /api/1/vault/revision/a86747d2.../gitfast/raw/ \ -O path/to/revision.gitfast.gz $ git init $ zcat path/to/revision.gitfast.gz | git fast-import $ git checkout HEAD #+END_SRC