diff --git a/.gitignore b/.gitignore
index 0b3af49..ba7b989 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,12 @@
*.csv
+*.egg*
*.log
-*.txt
+*.zip
__pycache__
compact.egg.info
content-revision
+build
+debian
dumps
PKG-INFO
revision-origin
diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 0000000..c329086
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,3 @@
+Copyright (C) 2019 The Software Heritage developers
+
+See http://www.softwareheritage.org/ for more information.
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000..0ad22b5
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,78 @@
+# Software Heritage Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as Software
+Heritage contributors and maintainers pledge to making participation in our
+project and our community a harassment-free experience for everyone, regardless
+of age, body size, disability, ethnicity, sex characteristics, gender identity
+and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at `conduct@softwareheritage.org`. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an
+incident. Further details of specific enforcement policies may be posted
+separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
new file mode 100644
index 0000000..e69de29
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..94a9ed0
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,674 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users. We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors. You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+ Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+ For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+ Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so. This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software. The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable. Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products. If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+ Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary. To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Use with the GNU Affero General Public License.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+ Copyright (C)
+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+.
+
+ The GNU General Public License does not permit incorporating your program
+into proprietary programs. If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License. But first, please read
+.
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..807e2e9
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,5 @@
+include Makefile
+include requirements*.txt
+include version.txt
+include README.md
+recursive-include swh py.typed
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..524175c
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,5 @@
+# Makefile driver for SWH Python modules. DO NOT CHANGE.
+# You can add custom Makefile rules to Makefile.local
+
+include ../Makefile.python
+-include Makefile.local
diff --git a/README.md b/README.md
index 02ba374..08b97d8 100644
--- a/README.md
+++ b/README.md
@@ -1,98 +1,5 @@
-Provenance compact model
-========================
+swh-py-template
+===============
-Prototype for the provenance compact model database.
+Python module template, used as skeleton to create new modules.
-Setup
------
-
-The solution works within the SWH environment, using its API to retrieve
-information from the archive. It assumes the SHW modules are installed and
-accessible, as well as a VPN connection to query the archive directly.
-
-
-Usage
------
-
-The solution consists of a simple command-line utility to query the
-compact model database and, eventually, reconstructing it.
-
-To simply query an existing compact model database, the utility just requires
-the connection credentials to it to be provided on a separate configuration
-file, and the amount of content blobs to be retrieved from the database (it
-retrieves them ordered by SWHID). The configuration content sections like:
-
-```
-[db_name]
-host=localhost
-database=db_name
-user=postgres
-password=postgres
-```
-
-Then the command-line utility can simply be invoked like this:
-
-```
-(swh) user@host:~/compact -c conf_file db_name count
-```
-
-where `confile` is the path to the configuration file, `db_name` is the name of
-the section containing the connection information (it should match the database
-actual name), and `count` is the amount of content blobs to be retrieved.
-
-
-For reconstructing the database it also requires a source of revisions to
-iterate. There are two option for specifying it: (1) giving a connection to the
-archive that will be queried using the provided API, (2) giving a local comma
-separated values file where each row consists of the revision's SWHID, its
-timestamp and its root directory SWHID.
-
-In the first case the command-line utility can simply be invoked with the `-d`
-option followed by the configuration file for the connection to the archive and
-the database name just as for the `-c` option:
-
-```
-(swh) user@host:~/compact -c conf_file db_name -d conf_file db_name count
-```
-
-In the second case the option `-f` should be used, followed by the path to the
-CSV file to be read.
-
-```
-(swh) user@host:~/compact -c conf_file db_name -f csv_file count
-```
-
-Both alternatives support the `-l` option, to limit the amount of revision to
-be considered for reconstruction the compact model database.
-
-WARNING: when reconstructing the database any previous content of the provided
-compact model database is erased!
-
-
-Algorithm
----------
-
-The algorithm for constructing the compact model database is based on that
-described in [1]. It iterates over a list of revisions assuming chronological
-order and building and isochrone graph for each element in the list. Every time
-a directory is found in the isochrone frontier of the current revision, the
-exploration is aborted given that all the elements is that branch are already
-known. Whenever a revision comes out-of-order, elements already known from a
-previously computed isochrone graph are treated as unknown.
-
-
-Structure of the code
----------------------
-
-The previously described algorithm is implemented in `compact.py`. It is
-actually done as a recursive walk over the directories of a given revision.
-The module `iterator.py` contains utilities to properly iterate over revisions
-coming both from a remote archive database or from a local CSV file. The module
-`model.py` implements an iterator for directory entries that retrieves only the
-necessary information from the archive and prevents multiple queries over the
-same directory.
-
-
-[1] Guillaume Rousseau, Roberto Di Cosmo, Stefano Zacchiroli. Software
-Provenance Tracking at the Scale of Public Source Code. In Empirical Software
-Engineering, volume 25, issue 4, pp. 2930-2959. ISSN 1382-3256, Springer. 2020.
diff --git a/database.conf b/database.conf
deleted file mode 100644
index 666e1d2..0000000
--- a/database.conf
+++ /dev/null
@@ -1,29 +0,0 @@
-[archive]
-database=softwareheritage
-host=somerset.internal.softwareheritage.org
-user=guest
-port=5433
-
-[compact]
-host=localhost
-database=compact
-user=postgres
-password=postgres
-
-[ordered]
-host=localhost
-database=ordered
-user=postgres
-password=postgres
-
-[reverse]
-host=localhost
-database=reverse
-user=postgres
-password=postgres
-
-[random]
-host=localhost
-database=random
-user=postgres
-password=postgres
diff --git a/docs/.gitignore b/docs/.gitignore
new file mode 100644
index 0000000..58a761e
--- /dev/null
+++ b/docs/.gitignore
@@ -0,0 +1,3 @@
+_build/
+apidoc/
+*-stamp
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..c30c50a
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1 @@
+include ../../swh-docs/Makefile.sphinx
diff --git a/docs/_static/.placeholder b/docs/_static/.placeholder
new file mode 100644
index 0000000..e69de29
diff --git a/docs/_templates/.placeholder b/docs/_templates/.placeholder
new file mode 100644
index 0000000..e69de29
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000..190deb7
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1 @@
+from swh.docs.sphinx.conf import * # NoQA
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000..b04a56d
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,19 @@
+.. _swh-py-template:
+
+Software Heritage - Python module template
+==========================================
+
+Python module template, used as skeleton to create new modules.
+
+
+.. toctree::
+ :maxdepth: 2
+ :caption: Contents:
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/draw-content.py b/draw-content.py
deleted file mode 100644
index 561ee5b..0000000
--- a/draw-content.py
+++ /dev/null
@@ -1,87 +0,0 @@
-import io
-import logging
-import os
-import utils
-
-from graphviz import Digraph
-from pathlib import PosixPath
-from swh.model.identifiers import identifier_to_str
-
-
-if __name__ == "__main__":
- """Compact model origin-revision layer utility."""
- # logging.basicConfig(level=logging.DEBUG)
- logging.basicConfig(filename='draw.log', level=logging.DEBUG)
-
- comp_conn = utils.connect('database.conf', 'compact')
-
- cursor = comp_conn.cursor()
- cursor.execute('''SELECT id FROM revision''')
- for idx in range(1000):
- rev = cursor.fetchone()[0]
-
- revid = identifier_to_str(rev)
- dot = Digraph(comment=f'Content of revision {revid}')
- dot.node(revid, f'REVISION\n{revid}')
-
- blobs = dict()
- dirs = dict()
- revcur = comp_conn.cursor()
-
- ########################################################################
- revcur.execute('''SELECT blob, path FROM content_early_in_rev WHERE rev=%s''',
- (rev,))
- for blob in revcur.fetchall():
- blobid = identifier_to_str(blob[0])
- blobpath = PosixPath(os.fsdecode(blob[1]))
- if blobid in blobs.keys():
- blobs[blobid][0].append(revid)
- blobs[blobid][1].append(blobpath)
- else:
- blobs[blobid] = ([revid], [blobpath])
-
-
- ########################################################################
- revcur.execute('''SELECT dir, path FROM directory_in_rev WHERE rev=%s''',
- (rev,))
- for dir in revcur.fetchall():
- dirid = identifier_to_str(dir[0])
- dirpath = PosixPath(os.fsdecode(dir[1]))
- if dirid in dirs.keys():
- dirs[dirid][0].append(revid)
- dirs[dirid][1].append(dirpath)
- else:
- dirs[dirid] = ([revid], [dirpath])
-
- dircur = comp_conn.cursor()
- dircur.execute('''SELECT blob, path FROM content_in_dir WHERE dir=%s''',
- (dir[0],))
- for blob in dircur.fetchall():
- blobid = identifier_to_str(blob[0])
- blobpath = PosixPath(os.fsdecode(blob[1]))
- if blobid in blobs.keys():
- blobs[blobid][0].append(dirid)
- blobs[blobid][1].append(dirpath / blobpath)
- else:
- blobs[blobid] = ([dirid], [dirpath / blobpath])
-
- ########################################################################
- for dirid, dirdata in dirs.items():
- dirpaths = "\n".join(map(str, dirdata[1]))
- dot.node(dirid, f'DIRECTORY\n{dirid}\n{dirpaths}')
- for parent in dirdata[0]:
- dot.edge(parent, dirid)
-
- for blobid, blobdata in blobs.items():
- blobpaths = "\n".join(map(str, blobdata[1]))
- dot.node(blobid, f'CONTENT\n{blobid}\n{blobpaths}')
- for parent in blobdata[0]:
- dot.edge(parent, blobid)
-
- with io.open(f'content-revision/{revid}.graph', 'w') as outfile:
- outfile.write(dot.source)
-
- dot.render(f'{revid}.gv', 'content-revision')
- os.remove(f'content-revision/{revid}.gv')
-
- comp_conn.close()
diff --git a/draw-origin.py b/draw-origin.py
deleted file mode 100644
index 0472fff..0000000
--- a/draw-origin.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import io
-import logging
-import os
-import utils
-
-from graphviz import Digraph
-from pathlib import PosixPath
-from swh.model.identifiers import identifier_to_str
-
-
-if __name__ == "__main__":
- """Compact model origin-revision layer utility."""
- # logging.basicConfig(level=logging.DEBUG)
- logging.basicConfig(filename='draw.log', level=logging.DEBUG)
-
- comp_conn = utils.connect('database.conf', 'compact')
-
- cursor = comp_conn.cursor()
- cursor.execute('''SELECT id, url FROM origin''')
- for idx in range(1):
- org = cursor.fetchone()
-
- orgid = str(org[0])
- orgurl = str(org[1])
- dot = Digraph(comment=f'Revision history of origin {orgurl}')
- dot.node(orgid, f'ORIGIN ({orgid})\n{orgurl}')
-
- revs = dict()
- orgcur = comp_conn.cursor()
-
- ########################################################################
- orgcur.execute('''SELECT rev FROM revision_in_org WHERE org=%s''',
- (orgid,))
- for rev in orgcur.fetchall():
- revid = identifier_to_str(rev[0])
- if revid in revs.keys():
- revs[revid].append(orgid)
- else:
- revs[revid] = ([orgid])
-
-
- revcur = comp_conn.cursor()
- revcur.execute('''SELECT prev FROM revision_before_rev WHERE next=%s''',
- (rev[0],))
- for prev in revcur.fetchall():
- previd = identifier_to_str(prev[0])
- if previd in revs.keys():
- revs[previd].append(revid)
- else:
- revs[previd] = ([revid])
-
- ########################################################################
- for revid, nexts in revs.items():
- dot.node(revid, f'REVISION\n{revid}')
- for next in nexts:
- dot.edge(revid, next)
-
- with io.open(f'revision-origin/{orgid}.graph', 'w') as outfile:
- outfile.write(dot.source)
-
- dot.render(f'{orgid}.gv', 'revision-origin', view=True)
- os.remove(f'revision-origin/{orgid}.gv')
-
- comp_conn.close()
diff --git a/mypy.ini b/mypy.ini
new file mode 100644
index 0000000..46f8db8
--- /dev/null
+++ b/mypy.ini
@@ -0,0 +1,15 @@
+[mypy]
+namespace_packages = True
+warn_unused_ignores = True
+
+
+# 3rd party libraries without stubs (yet)
+
+[mypy-pkg_resources.*]
+ignore_missing_imports = True
+
+[mypy-pytest.*]
+ignore_missing_imports = True
+
+# [mypy-add_your_lib_here.*]
+# ignore_missing_imports = True
diff --git a/origin.sql b/origin.sql
deleted file mode 100644
index 9214a19..0000000
--- a/origin.sql
+++ /dev/null
@@ -1,38 +0,0 @@
--- -- a Git object ID, i.e., a Git-style salted SHA1 checksum
--- drop domain if exists sha1_git cascade;
--- create domain sha1_git as bytea check (length(value) = 20);
-
-
-drop table if exists origin;
-create table origin
-(
- id bigserial primary key, -- id of the origin
- url unix_path unique -- url of the origin
-);
-
-comment on column origin.id is 'Origin internal identifier';
-comment on column origin.url is 'URL of the origin';
-
-
-drop table if exists revision_in_org;
-create table revision_in_org
-(
- rev sha1_git not null, -- id of the revision poined by the origin
- org bigint not null, -- id of the origin that points to the revision
- primary key (rev, org)
-);
-
-comment on column revision_in_org.rev is 'Revision identifier';
-comment on column revision_in_org.org is 'Origin identifier';
-
-
-drop table if exists revision_before_rev;
-create table revision_before_rev
-(
- prev sha1_git not null, -- id of the source revision
- next sha1_git not null, -- id of the destination revision
- primary key (prev, next)
-);
-
-comment on column revision_before_rev.prev is 'Source revision identifier';
-comment on column revision_before_rev.next is 'Destination revision identifier';
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..b5413f6
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,2 @@
+[tool.black]
+target-version = ['py37']
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..b712d00
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = docs .*
diff --git a/requirements-swh.txt b/requirements-swh.txt
new file mode 100644
index 0000000..24f0a5c
--- /dev/null
+++ b/requirements-swh.txt
@@ -0,0 +1,2 @@
+# Add here internal Software Heritage dependencies, one per line.
+swh.core
diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644
index 0000000..e079f8a
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1 @@
+pytest
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..54ce666
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,4 @@
+# Add here external Python modules dependencies, one per line. Module names
+# should match https://pypi.python.org/pypi names. For the full spec or
+# dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html
+
diff --git a/revisions.py b/revisions.py
deleted file mode 100644
index 2095682..0000000
--- a/revisions.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import io
-import logging
-import random
-import utils
-import sys
-
-from iterator import (
- RevisionEntry,
- ArchiveRevisionIterator
-)
-
-from swh.model.identifiers import identifier_to_str
-
-
-def rev_to_csv(revision: RevisionEntry):
- return ','.join([
- identifier_to_str(revision.swhid),
- str(revision.timestamp),
- identifier_to_str(revision.directory)
- ]) + '\n'
-
-
-if __name__ == "__main__":
- logging.basicConfig(level=logging.INFO)
-
- if len(sys.argv) != 4:
- print('Usage: revisions CONF_FILE DATABASE COUNT')
-
- database = (sys.argv[1], sys.argv[2])
- count = int(sys.argv[3])
-
- print(f'{count} {database}')
- data_conn = utils.connect(database[0], database[1])
- revisions = list(ArchiveRevisionIterator(data_conn, limit=count))
-
- random.shuffle(revisions)
- with io.open('random.csv', 'w') as outfile:
- for rev in revisions:
- outfile.write(rev_to_csv(rev))
-
- with io.open('ordered.csv', 'w') as outfile:
- for rev in sorted(revisions, key=lambda rev: rev.timestamp):
- outfile.write(rev_to_csv(rev))
-
- with io.open('reverse.csv', 'w') as outfile:
- for rev in sorted(revisions, key=lambda rev: rev.timestamp, reverse=True):
- outfile.write(rev_to_csv(rev))
-
- data_conn.close()
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..8d79b7e
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,6 @@
+[flake8]
+# E203: whitespaces before ':'
+# E231: missing whitespace after ','
+# W503: line break before binary operator
+ignore = E203,E231,W503
+max-line-length = 88
diff --git a/setup.py b/setup.py
old mode 100644
new mode 100755
index fc90cd5..9718259
--- a/setup.py
+++ b/setup.py
@@ -1,14 +1,74 @@
-from setuptools import setup
+#!/usr/bin/env python3
+# Copyright (C) 2019-2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+from setuptools import setup, find_packages
+
+from os import path
+from io import open
+
+here = path.abspath(path.dirname(__file__))
+
+# Get the long description from the README file
+with open(path.join(here, "README.md"), encoding="utf-8") as f:
+ long_description = f.read()
+
+
+def parse_requirements(name=None):
+ if name:
+ reqf = "requirements-%s.txt" % name
+ else:
+ reqf = "requirements.txt"
+
+ requirements = []
+ if not path.exists(reqf):
+ return requirements
+
+ with open(reqf) as f:
+ for line in f.readlines():
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ requirements.append(line)
+ return requirements
+
+
+# Edit this part to match your module.
+# Full sample:
+# https://forge.softwareheritage.org/diffusion/DCORE/browse/master/setup.py
setup(
- name='compact',
- version='0.1',
- py_modules=['compact'],
- install_requires=[
- 'Click',
+ name="swh.provenance",
+ description="Software Heritage code provenance",
+ long_description=long_description,
+ long_description_content_type="text/markdown",
+ python_requires=">=3.7",
+ author="Software Heritage developers",
+ author_email="swh-devel@inria.fr",
+ url="https://forge.softwareheritage.org/diffusion/222/",
+ packages=find_packages(), # packages's modules
+ install_requires=parse_requirements() + parse_requirements("swh"),
+ tests_require=parse_requirements("test"),
+ setup_requires=["setuptools-scm"],
+ use_scm_version=True,
+ extras_require={"testing": parse_requirements("test")},
+ include_package_data=True,
+ entry_points="""
+ [swh.cli.subcommands]
+ provenance=swh.provenance.cli
+ """,
+ classifiers=[
+ "Programming Language :: Python :: 3",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+ "Operating System :: OS Independent",
+ "Development Status :: 3 - Alpha",
],
- entry_points='''
- [console_scripts]
- compact=compact:cli
- ''',
+ project_urls={
+ "Bug Reports": "https://forge.softwareheritage.org/maniphest",
+ "Funding": "https://www.softwareheritage.org/donate",
+ "Source": "https://forge.softwareheritage.org/source/swh-provenance",
+ "Documentation": "https://docs.softwareheritage.org/devel/swh-provenance/",
+ },
)
diff --git a/swh/__init__.py b/swh/__init__.py
new file mode 100644
index 0000000..e780381
--- /dev/null
+++ b/swh/__init__.py
@@ -0,0 +1,4 @@
+from pkgutil import extend_path
+from typing import Iterable
+
+__path__: Iterable[str] = extend_path(__path__, __name__)
diff --git a/swh/provenance/__init__.py b/swh/provenance/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/swh/provenance/cli.py b/swh/provenance/cli.py
new file mode 100644
index 0000000..42e5de9
--- /dev/null
+++ b/swh/provenance/cli.py
@@ -0,0 +1,154 @@
+# Copyright (C) 2020 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+# WARNING: do not import unnecessary things here to keep cli startup time under
+# control
+import os
+from typing import Any, Dict, Optional
+
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+
+import click
+import yaml
+
+from swh.core import config
+from swh.core.cli import CONTEXT_SETTINGS
+from swh.core.cli import swh as swh_cli_group
+from swh.core.db import db_utils # TODO: remove this in favour of local db_utils module
+from swh.model.hashutil import (hash_to_bytes, hash_to_hex)
+from swh.storage import get_storage
+
+# All generic config code should reside in swh.core.config
+CONFIG_ENVVAR = "SWH_CONFIG_FILE"
+DEFAULT_CONFIG_PATH = os.path.join(click.get_app_dir("swh"), "global.yml")
+DEFAULT_PATH = os.environ.get(CONFIG_ENVVAR, DEFAULT_CONFIG_PATH)
+
+DEFAULT_CONFIG: Dict[str, Any] = {
+ "storage": {
+ "cls": "remote",
+ "url": "http://uffizi.internal.softwareheritage.org:5002"
+ },
+ "db": "postgresql://postgres:postgres@localhost/provenance"
+}
+
+
+CONFIG_FILE_HELP = f"""Configuration file:
+
+\b
+The CLI option or the environment variable will fail if invalid.
+CLI option is checked first.
+Then, environment variable {CONFIG_ENVVAR} is checked.
+Then, if cannot load the default path, a set of default values are used.
+Default config path is {DEFAULT_CONFIG_PATH}.
+Default config values are:
+
+\b
+{yaml.dump(DEFAULT_CONFIG)}"""
+PROVENANCE_HELP = f"""Software Heritage Scanner tools.
+
+{CONFIG_FILE_HELP}"""
+
+
+@swh_cli_group.group(
+ name="provenance", context_settings=CONTEXT_SETTINGS, help=PROVENANCE_HELP,
+)
+@click.option(
+ "-C",
+ "--config-file",
+ default=None,
+ type=click.Path(exists=False, dir_okay=False, path_type=str),
+ help="""YAML configuration file""",
+)
+@click.pass_context
+def cli(ctx, config_file: Optional[str]):
+ from .db_utils import adapt_conn
+
+ if config_file is None and config.config_exists(DEFAULT_PATH):
+ config_file = DEFAULT_PATH
+
+ if config_file is None:
+ conf = DEFAULT_CONFIG
+ else:
+ # read_raw_config do not fail on ENOENT
+ if not config.config_exists(config_file):
+ raise FileNotFoundError(config_file)
+ conf = config.read_raw_config(config.config_basepath(config_file))
+ conf = config.merge_configs(DEFAULT_CONFIG, conf)
+
+ ctx.ensure_object(dict)
+ ctx.obj["config"] = conf
+
+ conn = db_utils.connect_to_conninfo(conf["db"])
+ adapt_conn(conn)
+ ctx.obj["conn"] = conn
+
+
+@cli.command(name="create")
+@click.option("--name", default='provenance')
+@click.pass_context
+def create(ctx, name):
+ """Create new provenance database."""
+ from .provenance import create_database
+ from .db_utils import adapt_conn
+
+ # Close default connection as it won't be used
+ ctx.obj["conn"].close()
+
+ # Connect to server without selecting a database
+ conninfo = os.path.dirname(ctx.obj["config"]["db"])
+ conn = db_utils.connect_to_conninfo(conninfo)
+ adapt_conn(conn)
+
+ create_database(conn, conninfo, name)
+
+
+@cli.command(name="iter-revisions")
+@click.argument("filename")
+@click.option('-l', '--limit', type=int)
+@click.option('-t', '--threads', type=int, default=1)
+@click.pass_context
+def iter_revisions(ctx, filename, limit, threads):
+ """Iterate over provided list of revisions and add them to the provenance database."""
+ from .provenance import FileRevisionIterator
+ from .provenance import RevisionWorker
+
+ conninfo = ctx.obj["config"]["db"]
+ revisions = FileRevisionIterator(filename, limit=limit)
+ storage = get_storage(**ctx.obj["config"]["storage"])
+ workers = []
+
+ for id in range(threads):
+ worker = RevisionWorker(id, conninfo, storage, revisions)
+ worker.start()
+ workers.append(worker)
+
+ for worker in workers:
+ worker.join()
+
+
+@cli.command(name="find-first")
+@click.argument("swhid")
+@click.pass_context
+def find_first(ctx, swhid):
+ """Find first occurrence of the requested blob."""
+ from .provenance import content_find_first
+
+ conn = ctx.obj["conn"]
+ cursor = conn.cursor();
+ row = content_find_first(cursor, hash_to_bytes(swhid))
+ print(f'{hash_to_hex(row[0])}, {hash_to_hex(row[1])}, {row[2]}, {os.fsdecode(row[3])}')
+
+
+@cli.command(name="find-all")
+@click.argument("swhid")
+@click.pass_context
+def find_all(ctx, swhid):
+ """Find all occurrences of the requested blob."""
+ from .provenance import content_find_all
+
+ conn = ctx.obj["conn"]
+ cursor = conn.cursor();
+ for row in content_find_all(cursor, hash_to_bytes(swhid)):
+ print(f'{hash_to_hex(row[0])}, {hash_to_hex(row[1])}, {row[2]}, {os.fsdecode(row[3])}')
diff --git a/compact.sql b/swh/provenance/db/provenance.sql
similarity index 76%
rename from compact.sql
rename to swh/provenance/db/provenance.sql
index 2b4afd8..0002d91 100644
--- a/compact.sql
+++ b/swh/provenance/db/provenance.sql
@@ -1,90 +1,124 @@
-- a Git object ID, i.e., a Git-style salted SHA1 checksum
drop domain if exists sha1_git cascade;
create domain sha1_git as bytea check (length(value) = 20);
-- UNIX path (absolute, relative, individual path component, etc.)
drop domain if exists unix_path cascade;
create domain unix_path as bytea;
drop table if exists content;
create table content
(
id sha1_git primary key, -- id of the content blob
date timestamptz not null -- timestamp of the revision where the blob appears early
);
comment on column content.id is 'Content identifier';
comment on column content.date is 'Earliest timestamp for the content (first seen time)';
-drop table if exists directory;
-create table directory
-(
- id sha1_git primary key, -- id of the directory appearing in an isochrone inner frontier
- date timestamptz not null -- max timestamp among those of the directory children's
-);
-
-comment on column directory.id is 'Directory identifier';
-comment on column directory.date is 'Latest timestamp for the content in the directory';
-
-
-drop table if exists revision;
-create table revision
-(
- id sha1_git primary key, -- id of the revision
- date timestamptz not null, -- timestamp of the revision
- org bigint -- id of the prefered origin
-);
-
-comment on column revision.id is 'Revision identifier';
-comment on column revision.date is 'Revision timestamp';
-comment on column revision.org is 'Prefered origin for the revision';
-
-
drop table if exists content_early_in_rev;
create table content_early_in_rev
(
blob sha1_git not null, -- id of the content blob
rev sha1_git not null, -- id of the revision where the blob appears for the first time
path unix_path not null, -- path to the content relative to the revision root directory
primary key (blob, rev, path)
-- foreign key (blob) references content (id),
-- foreign key (rev) references revision (id)
);
comment on column content_early_in_rev.blob is 'Content identifier';
comment on column content_early_in_rev.rev is 'Revision identifier';
comment on column content_early_in_rev.path is 'Path to content in revision';
drop table if exists content_in_dir;
create table content_in_dir
(
blob sha1_git not null, -- id of the content blob
dir sha1_git not null, -- id of the directory contaning the blob
path unix_path not null, -- path name relative to its parent on the isochrone frontier
primary key (blob, dir, path)
-- foreign key (blob) references content (id),
-- foreign key (dir) references directory (id)
);
comment on column content_in_dir.blob is 'Content identifier';
comment on column content_in_dir.dir is 'Directory identifier';
-- comment on column content_early_in_rev.path is 'Path to content in directory';
+drop table if exists directory;
+create table directory
+(
+ id sha1_git primary key, -- id of the directory appearing in an isochrone inner frontier
+ date timestamptz not null -- max timestamp among those of the directory children's
+);
+
+comment on column directory.id is 'Directory identifier';
+comment on column directory.date is 'Latest timestamp for the content in the directory';
+
+
drop table if exists directory_in_rev;
create table directory_in_rev
(
dir sha1_git not null, -- id of the directory appearing in the revision
rev sha1_git not null, -- id of the revision containing the directory
path unix_path not null, -- path to the directory relative to the revision root directory
primary key (dir, rev, path)
-- foreign key (dir) references directory (id),
-- foreign key (rev) references revision (id)
);
comment on column directory_in_rev.dir is 'Directory identifier';
comment on column directory_in_rev.rev is 'Revision identifier';
comment on column directory_in_rev.path is 'Path to directory in revision';
+
+drop table if exists origin;
+create table origin
+(
+ id bigserial primary key, -- id of the origin
+ url unix_path unique -- url of the origin
+);
+
+comment on column origin.id is 'Origin internal identifier';
+comment on column origin.url is 'URL of the origin';
+
+
+drop table if exists revision;
+create table revision
+(
+ id sha1_git primary key, -- id of the revision
+ date timestamptz not null, -- timestamp of the revision
+ org bigint -- id of the prefered origin
+);
+
+comment on column revision.id is 'Revision identifier';
+comment on column revision.date is 'Revision timestamp';
+comment on column revision.org is 'Prefered origin for the revision';
+
+
+drop table if exists revision_before_rev;
+create table revision_before_rev
+(
+ prev sha1_git not null, -- id of the source revision
+ next sha1_git not null, -- id of the destination revision
+ primary key (prev, next)
+);
+
+comment on column revision_before_rev.prev is 'Source revision identifier';
+comment on column revision_before_rev.next is 'Destination revision identifier';
+
+
+drop table if exists revision_in_org;
+create table revision_in_org
+(
+ rev sha1_git not null, -- id of the revision poined by the origin
+ org bigint not null, -- id of the origin that points to the revision
+ primary key (rev, org)
+);
+
+comment on column revision_in_org.rev is 'Revision identifier';
+comment on column revision_in_org.org is 'Origin identifier';
diff --git a/utils.py b/swh/provenance/db_utils.py
similarity index 95%
rename from utils.py
rename to swh/provenance/db_utils.py
index 2ffa335..2be0ece 100644
--- a/utils.py
+++ b/swh/provenance/db_utils.py
@@ -1,66 +1,66 @@
import io
import psycopg2
from configparser import ConfigParser
from pathlib import PosixPath
def config(filename: PosixPath, section: str):
# create a parser
parser = ConfigParser()
# read config file
parser.read(filename)
# get section, default to postgresql
db = {}
if parser.has_section(section):
params = parser.items(section)
for param in params:
db[param[0]] = param[1]
else:
raise Exception(f'Section {section} not found in the {filename} file')
return db
def typecast_bytea(value, cur):
if value is not None:
data = psycopg2.BINARY(value, cur)
return data.tobytes()
def adapt_conn(conn):
"""Makes psycopg2 use 'bytes' to decode bytea instead of
'memoryview', for this connection."""
t_bytes = psycopg2.extensions.new_type((17,), "bytea", typecast_bytea)
psycopg2.extensions.register_type(t_bytes, conn)
t_bytes_array = psycopg2.extensions.new_array_type((1001,), "bytea[]", t_bytes)
psycopg2.extensions.register_type(t_bytes_array, conn)
def connect(filename: PosixPath, section: str):
""" Connect to the PostgreSQL database server """
conn = None
try:
# read connection parameters
params = config(filename, section)
# connect to the PostgreSQL server
# print('Connecting to the PostgreSQL database...')
conn = psycopg2.connect(**params)
adapt_conn(conn)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
return conn
-def execute_sql(conn: psycopg2.extensions.cursor, filename: PosixPath):
+def execute_sql(conn: psycopg2.extensions.connection, filename: PosixPath):
with io.open(filename) as file:
cur = conn.cursor()
cur.execute(file.read())
cur.close()
conn.commit()
diff --git a/model.py b/swh/provenance/model.py
similarity index 100%
rename from model.py
rename to swh/provenance/model.py
diff --git a/origin.py b/swh/provenance/origin.py
similarity index 98%
rename from origin.py
rename to swh/provenance/origin.py
index a26ece2..e4066f2 100644
--- a/origin.py
+++ b/swh/provenance/origin.py
@@ -1,248 +1,248 @@
import logging
import psycopg2
import utils
import swh.storage
import swh.storage.algos.origin
import swh.storage.algos.snapshot
import swh.storage.interface
from swh.model.identifiers import identifier_to_str
class OriginEntry:
def __init__(self, url, revisions, id=None):
self.id = id
self.url = url
self.revisions = revisions
def __str__(self):
# return f'{type(self).__name__}(id={self.id}, url={self.url}, revisions={list(map(str, self.revisions))})'
return f'{type(self).__name__}(id={self.id}, url={self.url})'
class RevisionEntry:
def __init__(self, swhid, parents):
self.swhid = swhid
self.parents = list(dict.fromkeys(parents))
def __str__(self):
return f'{type(self).__name__}(swhid={identifier_to_str(self.swhid)}, parents={list(map(identifier_to_str, self.parents))})'
class OriginIterator:
def __init__(self, storage: swh.storage.interface.StorageInterface):
self.storage = storage
def __iter__(self):
yield from self.iterate()
def iterate(self):
idx = 0
for origin in swh.storage.algos.origin.iter_origins(self.storage):
# print(f'{idx:03} -> {origin}')
origin = origin.to_dict()
for visit in swh.storage.algos.origin.iter_origin_visits(self.storage, origin['url']):
# print(f' +--> {visit}')
visit = visit.to_dict()
if 'visit' in visit:
for status in swh.storage.algos.origin.iter_origin_visit_statuses(self.storage, origin['url'], visit['visit']):
# print(f' +--> {status}')
# TODO: may filter only those whose status is 'full'??
status = status.to_dict()
targets = []
releases = []
snapshot = swh.storage.algos.snapshot.snapshot_get_all_branches(storage, status['snapshot'])
if snapshot is not None:
branches = snapshot.to_dict()['branches']
for branch in branches:
# print(f' +--> {branch} : {branches[branch]}')
target = branches[branch]['target']
target_type = branches[branch]['target_type']
if target_type == 'revision':
targets.append(target)
elif target_type == 'release':
releases.append(target)
# print(f' ############################################################')
# print(list(map(identifier_to_str, releases)))
# print(f' ### RELEASES ###############################################')
# This is done to keep the query in release_get small, hence avoiding a timeout.
limit = 100
for i in range(0, len(releases), limit):
for release in storage.release_get(releases[i:i+limit]):
if release is not None:
# print(f'** {release}')
release = release.to_dict()
target = release['target']
target_type = release['target_type']
if target_type == 'revision':
targets.append(target)
# print(f' ############################################################')
# print(list(map(identifier_to_str, targets)))
# print(f' ### REVISIONS ##############################################')
# This is done to keep the query in revision_get small, hence avoiding a timeout.
revisions = []
limit = 100
for i in range(0, len(targets), limit):
for revision in storage.revision_get(targets[i:i+limit]):
if revision is not None:
# print(f'** {revision}')
revision = revision.to_dict()
revisions.append(RevisionEntry(revision['id'], revision['parents']))
yield OriginEntry(status['origin'], revisions)
idx = idx + 1
if idx == 100: return
def origin_add_revision(
cursor: psycopg2.extensions.cursor,
origin: OriginEntry,
revision: RevisionEntry
):
stack = [(origin, None, revision)]
while stack:
origin, relative, revision = stack.pop()
# Check if current revision has no prefered origin and update if necessary.
cursor.execute('''SELECT COALESCE(org, 0) FROM revision WHERE id=%s''',
(revision.swhid,))
row = cursor.fetchone()
prefered = row[0] if row is not None else None
print(f'Prefered origin for revision {identifier_to_str(revision.swhid)}: {prefered}')
if prefered == 0:
cursor.execute('''UPDATE TABLE revision SET org=%s WHERE id=%s''',
(origin.id, revision.swhid))
########################################################################
if relative is None:
# This revision is pointed directly by the origin.
logging.debug(f'Adding revision {identifier_to_str(revision.swhid)} to origin {origin.id}')
cursor.execute('''SELECT 1 FROM revision_in_org WHERE rev=%s''',
(revision.swhid,))
visited = cursor.fetchone() is not None
print(f'Revision {identifier_to_str(revision.swhid)} in origin {origin.id}: {visited}')
cursor.execute('''INSERT INTO revision_in_org VALUES (%s, %s)
ON CONFLICT DO NOTHING''',
(revision.swhid, origin.id))
if not visited:
# revision_walk_history(cursor, origin, revision.swhid, revision)
stack.append((origin, revision.swhid, revision))
else:
# This revision is a parent of another one in the history of the
# relative revision.
to_org = []
to_rev = []
for parent in revision.parents:
cursor.execute('''SELECT 1 FROM revision_in_org WHERE rev=%s''',
(parent,))
visited = cursor.fetchone() is not None
print(f'Parent {identifier_to_str(parent)} in some origin: {visited}')
if not visited:
# The parent revision has never been seen before pointing
# directly to an origin.
cursor.execute('''SELECT 1 FROM revision_before_rev WHERE prev=%s''',
(parent,))
known = cursor.fetchone() is not None
print(f'Revision {identifier_to_str(parent)} before revision: {visited}')
if known:
# The parent revision is already known in some other
# revision's history. We should point it directly to
# the origin and (eventually) walk its history.
to_org.append(parent)
else:
# The parent revision was never seen before. We should
# walk its history and associate it with the same
# relative revision.
to_rev.append(parent)
else:
# The parent revision already points to an origin, so its
# history was properly processed before. We just need to
# make sure it points to the current origin as well.
logging.debug(f'Adding parent revision {identifier_to_str(parent)} to origin {origin.id}')
cursor.execute('''INSERT INTO revision_in_org VALUES (%s,%s)
ON CONFLICT DO NOTHING''',
(parent, origin.id))
for parent in storage.revision_get(to_org):
if parent is not None:
parent = parent.to_dict()
parent = RevisionEntry(parent['id'], parent['parents'])
# origin_add_revision(cursor, origin, parent)
stack.append((origin, None, parent))
for parent in storage.revision_get(to_rev):
if parent is not None:
parent = parent.to_dict()
parent = RevisionEntry(parent['id'], parent['parents'])
logging.debug(f'Adding parent revision {identifier_to_str(parent.swhid)} to revision {identifier_to_str(relative)}')
cursor.execute('''INSERT INTO revision_before_rev VALUES (%s,%s)''',
(parent.swhid, relative))
# revision_walk_history(cursor, origin, relative, parent)
stack.append((origin, relative, parent))
if __name__ == "__main__":
"""Compact model origin-revision layer utility."""
# logging.basicConfig(level=logging.DEBUG)
logging.basicConfig(filename='origin.log', level=logging.DEBUG)
comp_conn = utils.connect('database.conf', 'compact')
cursor = comp_conn.cursor()
utils.execute_sql(comp_conn, 'origin.sql') # Create tables dopping existing ones
kwargs = {
"cls" : "remote",
"url" : "http://uffizi.internal.softwareheritage.org:5002"
}
storage = swh.storage.get_storage(**kwargs)
for origin in OriginIterator(storage):
print(f'* {origin}')
# Check if current origin is already known and retrieve its internal id.
cursor.execute('''SELECT id FROM origin WHERE url=%s''', (origin.url,))
row = cursor.fetchone()
origin.id = row[0] if row is not None else None
for revision in origin.revisions:
print(f'** {revision}')
if origin.id is None:
# If the origin is seen for the first time, current revision is
# the prefered one.
- cursor.execute('''INSERT INTO origin (url) VALUES (%s)''',
+ cursor.execute('''INSERT INTO origin (url) VALUES (%s) RETURNING id''',
(origin.url,))
# Retrieve current origin's internal id (just generated).
- cursor.execute('''SELECT id FROM origin WHERE url=%s''',
- (origin.url,))
+ # cursor.execute('''SELECT id FROM origin WHERE url=%s''',
+ # (origin.url,))
origin.id = cursor.fetchone()[0]
origin_add_revision(cursor, origin, revision)
comp_conn.commit()
print(f'##################################################################')
comp_conn.close()
diff --git a/compact.py b/swh/provenance/provenance.py
similarity index 70%
rename from compact.py
rename to swh/provenance/provenance.py
index 01b6a75..599657c 100644
--- a/compact.py
+++ b/swh/provenance/provenance.py
@@ -1,391 +1,348 @@
-import click
import logging
+import os
import psycopg2
-import swh.storage
-import sys
import time
import threading
-import utils
-from datetime import datetime
-from pathlib import PosixPath
-
-from iterator import (
- RevisionEntry,
- RevisionIterator,
- ArchiveRevisionIterator,
- FileRevisionIterator
+from .db_utils import (
+ adapt_conn,
+ execute_sql
)
-from model import (
+from .model import (
DirectoryEntry,
FileEntry,
TreeEntry,
Tree
)
+from .revision import (
+ RevisionEntry,
+ RevisionIterator,
+ ArchiveRevisionIterator,
+ FileRevisionIterator
+)
+
+from datetime import datetime
+from pathlib import PosixPath
-from swh.model.identifiers import identifier_to_str
+from swh.core.db import db_utils # TODO: remove this in favour of local db_utils module
+from swh.model.hashutil import hash_to_hex
+from swh.storage.interface import StorageInterface
+
+
+def create_database(
+ conn: psycopg2.extensions.connection,
+ conninfo: str,
+ name: str
+):
+ conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+
+ # Create new database dropping previous one if exists
+ cursor = conn.cursor();
+ cursor.execute(f'''DROP DATABASE IF EXISTS {name}''')
+ cursor.execute(f'''CREATE DATABASE {name};''');
+ conn.close()
+
+ # Reconnect to server selecting newly created database to add tables
+ conn = db_utils.connect_to_conninfo(os.path.join(conninfo, name))
+ adapt_conn(conn)
+
+ dir = os.path.dirname(os.path.realpath(__file__))
+ execute_sql(conn, os.path.join(dir, 'db/provenance.sql'))
def revision_add(
cursor: psycopg2.extensions.cursor,
- storage: swh.storage.interface.StorageInterface,
+ storage: StorageInterface,
revision: RevisionEntry,
id : int
):
- logging.info(f'Thread {id} - Processing revision {identifier_to_str(revision.swhid)} (timestamp: {revision.timestamp})')
+ logging.info(f'Thread {id} - Processing revision {hash_to_hex(revision.swhid)} (timestamp: {revision.timestamp})')
# Processed content starting from the revision's root directory
directory = Tree(storage, revision.directory).root
revision_process_directory(cursor, revision, directory, directory.name)
# Add current revision to the compact DB
cursor.execute('INSERT INTO revision VALUES (%s,%s, NULL)', (revision.swhid, revision.timestamp))
def content_find_first(
cursor: psycopg2.extensions.cursor,
swhid: str
):
- logging.info(f'Retrieving first occurrence of content {identifier_to_str(swhid)}')
+ logging.info(f'Retrieving first occurrence of content {hash_to_hex(swhid)}')
cursor.execute('''SELECT blob, rev, date, path
FROM content_early_in_rev JOIN revision ON revision.id=content_early_in_rev.rev
WHERE content_early_in_rev.blob=%s ORDER BY date, rev, path ASC LIMIT 1''', (swhid,))
return cursor.fetchone()
def content_find_all(
cursor: psycopg2.extensions.cursor,
swhid: str
):
- logging.info(f'Retrieving all occurrences of content {identifier_to_str(swhid)}')
+ logging.info(f'Retrieving all occurrences of content {hash_to_hex(swhid)}')
cursor.execute('''(SELECT blob, rev, date, path
FROM content_early_in_rev JOIN revision ON revision.id=content_early_in_rev.rev
WHERE content_early_in_rev.blob=%s)
UNION
(SELECT content_in_rev.blob, content_in_rev.rev, revision.date, content_in_rev.path
FROM (SELECT content_in_dir.blob, directory_in_rev.rev,
CASE directory_in_rev.path
WHEN '.' THEN content_in_dir.path
ELSE (directory_in_rev.path || '/' || content_in_dir.path)::unix_path
END AS path
FROM content_in_dir
JOIN directory_in_rev ON content_in_dir.dir=directory_in_rev.dir
WHERE content_in_dir.blob=%s) AS content_in_rev
JOIN revision ON revision.id=content_in_rev.rev)
ORDER BY date, rev, path''', (swhid, swhid))
+ # POSTGRESQL EXPLAIN
yield from cursor.fetchall()
################################################################################
################################################################################
################################################################################
def normalize(path: PosixPath) -> PosixPath:
spath = str(path)
if spath.startswith('./'):
return PosixPath(spath[2:])
return path
def content_get_early_timestamp(
cursor: psycopg2.extensions.cursor,
blob: FileEntry
):
- logging.debug(f'Getting content {identifier_to_str(blob.swhid)} early timestamp')
+ logging.debug(f'Getting content {hash_to_hex(blob.swhid)} early timestamp')
start = time.perf_counter_ns()
cursor.execute('SELECT date FROM content WHERE id=%s', (blob.swhid,))
row = cursor.fetchone()
stop = time.perf_counter_ns()
logging.debug(f' Time elapsed: {stop-start}ns')
return row[0] if row is not None else None
def content_set_early_timestamp(
cursor: psycopg2.extensions.cursor,
blob: FileEntry,
timestamp: datetime
):
- logging.debug(f'EARLY occurrence of blob {identifier_to_str(blob.swhid)} (timestamp: {timestamp})')
+ logging.debug(f'EARLY occurrence of blob {hash_to_hex(blob.swhid)} (timestamp: {timestamp})')
start = time.perf_counter_ns()
cursor.execute('''INSERT INTO content VALUES (%s,%s)
ON CONFLICT (id) DO UPDATE SET date=%s''',
(blob.swhid, timestamp, timestamp))
stop = time.perf_counter_ns()
logging.debug(f' Time elapsed: {stop-start}ns')
def content_add_to_directory(
cursor: psycopg2.extensions.cursor,
directory: DirectoryEntry,
blob: FileEntry,
prefix: PosixPath
):
- logging.debug(f'NEW occurrence of content {identifier_to_str(blob.swhid)} in directory {identifier_to_str(directory.swhid)} (path: {prefix / blob.name})')
+ logging.debug(f'NEW occurrence of content {hash_to_hex(blob.swhid)} in directory {hash_to_hex(directory.swhid)} (path: {prefix / blob.name})')
start = time.perf_counter_ns()
cursor.execute('INSERT INTO content_in_dir VALUES (%s,%s,%s)',
(blob.swhid, directory.swhid, bytes(normalize(prefix / blob.name))))
stop = time.perf_counter_ns()
logging.debug(f' Time elapsed: {stop-start}ns')
def content_add_to_revision(
cursor: psycopg2.extensions.cursor,
revision: RevisionEntry,
blob: FileEntry,
prefix: PosixPath
):
- logging.debug(f'EARLY occurrence of blob {identifier_to_str(blob.swhid)} in revision {identifier_to_str(revision.swhid)} (path: {prefix / blob.name})')
+ logging.debug(f'EARLY occurrence of blob {hash_to_hex(blob.swhid)} in revision {hash_to_hex(revision.swhid)} (path: {prefix / blob.name})')
start = time.perf_counter_ns()
cursor.execute('INSERT INTO content_early_in_rev VALUES (%s,%s,%s)',
(blob.swhid, revision.swhid, bytes(normalize(prefix / blob.name))))
stop = time.perf_counter_ns()
logging.debug(f' Time elapsed: {stop-start}ns')
def directory_get_early_timestamp(
cursor: psycopg2.extensions.cursor,
directory: DirectoryEntry
):
- logging.debug(f'Getting directory {identifier_to_str(directory.swhid)} early timestamp')
+ logging.debug(f'Getting directory {hash_to_hex(directory.swhid)} early timestamp')
start = time.perf_counter_ns()
cursor.execute('SELECT date FROM directory WHERE id=%s', (directory.swhid,))
row = cursor.fetchone()
stop = time.perf_counter_ns()
logging.debug(f' Time elapsed: {stop-start}ns')
return row[0] if row is not None else None
def directory_set_early_timestamp(
cursor: psycopg2.extensions.cursor,
directory: DirectoryEntry,
timestamp: datetime
):
- logging.debug(f'EARLY occurrence of directory {identifier_to_str(directory.swhid)} on the ISOCHRONE FRONTIER (timestamp: {timestamp})')
+ logging.debug(f'EARLY occurrence of directory {hash_to_hex(directory.swhid)} on the ISOCHRONE FRONTIER (timestamp: {timestamp})')
start = time.perf_counter_ns()
cursor.execute('''INSERT INTO directory VALUES (%s,%s)
ON CONFLICT (id) DO UPDATE SET date=%s''',
(directory.swhid, timestamp, timestamp))
stop = time.perf_counter_ns()
logging.debug(f' Time elapsed: {stop-start}ns')
def directory_add_to_revision(
cursor: psycopg2.extensions.cursor,
revision: RevisionEntry,
directory: DirectoryEntry,
path: PosixPath
):
- logging.debug(f'NEW occurrence of directory {identifier_to_str(directory.swhid)} on the ISOCHRONE FRONTIER of revision {identifier_to_str(revision.swhid)} (path: {path})')
+ logging.debug(f'NEW occurrence of directory {hash_to_hex(directory.swhid)} on the ISOCHRONE FRONTIER of revision {hash_to_hex(revision.swhid)} (path: {path})')
start = time.perf_counter_ns()
cursor.execute('INSERT INTO directory_in_rev VALUES (%s,%s,%s)',
(directory.swhid, revision.swhid, bytes(normalize(path))))
stop = time.perf_counter_ns()
logging.debug(f' Time elapsed: {stop-start}ns')
def directory_process_content(
cursor: psycopg2.extensions.cursor,
directory: DirectoryEntry,
relative: DirectoryEntry,
prefix: PosixPath
):
stack = [(directory, relative, prefix)]
while stack:
directory, relative, prefix = stack.pop()
for child in iter(directory):
if isinstance(child, FileEntry):
# Add content to the relative directory with the computed prefix.
content_add_to_directory(cursor, relative, child, prefix)
else:
# Recursively walk the child directory.
# directory_process_content(cursor, child, relative, prefix / child.name)
stack.append((child, relative, prefix / child.name))
def revision_process_directory(
cursor: psycopg2.extensions.cursor,
revision: RevisionEntry,
directory: DirectoryEntry,
path: PosixPath
):
stack = [(revision, directory, path)]
+ # TODO: try to cache the info and psotpone inserts
while stack:
revision, directory, path = stack.pop()
timestamp = directory_get_early_timestamp(cursor, directory)
logging.debug(timestamp)
if timestamp is None:
# The directory has never been seen on the isochrone graph of a
# revision. Its children should be checked.
timestamps = []
for child in iter(directory):
logging.debug(f'child {child}')
if isinstance(child, FileEntry):
timestamps.append(content_get_early_timestamp(cursor, child))
else:
timestamps.append(directory_get_early_timestamp(cursor, child))
logging.debug(timestamps)
if timestamps != [] and None not in timestamps and max(timestamps) <= revision.timestamp:
# The directory belongs to the isochrone frontier of the current
# revision, and this is the first time it appears as such.
directory_set_early_timestamp(cursor, directory, max(timestamps))
directory_add_to_revision(cursor, revision, directory, path)
- directory_process_content(cursor, directory, directory, PosixPath('.'))
+ directory_process_content(cursor, directory=directory, relative=directory, prefix=PosixPath('.'))
else:
# The directory is not on the isochrone frontier of the current
# revision. Its child nodes should be analyzed.
# revision_process_content(cursor, revision, directory, path)
################################################################
for child in iter(directory):
if isinstance(child, FileEntry):
+ # TODO: store info from previous iterator to avoid quering twice!
timestamp = content_get_early_timestamp(cursor, child)
if timestamp is None or revision.timestamp < timestamp:
content_set_early_timestamp(cursor, child, revision.timestamp)
content_add_to_revision(cursor, revision, child, path)
else:
# revision_process_directory(cursor, revision, child, path / child.name)
stack.append((revision, child, path / child.name))
################################################################
elif revision.timestamp < timestamp:
# The directory has already been seen on the isochrone frontier of a
# revision, but current revision is earlier. Its children should be
# updated.
# revision_process_content(cursor, revision, directory, path)
####################################################################
for child in iter(directory):
if isinstance(child, FileEntry):
timestamp = content_get_early_timestamp(cursor, child)
if timestamp is None or revision.timestamp < timestamp:
content_set_early_timestamp(cursor, child, revision.timestamp)
content_add_to_revision(cursor, revision, child, path)
else:
# revision_process_directory(cursor, revision, child, path / child.name)
stack.append((revision, child, path / child.name))
####################################################################
directory_set_early_timestamp(cursor, directory, revision.timestamp)
else:
# The directory has already been seen on the isochrone frontier of an
# earlier revision. Just add it to the current revision.
directory_add_to_revision(cursor, revision, directory, path)
################################################################################
################################################################################
################################################################################
-class Worker(threading.Thread):
+class RevisionWorker(threading.Thread):
def __init__(
self,
id : int,
- conf : str,
- database : str,
- storage : swh.storage.interface.StorageInterface,
+ conninfo : str,
+ storage : StorageInterface,
revisions : RevisionIterator
):
super().__init__()
self.id = id
- self.conf = conf
- self.database = database
+ self.conninfo = conninfo
self.revisions = revisions
self.storage = storage
def run(self):
- conn = utils.connect(self.conf, self.database)
+ conn = db_utils.connect_to_conninfo(self.conninfo)
+ adapt_conn(conn)
with conn.cursor() as cursor:
while True:
processed = False
revision = self.revisions.next()
if revision is None: break
while not processed:
try:
revision_add(cursor, self.storage, revision, self.id)
conn.commit()
processed = True
except psycopg2.DatabaseError:
- logging.warning(f'Thread {self.id} - Failed to process revision {identifier_to_str(revision.swhid)} (timestamp: {revision.timestamp})')
+ logging.warning(f'Thread {self.id} - Failed to process revision {hash_to_hex(revision.swhid)} (timestamp: {revision.timestamp})')
conn.rollback()
+ # TODO: maybe serialize and auto-merge transations.
+ # The only conflicts are on:
+ # - content: we keep the earliest timestamp
+ # - directory: we keep the earliest timestamp
+ # - content_in_dir: there should be just duplicated entries.
except Exception as error:
logging.warning(f'Exection: {error}')
conn.close()
-
-
-@click.command()
-@click.argument('count', type=int)
-@click.option('-c', '--compact', nargs=2, required=True)
-@click.option('-a', '--archive', nargs=2)
-@click.option('-d', '--database', nargs=2)
-@click.option('-f', '--filename')
-@click.option('-l', '--limit', type=int)
-@click.option('-t', '--threads', type=int, default=1)
-def cli(count, compact, archive, database, filename, limit, threads):
- """Compact model revision-content layer utility."""
- logging.basicConfig(level=logging.INFO)
- # logging.basicConfig(filename='compact.log', level=logging.DEBUG)
-
- click.echo(f'{count} {compact} {archive} {database} {filename} {limit}')
- if not database: database = None
- if not archive: archive = None
-
- reset = database is not None or filename is not None
- if reset and archive is None:
- logging.error('Error: -a option is compulsatory when -d or -f options are set')
- exit()
-
- # TODO: improve the way a connection to PSQL is configured.
- comp_conn = utils.connect(compact[0], compact[1])
- cursor = comp_conn.cursor()
-
- if reset:
- utils.execute_sql(comp_conn, 'compact.sql') # Create tables dopping existing ones
-
- if database is not None:
- logging.info(f'Reconstructing compact model from {database} database (limit={limit})')
- data_conn = utils.connect(database[0], database[1])
- revisions = ArchiveRevisionIterator(data_conn, limit=limit)
- else:
- logging.info(f'Reconstructing compact model from {filename} CSV file (limit={limit})')
- revisions = FileRevisionIterator(filename, limit=limit)
-
- # TODO: take this arguments from the command line.
- kwargs = {
- "cls" : "remote",
- "url" : "http://uffizi.internal.softwareheritage.org:5002"
- }
- storage = swh.storage.get_storage(**kwargs)
-
- workers = []
- for id in range(threads):
- worker = Worker(id, compact[0], compact[1], storage, revisions)
- worker.start()
- workers.append(worker)
-
- for worker in workers:
- worker.join()
-
- if database is not None:
- data_conn.close()
-
- cursor.execute(f'SELECT DISTINCT id FROM content ORDER BY id LIMIT {count}')
- for idx, row in enumerate(cursor.fetchall()):
- swhid = row[0]
- print(f'Test blob {idx}: {identifier_to_str(swhid)}')
-
- fst = content_find_first(cursor, swhid)
- print(f' First occurrence:')
- print(f' {identifier_to_str(fst[0])}, {identifier_to_str(fst[1])}, {fst[2]}, {fst[3].decode("utf-8")}')
-
- print(f' All occurrences:')
- for row in content_find_all(cursor, swhid):
- print(f' {identifier_to_str(row[0])}, {identifier_to_str(row[1])}, {row[2]}, {row[3].decode("utf-8")}')
-
- print(f'========================================')
-
- comp_conn.close()
diff --git a/swh/provenance/py.typed b/swh/provenance/py.typed
new file mode 100644
index 0000000..1242d43
--- /dev/null
+++ b/swh/provenance/py.typed
@@ -0,0 +1 @@
+# Marker file for PEP 561.
diff --git a/iterator.py b/swh/provenance/revision.py
similarity index 100%
rename from iterator.py
rename to swh/provenance/revision.py
diff --git a/swh/provenance/tests/__init__.py b/swh/provenance/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..7a2c21c
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,35 @@
+[tox]
+envlist=black,flake8,mypy,py3
+
+[testenv]
+extras =
+ testing
+deps =
+ pytest-cov
+commands =
+ pytest --doctest-modules \
+ {envsitepackagesdir}/swh/provenance \
+ --cov={envsitepackagesdir}/swh/provenance \
+ --cov-branch {posargs}
+
+[testenv:black]
+skip_install = true
+deps =
+ black
+commands =
+ {envpython} -m black --check swh
+
+[testenv:flake8]
+skip_install = true
+deps =
+ flake8
+commands =
+ {envpython} -m flake8
+
+[testenv:mypy]
+extras =
+ testing
+deps =
+ mypy
+commands =
+ mypy swh