diff --git a/.dockerignore b/.dockerignore new file mode 100644 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.tox +.eggs diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ PKG-INFO revision-origin /misc/ +.venv diff --git a/Dockerfile b/Dockerfile new file mode 100644 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.9 + +COPY requirements* / +COPY docker/pyutils.sh /usr/local/bin + +RUN apt-get update && apt-get -y install rsync libcmph-dev && \ + addgroup --gid 1000 swh && \ + useradd --gid 1000 --uid 1000 -m -d /src swh && \ + chmod a+x /usr/local/bin/pyutils.sh + +USER swh + +RUN python -m venv /src/venv && \ + . /src/venv/bin/activate && \ + python -m pip install --upgrade pip && \ + ls /requirements* | xargs -t -n1 pip install -r + +ENTRYPOINT /entrypoint.sh + +ENV SWH_CONFIG_FILENAME=/config.yml diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -34,3 +34,17 @@ ``` + +Running in Docker +----------------- + +### Build the image +``` +docker build -t swh-provenance . +``` + +### Run the services +``` +docker-compose up -d +docker-compose logs -f +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,54 @@ +version: '3' + +volumes: + storage-db: + +services: + storage-db: + image: postgres:13 + command: [ "-c", "max_connections=1000", "-c", "config_file=/etc/postgresql/postgresql.conf" ] + environment: + POSTGRES_PASSWORD: provenancepassword + POSTGRES_DB: provenance + POSTGRES_USER: provenance + volumes: + - storage-db:/var/lib/postgresql + - ./swh/provenance/sql:/docker-entrypoint-initdb.d + - ./docker/postgresql/postgresql.conf:/etc/postgresql/postgresql.conf + + rabbitmq: + image: rabbitmq:3.6-management + ports: + - 5072:5672 + + storage: + image: swh-provenance + build: . + volumes: + - $PWD:/src/swh-provenance + - ./docker/storage/entrypoint.sh:/entrypoint.sh + - ./docker/storage/config.yml:/config.yml + depends_on: + - storage-db + - rabbitmq + + origin_server: + image: swh-provenance + build: . + volumes: + - $PWD:/src/swh-provenance + - ./docker/origin_server/entrypoint.sh:/entrypoint.sh + - ./docker/origin_server/config.yml:/config.yml + - ./docker/data/origins.csv:/origins.csv + ports: + - 5555:5555 + + origin_client: + image: swh-provenance + build: . + volumes: + - $PWD:/src/swh-provenance + - ./docker/origin_client/entrypoint.sh:/entrypoint.sh + - ./docker/origin_client/config.yml:/config.yml + environment: + NB_CLIENTS: 3 diff --git a/docker/data/origins.csv b/docker/data/origins.csv new file mode 100644 --- /dev/null +++ b/docker/data/origins.csv @@ -0,0 +1,442 @@ +HTTPS://GitHub.com/hexatomic/hexatomic.git,1e67f89d2aa7a949f981cd278f93341ce25664f9 +HTTPS://GitHub.com/hexatomic/hexatomic.git,daf728acfa03012980177e4843e594014989fe1a +Https://GitHub.com/Go-HEP/hep,7b372bd07a592d0823ab218033e52a8db81f52ea +Https://GitHub.com/IBM/BlueMatter,cc6742e6a032d165a684fb6c4a357744277b6a91 +deb://Debian-Security/packages/389-ds-base,010db80f69cab3f7a48bc86fd60c978e19a0e1a6 +deb://Debian-Security/packages/389-ds-base,4b1cbde78348b5d5701ef64281cdc03e6eda4a01 +deb://Debian-Security/packages/389-ds-base,54828ac2fdbab489e256c29ad2cbffab7641a70a +deb://Debian-Security/packages/389-ds-base,82baef9a5e007802cf402169ea206543cfee13fc +deb://Debian-Security/packages/a2ps,ed306a0b9220cdd59bedef8f546712d61536b949 +deb://Debian-Security/packages/acpi-support,6c6b8df7710611e2d549ec7fb303252f9be10ad2 +deb://Debian-Security/packages/activemq,255428ec0f0546b9d5e85b351953f817882555ea +deb://Debian-Security/packages/activemq,870850f84fb5f9729d7b7ca2b85efe16a0f2ae24 +deb://Debian-Security/packages/activemq,8b5c33bf9d7b95ede5e175aa977ee2e8c64bb7a7 +deb://Debian-Security/packages/adblock-plus,437e0949b1f483bf2e6e87eaf372f6c3f0c2299b +deb://Debian-Security/packages/adminer,88d1033cb5fa773f796453c46622506923756ea3 +deb://Debian-Security/packages/adminer,a675b19351e3f3c816e12e7e01580f4f6d1aa409 +deb://Debian-Security/packages/adminer,e9d135fb28762e243ae70052a84311231c02502f +deb://Debian-Security/packages/adplug,263dd21db49647f9463f8530b74e6e012b9f473e +deb://Debian-Security/packages/advancecomp,c02802e0e1cc9a556e882a3e8b545e920cb3bf96 +deb://Debian-Security/packages/advancecomp,ddb111dd42b66e7e6b525a432c8411181a1dda7f +deb://Debian-Security/packages/agg,0657b9cdc54656fafa3f156b0849a3587ef83d43 +deb://Debian-Security/packages/akonadi,c147b1d0ec661c51a94ecce064eb74dabfb88201 +deb://Debian-Security/packages/alabaster,8f59a8239dbdeb0cc5f8e11372d09f5e83e9956f +deb://Debian-Security/packages/alabaster,b4b186dd9b2f8ff1e84e764fa3a1336d30b6b751 +deb://Debian-Security/packages/alabaster,c6c578b0a0e36cfb8171637e9a90eede42166575 +deb://Debian-Security/packages/amd64-microcode,786f9f166827bcb39d4e50af8b2bfb2288379016 +deb://Debian-Security/packages/amd64-microcode,cd35a7bdaa96acfb8a47368332a149d35d741527 +deb://Debian-Security/packages/ansible,8ac94aec9fd42c0eabe5a3c5aa69d7dc3229a6c6 +deb://Debian-Security/packages/ansible,ba338a62883a8a5c52be5043755ac53da5d9d550 +deb://Debian-Security/packages/ansible,c06bd9c78f396eaa94b6290ff7872f35b67b3157 +deb://Debian-Security/packages/ant,9398c52fcc17ad697099f0a7b09c5f7604903a25 +deb://Debian-Security/packages/ant,a54dfaf3fb9789786e0f5ae84dc91a764e53d64e +deb://Debian-Security/packages/ant,cdc9f335713e636f4d963d347900c7a39ee2f2e0 +http://06th2d-nhom06-quanlynhasach.googlecode.com/svn/,df13198611cbc0487d78837f7ea1944f974ac5f6 +http://06th2d-nhom1-quanlynhasach.googlecode.com/svn/,c16eed779de148cfe2ab3acdeeddd1818100109a +http://06th2d-nhom10-quanlynhasach.googlecode.com/svn/,dfbb3ecb31f2df14973d01abc90f0d96e288eb45 +http://06th2d-nhom11-quanlynhasach.googlecode.com/svn/,584c8b86596fefdc415c324c5204e306d2f66e66 +http://06th2d-nhom2-quanlynhasach.googlecode.com/svn/,757aadaa051e7c71b552ddc8d86e6aacba1ea778 +http://06th2d-nhom4-quanlynhasach.googlecode.com/svn/,c913a4a69d42f01aae3cdc42aaf4e4af8bcca169 +http://06th2d-nhom5-quanlynhasach.googlecode.com/svn/,38d0e797bc79d373473299e72eb995cf5b6cda8d +http://06th2d-nhom7-quanlynhasach.googlecode.com/svn/,5518103390c1797c8ff2e533b9511232b1386c6d +http://06th2d-nhom8-quanlybansach.googlecode.com/svn/,f88fb6f00b85fd795618efe1c9fb5ed17828857f +http://06th2d-nhom9-quanlynhasach.googlecode.com/svn/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://06th2d-nhom9-quanlynhasach.googlecode.com/svn/,e0ba422d04546bd48f184f99a351a81baf1ec9bf +http://06th3d--5-quanlybansach.googlecode.com/svn/,3836bce74dedaf1e47e3c8d417bc3642c031146f +http://06th3d-1-quanlynhasach.googlecode.com/svn/,128396663d4676a8a70e0ed680056004684550cd +http://06th3d-2-quanlynhasach.googlecode.com/svn/,a9a3107f42eef7f4763ce6280a70ac9bdaf29abe +http://06th3d-7-quanlynhasach.googlecode.com/svn/,392a1dca8d6972059381a949c7a7662d0ac5484c +http://06th3d-8-quanlynhasach.googlecode.com/svn/,2e1cad72e6912837ae81b04f0448aea0fc3282dc +http://a-d-h.googlecode.com/svn/,82dee191452f860877bb6d6336fd0ba6d04f9295 +http://a-dda.googlecode.com/svn/,e73f0d9b5255455cc3b235751936fc559d7bd88f +http://a-ddos.googlecode.com/svn/,2118726a48bc9d2b3e558eb3f3ce7f32eeb9484f +http://a-distributed-file-system.googlecode.com/svn/,dcbd5ca98a23fb4fc4f4d7c9e5261de217395949 +http://a-drake.googlecode.com/svn/,3251f167259cbbf9dfbe64914681540761e4905b +http://a-droid-tale.googlecode.com/svn/,70dcee7f8a8b634f567ddf3dbf58d8faf16fdec3 +http://a-e-tets.googlecode.com/svn/,27bf009870f0fef8252103fb0aa6262bf942a9f8 +http://a-elsesky-backup.googlecode.com/svn/,d85b6c552d4fb917e5ae1f054c8fc2c398e72d7a +http://a-eng.googlecode.com/svn/,67b1d2496d8bc6ce15f65d630305320238485885 +http://a-estrela.googlecode.com/svn/,cb1e58963108feb5270563782ba5144043045178 +http://a-evolving-monkey.googlecode.com/svn/,066285d62497ea46ed9ec6e6e2927757c3e98b0c +http://a-f-clothes.googlecode.com/svn/,0b38688704fa196652ac77c3f78546c093535d1a +http://a-f-d-c.googlecode.com/svn/,cab660736bfcfcebd0bd550a87a78de5a66b371c +http://a-falcon.googlecode.com/svn/,428b099e1e70f1eecb401d8d08835ba6a9781f43 +http://a-few-random-aard-plugins.googlecode.com/svn/,c84a75855a90771f22413169feada3eae94a8728 +http://a-fi.googlecode.com/svn/,99c5104a30306912f3438f6841d961f91b5a7bbc +http://a-foto.googlecode.com/svn/,e57919ea12e4a28b05dd968c4c2f16e63d793781 +http://a-fuga.googlecode.com/svn/,9b3658b29d107b0cada0b4a1e621991cd29b455a +http://a-funny-ball.googlecode.com/svn/,0dbf5b1f1a4cac08010a6e464f887df6f9849f99 +http://b-trunk-project.googlecode.com/svn/,90b8c5421edef521c153a17a86e55e6549f25958 +http://b-u-g.googlecode.com/svn/,1976a04c77cf42ae4eefa26a2bcf99faa028c6d3 +http://b-work.googlecode.com/svn/,e374638d346747a6d6ea68c7db738a9af0e97871 +http://b-z-p.googlecode.com/svn/,5546d43109d6cd167eaf5d1be51e1240aa023bb7 +http://b00ks-d0c.googlecode.com/svn/,e9e389665afb0c9e40fcb185d14277d4e9c93594 +http://b00nc0n.googlecode.com/svn/,d094936cc2a15879d0cdab41e25041c53932898e +http://b024.googlecode.com/svn/,fb2e839fa4f822e2cedc279ee55d673f63fc6476 +http://b07017.googlecode.com/svn/,8ed8b5065cc64ec0063fdf8f2624325f31118f41 +http://b07522-623.googlecode.com/svn/,9b104125983068a838e603796509a4c09d671475 +http://b0922964032.googlecode.com/svn/,dfdf9e7da0243e6637690ce1af20bde7c5f2080a +http://b0ark-rutorrent.googlecode.com/svn/,dc1f47f6ce6ab94fcf3d35a2bc34aed9d6e159df +http://b0mberman.googlecode.com/svn/,c0d1b665801d9a6514400f7cb94aa68e8f927465 +http://b0rder.googlecode.com/svn/,806402ac50603dd466fb2a9f349ab3739542b744 +http://b0rx0r.googlecode.com/svn/,f5371094793dc9d93a7e4959161d6c6cece5c88e +http://b0tn3t.googlecode.com/svn/,4a1ad362c9e96cb773c530240f6ab56ad0a5721d +http://b0undless.googlecode.com/svn/,06f9efb382f19b61e5cffabcafaba640195b8ef1 +http://b100123thongdangngan.googlecode.com/svn/,65331816e98ff2fdd8510c2c80f8456f71f6e10b +http://b12.googlecode.com/svn/,eadbc1f3d7d6d2bf1e8954570a6787afff0b2efa +http://b121boop.googlecode.com/svn/,12ff7a2d4eedb8a3419c175bf7a2b6380c8527e4 +http://b125-keyring.googlecode.com/svn/,af0ad3a03cb391ab68566ddf052bf4893fa26aab +http://b12cgp.googlecode.com/svn/,9495d6bb0be754d2a8a474fe389145e3fc447d93 +http://b12luchito.googlecode.com/svn/,8240f77d0f6d4f32908fe589fde7b63c1833a310 +http://b13-munirka.googlecode.com/svn/,4b8a6adafc3a83da934cc07bc92b9abaaa1ffbc6 +http://b13eplugin.googlecode.com/svn/,d20c2f539347a865401653c50e0f7ec34530a80e +http://git.baserock.org/git/delta/qemu-SLOF.git,4e85a262d85bf6decf97cd633de33ec3f30fd332 +http://git.baserock.org/git/delta/qemu-SLOF.git,5bd92e386ed4e2bf4c56e38fdb83d9dbe1424bd7 +http://git.baserock.org/git/delta/qemu-ipxe.git,3b4dfdf44158a732a9bd7a5e2dbb84a91115c441 +http://git.baserock.org/git/delta/qemu-ipxe.git,52ecf53962282ae8aaa6730ec74e19bd4a870b8e +http://git.baserock.org/git/delta/qemu-openbios.git,e7f275ed70793369e1538c4c34a7e2e29f9bf7f3 +http://git.baserock.org/git/delta/qemu-openhackware.git,ccb808e3fe11b3af66234bf80fea58172b554273 +http://git.baserock.org/git/delta/qemu-palcode.git,7866cec2174f01289239d6f01c86e1ba4fb56676 +http://git.baserock.org/git/delta/qemu-palcode.git,d7c887cf63077c31a702c5b4ac15098f5bac3e2b +http://git.baserock.org/git/delta/qemu-seabios.git,2fbf691a803eb7fb847c88644b1027f7528bee06 +http://git.baserock.org/git/delta/qemu-seabios.git,97456b7f15f3a90957ebf62c70420633511df4d9 +http://hdiff.luite.com/cgit/keera-hails-reactive-fs/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://hdiff.luite.com/cgit/keera-hails-reactive-gtk,eb9c79e6efcb930cf35271ce986c2d50bff4fcd1 +http://hdiff.luite.com/cgit/keera-hails-reactive-gtk/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://hdiff.luite.com/cgit/keera-hails-reactive-htmldom,ff6054b77f01b4835271d0aaaa6236978a11dd0c +http://hdiff.luite.com/cgit/keera-hails-reactive-network,5968ad1ae82a7d2a130b04c339a61e1697108a7c +http://hdiff.luite.com/cgit/keera-hails-reactive-network/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://hdiff.luite.com/cgit/keera-hails-reactive-polling,9ed0167fb6ec7dadf3913acae507c6b170871300 +http://hdiff.luite.com/cgit/keera-hails-reactive-polling/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://hdiff.luite.com/cgit/keera-hails-reactive-wx,091e630d5e747ed37043141c33fd38ee9c730632 +http://hdiff.luite.com/cgit/keera-hails-reactive-wx/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://hdiff.luite.com/cgit/keera-hails-reactive-yampa,9c4e09e90a9e939411a9c1f206d0ab1d5059e31d +http://hdiff.luite.com/cgit/keera-hails-reactive-yampa/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://hdiff.luite.com/cgit/keera-hails-reactivelenses,f696e38f464fbda1c042a1feb9ef2821261c5fc0 +http://hdiff.luite.com/cgit/keera-hails-reactivelenses/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://hdiff.luite.com/cgit/keera-hails-reactivevalues,a179b99743a5039f94dd7e180dbadf054e556921 +http://hdiff.luite.com/cgit/keera-hails-reactivevalues/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://hdiff.luite.com/cgit/keera-posture,12c3a971522e4b86d8e8ebb16d12035993e2280f +http://hdiff.luite.com/cgit/keera-posture/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://hdiff.luite.com/cgit/keid-core,133bbc62a290c0ff964cde8ec4f0877306e168bd +http://hdiff.luite.com/cgit/keid-geometry,8d5f2d416faea8db571d239be76935e4d90f6b40 +http://bitproject.googlecode.com/svn/,8bc2d2c5a7e2501a01b96635fde78772f81fefc6 +http://bitproject123.googlecode.com/svn/,b51d1de0988e8e2632ea0f38c67b7a41f2bfe0e1 +http://bitproject234.googlecode.com/svn/,c07957038219cd52d9e7d580cb05565f6337423e +http://bitps3.googlecode.com/svn/,afdcadd6eb2723d1dd70cc54cd93d52e70ddd5f1 +http://bitr.googlecode.com/svn/,f827c34a546d355d49ba158497cce64d46085aff +http://bitraptor.googlecode.com/svn/,8f28d137b70139c16a0eeff5f7fa1166e6a9d4a0 +http://bitrat.googlecode.com/svn/,9f20421bc11ce6bbb9816bce0d37ea2365692013 +http://bitrate-calculator.googlecode.com/svn/,7a0172f138a0ee169919321358613ada98d5dcfb +http://bitreap.googlecode.com/svn/,094a12b0baeffe983e3789a60e2de9c45df51257 +http://bitree.googlecode.com/svn/,4971fbae5e8cc28d17172b031de4b1c49ba6e2dc +http://bitrix-code-generator.googlecode.com/svn/,493d00a33f0d8d8ead6c538f96956c7546e0583d +http://bitrix-tables.googlecode.com/svn/,4df4efb01311f08f98bc16153046d57f0430ccf6 +http://bitrix-trinhphat.googlecode.com/svn/,ae671f1bd2286a3de941e4017084eac0c17e522f +http://bitrixonrails.googlecode.com/svn/,fb30bd5e324b4e5a86681c98278e485846fbdb3f +http://bitro.googlecode.com/svn/,231930fee031445ce4b08998a839629741feaa6a +http://bitrober.googlecode.com/svn/,a423857e634e7bfabc8c79ce8bd64d62c99de79d +http://bitroid.googlecode.com/svn/,ae532b6ecc2ecdedc167c70603fdc82694a1398b +http://bits-bazaar.googlecode.com/svn/,af9579884d8bdc4aa793d1ada7009aebb185b1c8 +http://bits-crawler.googlecode.com/svn/,6792ff714278959fd5d0c47b8b1100f7cb541337 +http://bits-hotel.googlecode.com/svn/,71ce3f9ed72443754a6d3795940f7a31a022bfdf +http://bitbucket-s3.googlecode.com/svn/,39815f57c961645edd0c0a4e23e12bb57865abcd +http://bitbucket.org/leonate/jxls-reader.git,944468a76aa865bb27a27de6b7b3a1208349f6a4 +http://git.baserock.org/git/delta/python-setuptools-bitbucket.git,5e8ca414d32077e876c4e0e112e21eaa0026db71 +http://leo-bitbucket.googlecode.com/svn/,07560fcb4dcd397e3a130d86cfdab9f2dda49253 +http://snapshot.debian.org/package/python-bitbucket/,80c281218710f6995a8f6bd36be9eafeb4bfa0b5 +http://snapshot.debian.org/package/ruby-omniauth-bitbucket/,17bcd3f23bd6e915554bc6a3301e4ec979eade92 +https://bitbucket-cli.googlecode.com,f114be4e469045446425d725e5aac95bb46ebb7d +https://bitbucket.org/---------------_-/esistatus.git,3689b128c023ac7f1b20758bf5625a0f715912bd +https://bitbucket.org/---------------_-/esistatus.git,618a45d241ff4668220ce994d47fa2136ea9e73e +https://bitbucket.org/----Qq482571670-----/gu-you-nai-hao-guo-nei-ding-ji-you-zi-da-ka-ma-shang-jiang-zai.git,d667ec4f425f83bf6041cfb11f960d0459e4f6bd +https://bitbucket.org/---david/pruebas_curso.git,26d352519dab084ea4a287180d86a12a8215ec28 +https://bitbucket.org/---david/pruebas_curso.git,7b27f0a4d03cb2f3c1c0f1ff0328994b2d07c4f2 +https://bitbucket.org/--DEvelopER--/files.git,0d7e2300e506d504db7466b879bb4fc4578863a3 +https://bitbucket.org/--Dim4ik--/caesaria_18_05_22.git,03188f2d2ac174f8d3b308c0cdd661faf6f68772 +https://bitbucket.org/--Dim4ik--/caesaria_18_05_22.git,550fd50538c508534604cd5d05d742b47aa54639 +https://bitbucket.org/--Raavan/pinthecirclereplica.git,da46283297f572b5373da29399f57865cf3c722b +https://bitbucket.org/--Raavan/pinthecirclereplica.git,fb522e57a8dae6aa22a5ffb3cd26d321834877a4 +https://bitbucket.org/--Raavan/stats.git,bbe0a62719e7157f86bc7447e7b568f1aeac6372 +https://bitbucket.org/--Raavan/stats.git,f7cfc74988ec57168ae853a6f9f3376040c0a4d6 +https://bitbucket.org/--Raavan/vrstore.git,755c173e27c9748216f39c66c3972481a055b24c +https://bitbucket.org/--Raavan/vrstore.git,c5b171a50a65e0aba050614d9e35248343119e99 +https://bitbucket.org/--SSNS--/3.3.5a.git,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +https://bitbucket.org/--Shobhit--/cube-rotation-shobhitmangla.git,21c31eef7d0c613bb19fb6f233516ce079d2a4de +https://bitbucket.org/--Shobhit--/todo-list-shobhit-mangla.git,2f54dba61c4b512595420b959e599f25c76963d4 +https://bitbucket.org/--a-team----name-/ini.git,15ffc54b1aedff22d5a000325017dcd210514530 +https://bitbucket.org/--a-team/resume-apirest.git,8b3491363101cd35171d2cc4f2f286ae9c7d6a1e +https://bitbucket.org/--anandk/begin0.git,2afeba392182cbf05b60dd846770508fbe0181c4 +https://bitbucket.org/--anandk/begin0.git,b2db8393623e5c48d0808c2facb52d703751c68c +https://bitbucket.org/--nana/half-moon-form.git,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +https://bitbucket.org/--skald--/task.git,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +https://bitbucket.org/--ssh0--/ssh0-.bitbucket.org.git,3d133540406e9c95db5984dfcc2f993fdb3d16d8 +https://bitbucket.org/--ssh0--/ssh0-.bitbucket.org.git,c420cfc6c0fff05047c539bbff9513ef2c42e4f9 +https://bitbucket.org/-0w0-/.emacs.d.git,298cc970f1d8c661e71461a319ac26ba88b464a2 +https://bitbucket.org/-0w0-/.emacs.d.git,9fd48f72c20e659379a9b07eeefd8e3c16ae459d +https://bitbucket.org/-1/mmtk,96867198e87768b8dee8ecacee0fced00fd08fed +https://bitbucket.org/-1/mmtk,a7cf3f7c9131b2cdfcb33f552d898c5ee9286fa2 +https://bitbucket.org/-1/mmtk,c08e28bd377be0c29a9ac1b6bc3092c9ee56adb9 +https://bitbucket.org/-1/objectlistview,15ccfbff7a5f1c51a86c5e21d0ed2052fe1c27ad +https://bitbucket.org/-1/objectlistview,c7746587f9f5103b77ca90c55a4dea7b6ba4a68b +https://bitbucket.org/-Al-/tabajara.git,959c18f5a1fb2f362404b6b97ee95f8efedc028f +https://bitbucket.org/-Al-/tabajara.git,f0380bfb119136bcda76d9799396018d50d263a3 +https://bitbucket.org/-Atal/productsrepo.git,a652e1ce84e00b4d54cd899e1a94246d3ee8f7ad +https://bitbucket.org/-Avatarchik-/assetbundledemo,55a19595eca31a8d59690804e777197f8cdc4d73 +https://bitbucket.org/-Avatarchik-/assetbundledemo,5b7820f39c788b8c5666a81757c93c03672a4b4c +https://bitbucket.org/-Avatarchik-/googletounity.git,162f01c64d902f6d40586a31c07adf84a15a85ba +https://bitbucket.org/-Avatarchik-/googletounity.git,f8b7e1d047a3d2acf520c3c18dccada71f0072bd +https://bitbucket.org/-Avatarchik-/hive-deploy.git,511125ce4cb6b25beba3fadf7ee8a9b52df11345 +https://bitbucket.org/-Avatarchik-/pnet2.git,0c92a90e1d6f9b81f130d7e65ede58b8b07e55f6 +https://bitbucket.org/-Avatarchik-/pnet2.git,d004e65973ab6973799a84d4f6022bbb41572802 +https://bitbucket.org/-Avatarchik-/pong_solitaire.git,226f0b7c9cc671529c15b3fbcdd928631b216007 +https://bitbucket.org/-Avatarchik-/pong_solitaire.git,3d350af7314ce77d58fb736f57ce162fe03360a8 +https://bitbucket.org/-Avatarchik-/slot-machine.git,062aef46a0ba6d9a196ea5a745f6567186bbc958 +https://bitbucket.org/-Avatarchik-/slot-machine.git,10e8bfcefcfbd8f2130aced6c32ed3444ff2bc03 +https://bitbucket.org/-Avatarchik-/tstableview.git,0476f944285eef71f89f1b42b932221ff3537fc9 +https://bitbucket.org/-Avatarchik-/tstableview.git,44452be8a72547146d8fbe5a7f99a04105ff56e9 +https://bitbucket.org/-Avatarchik-/unity3d-fpsphotoncloud-beta.git,96f12fec41721b8fc133d71db829a69bcac44349 +https://bitbucket.org/-Avatarchik-/unity3d-fpsphotoncloud-beta.git,bd4669e3508803f6dc2f20de88de7b47b955aa49 +https://bitbucket.org/-Avatarchik-/wad2unity.git,0ccd01043dba96c84eecfc99213322e45f312159 +https://bitbucket.org/-Avatarchik-/wad2unity.git,e6a096417a482a2276cd1a1ef594e22ec1e6c640 +https://bitbucket.org/-Cifer-/the_swarm.git,f20377a9e71c69c95fb0671bb986d258121cfa3d +https://bitbucket.org/-Cifer-/version-control-turtorial.git,05ca5a2140be64aef486eca26bbf39d2bfa27aae +https://bitbucket.org/-David09/aseos_s.a.git,39ad73ccd3679ba01742d3dfda1580ac1830a390 +https://bitbucket.org/-David09/aseos_s.a.git,9379e916c5b8de39b307a4f9c0aac07c5fce81f3 +https://bitbucket.org/-Dreamer-/lasertag.git,dddb0471d120254b4a059840ef65913ced7045f7 +https://bitbucket.org/-Dreamer-/lasertag.git,ee8b3525aeb64610975d5d4bbf6285804b645547 +https://bitbucket.org/-Dreamer-/worldtablet.git,242bcdd2eed6ba11a5d5bf03e2cedae032e20d56 +https://bitbucket.org/-Dreamer-/worldtablet.git,2825956e7537473106d1189225657e4f0e343148 +https://bitbucket.org/-Dreamer-/worldtablet.git,a71557cd02893e9f3ab414f53b11750f2af022f7 +https://bitbucket.org/-Elmer-/plinkseq.git,0f139277823ffc0ad2db1ddb41ee06ec7f053e95 +https://bitbucket.org/-Elmer-/plinkseq.git,41aa8872b029c2f12b02fc8e60fbe071905698a9 +https://bitbucket.org/-FAL-/tutorials.git.bitbucket.org.git,2bddeeee228517d4fac64790ad33dc6c53efa3a1 +https://bitbucket.org/-FAL-/tutorials.git.bitbucket.org.git,4cb5fb70166e9c035c8f491e16056df0ed5904e7 +https://bitbucket.org/-Flio/check.git,8a968ff657d62400874c681283c14e29c211e00f +https://bitbucket.org/-Flio/check_fork.git,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +https://bitbucket.org/-Foley-/factorio-pub.git,40862acbc3a3ad81831a609cee886954f8f8f0b3 +https://bitbucket.org/-Freezy/get_next_line.git,793fcea3d261effcb9da7230b0087f96445a41eb +https://bitbucket.org/-Freezy/get_next_line.git,b4ed49bfb5c69b64a7dda7a3680d49eb85da3661 +https://bitbucket.org/-Freezy/minitalk.git,00861d50eea564fe427c353075cd25c848fd339e +https://bitbucket.org/-Freezy/minitalk.git,16f431fbc9800ecbe694dfdd13e6f0b5bf3e39f7 +https://bitbucket.org/-GhostShadow-/test.git,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +https://bitbucket.org/-Innovators-/game-project.git,0189c9a685e29d8921e9e95d1f481cd887a2bb4e +https://bitbucket.org/-Innovators-/game-project.git,f530f7faf7473751128cd5f080a52e6b01801db7 +https://bitbucket.org/-JCA-/larepublica.git,7397a3c59026c59023a849fbfdcee3aeaf5c4500 +https://bitbucket.org/-L0Lock-/blender_community_fr.git,4c4bbe4a8868399e1c7ed146ab1b00e3264b5b40 +https://bitbucket.org/-L0Lock-/blender_community_fr.git,9637abc5ddead046737ffcc955191625abd5d5f9 +https://bitbucket.org/-L0Lock-/ffmpeg-bat-collection.git,4e6dd0ae26da6682ff62e68a0c4e28676255a4ac +https://bitbucket.org/-L0Lock-/ffmpeg-bat-collection.git,ae74771a14127930edf76ebdd292422d57aabb83 +https://bitbucket.org/-L0Lock-/riggingtools.git,5b4835ffac22fe315e1a2a15338a8d1619833d77 +https://bitbucket.org/-L0Lock-/riggingtools.git,675eb0d5d45cbc73f2a580b9ce29abaae668fbb1 +https://bitbucket.org/-Luxe/learning.git,69f0124cf99b94d1319d8c3dac7dad3b38b2f0c1 +https://bitbucket.org/-Luxe/learning.git,ea1a8f1fa3dc44ebcc0df6040296fffad691c644 +https://bitbucket.org/-Marcin/opa-site.git,020a31e5842eb3179b2c4a6343cbe9139b5a5f4b +https://bitbucket.org/-Marcin/opa-site.git,05908ff49be332b35da3e1e98e68fa21e03eae14 +https://bitbucket.org/-Marcin/pro-inz.git,11e183460a48eb6fa27db0ae9be24fc0e22c6540 +https://bitbucket.org/-Marcin/pro-inz.git,31dd65dff6a6bbab1dbec213788aabb809f0f5a0 +https://bitbucket.org/-Mateus/simuladordecache.git,a557ead134f93c663c231b71ce8ab39f3821d0a8 +https://bitbucket.org/-Micky-/proyecto4.git,4ab1de359d4c52bdb6efa9096612d6754b32c0b8 +https://bitbucket.org/-Micky-/proyecto4.git,ee70105c1c5281f75549eb25af6df32b644b06e7 +https://bitbucket.org/-Muneeb-/jquery-game.git,3dde1d29f870762fc74feea353daef8847ed33f3 +https://bitbucket.org/-Muneeb-/jquery-game.git,b23c847484404e763f3fb06b4a5830ccf3665ca0 +http://101-com-8852-int-969.googlecode.com/svn/,31f6907ba26afab1dfd71ba161f624c641627086 +http://101-intro-exercises.googlecode.com/svn/,f14fa6f5a5f8a352707723de3376f6847a596e36 +http://101-intro.googlecode.com/svn/,55f72fa90b577ff35763b239589048812d9f1949 +http://101-things-gadget.googlecode.com/svn/,6ba08621c42f3c712fe3d816b2bb40d5904618fd +http://101-things-to-do-list.googlecode.com/svn/,5e31641c17314d0eeb5cd93ef2832a88d3c808cc +http://10100ckua.googlecode.com/svn/,fd0df2e4c095b04ffe8c2ecb55bb078dd5f6772a +http://101010101s110001100101010110.googlecode.com/svn/,efcb960c1dfb75175fd41b0eb29b0e7375fcaa40 +http://10102012-name.googlecode.com/svn/,cd8e4c4927ec0ea41cb3dc4143d8d4ebd41f1268 +http://1010q.googlecode.com/svn/,bc3cf85c8a63a00cb865245bb7c2525137903e3d +http://101110blog.googlecode.com/svn/,1519902ab1f6cf4bc2ba5070f989f0a111071e84 +http://1011s1dfsi.googlecode.com/svn/,ae156d50c67a4030687642d92ae3d977355724ff +http://1012-game-engine.googlecode.com/svn/,0eea4c4c646ee8aa8e60e50d66b3b8b183aaccd4 +http://1012-group-trunk.googlecode.com/svn/,3fa562fe14e3f4900fe47207d29f9b6601ce2f00 +http://1012142-nmcnpm.googlecode.com/svn/,24c708410a1ecc87b50036298a5487074ce3bcef +http://1018djj.googlecode.com/svn/,7c9ea94d4c3dff90a006df84e6a71a830136da64 +http://101a03.googlecode.com/svn/,e0c7bf8b2aa7b330d41162bc59b577f3ee625cd0 +http://101app.googlecode.com/svn/,64bac43d50552f12be6836a0b332d606f228ac58 +http://101j.googlecode.com/svn/,303afdf325162c48963a011a2450607b4773ef38 +http://101kompilator.googlecode.com/svn/,dc7df60cab80aa31d2672706d7b93fe08094d28c +http://102-lineadelosderechos.googlecode.com/svn/,6b63750ff2e03ec533dfa34d73131a47edda74b0 +http://102-s23.googlecode.com/svn/,71afacef6edf545d5aa63679aa0708c125419302 +http://1027test.googlecode.com/svn/,b4b1728e1f0901f15b52f78fcccdd9b239820573 +http://102chute.googlecode.com/svn/,80d6c9b24fd179282ed93d4eb04938e0c578eed4 +http://103-weather-symbol-markers-in-cdat.googlecode.com/svn/,65cdfd55060fd5cbd0d28771a91a0919e501c706 +http://1030digitalhuman.googlecode.com/svn/,ee7a72830bd55ce6802d54907effd6b00693072a +http://1030epoxy.googlecode.com/svn/,882ef81588e11af3606d632c8ed77b9e4210320a +http://1038427237823489234772348234a.googlecode.com/svn/,3b9e5d342719eb60ad3040a507017ad43a098cb3 +http://103956-kirak-cpp.googlecode.com/svn/,76e39161236b3d6718c021339c8b3f1645755b7c +http://103956-kirak-java.googlecode.com/svn/,01bcfd0e90c04006a83371451979e2e4f99d0a74 +http://1041444-ntctbt.googlecode.com/svn/,60bd79275b267749c82086a5dcf62ccb87542b24 +http://1041448-datahiding-daily-practices.googlecode.com/svn/,17e58e8bf91086299f446cd54ffa1fa04d101e72 +http://1041448-ec-daily-practices.googlecode.com/svn/,c94ef2d6ae998f62620719fc9ecd1d71e84fa7a1 +http://1042003-project.googlecode.com/svn/,207f34f1c302e86539f62a92e9ccd8570bdc9607 +http://1042010-quanlythuvien.googlecode.com/svn/,e08cc99e69f67d4055740b675865c27d3eebd28d +http://1042034-tdhau-project1.googlecode.com/svn/,93be1d361b7a021ba951252cbaa82377c047ae1e +http://1042037-project.googlecode.com/svn/,20aca0ea527ced40aad5c4e92b6641795a4be7e1 +http://1042044-1042122-nmcnpm.googlecode.com/svn/,45d2e802ea508dff20bc4ceebfc47243b922c149 +http://1042044-nmcnpm.googlecode.com/svn/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://1042044-nmcnpm.googlecode.com/svn/,c6bac3637c83aa2c9837478730f7ded5841cf764 +http://1042053quanlynhasach.googlecode.com/svn/,48916743bfb1c4de36550ecc92468802527cbf31 +http://1042070-hcb10-do-an-nmcnpm.googlecode.com/svn/,f20be9537fd3926e6b1fc5913c73e8aa9d552705 +http://1042073quanlykhachsan-cnpm.googlecode.com/svn/,2b8ac155c88c862924d39d21a2fdfa6836666a62 +http://1042073quanlykhachsan.googlecode.com/svn/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://1042073quanlykhachsan.googlecode.com/svn/,71c59ec729c17a22d58dc664aa2c0b9fa43e449f +http://1042078-phanmemquanlythuvien.googlecode.com/svn/,9676dc4358a2b86c5253738185ba936f08f61a0d +http://10421016-nhap-mon-cong-nge-phan-mem.googlecode.com/svn/,ba6408ee48a5f40d81f62b622734af898337ccee +http://1042122-nmcnpm.googlecode.com/svn/,0b685396fbad534a3153e92128064d062647f7c6 +http://1042123svn.googlecode.com/svn/,9c06000fe5514ec263c5e295cc99c5bc193baea2 +http://1042133-quanlynhasach.googlecode.com/svn/,7a65111f16c75c48b044e54dc72b3e0a92e2a0ab +http://1042134nmcnpm.googlecode.com/svn/,bdefcafaf760639d7c81193bf8731735da8dd9e6 +http://104436vorgurak.googlecode.com/svn/,1a8893e6a86f444e8be8e7bda6cb34fb1735a00e +http://104436vorgurak.googlecode.com/svn/,e490e1bb6e979537d1e00ec8e58192887bbc15ab +http://104epms.googlecode.com/svn/,0fc69eb05c7dd2f41118fd313f8c0565674092e9 +http://104pmi-vvs.googlecode.com/svn/,439f5cb2b5f520dd0ba40eda2b6766834d2f2144 +http://10520184-khoa-luan.googlecode.com/svn/,7a0cbf1a6e1ebb857419a6fb9980a9b8de10a9b9 +http://105radioalarm.googlecode.com/svn/,ecd8721572278e4dfda912ec7fc284a130950c17 +http://10650251065124quanlyhocsinh.googlecode.com/svn/,3eada98a9b5015dfeb270c59117531cf86f579ef +http://10650931065130doan.googlecode.com/svn/,03a702783ecbdfec294ddaf7b67cd343020854b6 +http://1065170udql1.googlecode.com/svn/,33170262d60bcfc65553cb5afd845a8eeb6f776c +http://1069-uv1.googlecode.com/svn/,e397523828dc1440e3a4b39da604ce5f91bc730a +http://1071017tese.googlecode.com/svn/,e2859fdde03e90929668896257144c517bae6898 +http://1073s.googlecode.com/svn/,e0a92dae640bcd5eaff6d4d498ca362effb06a94 +http://108bubblefishcode-conradshun.googlecode.com/svn/,a9c8c967b46634adcd1f24e5b64f83a4f4ed8433 +http://108fantasy-tactics.googlecode.com/svn/,a5a2b9a438ca2e07388571dc7a1d21299b5fd5e7 +http://108km.googlecode.com/svn/,d8993d6c35bbd62e7d65949974d0f8666db41333 +http://108shiki.googlecode.com/svn/,31157fa2b49a37721bcfbf8456ba7c9fa9583007 +http://108ukulele.googlecode.com/svn/,723a5849c1b66d143ca011140430002f918e4882 +http://108wo.googlecode.com/svn/,d8bb7a2bd5dc8647e0e534fda501d044f553f041 +http://109a-cd-pros.googlecode.com/svn/,b7629bb3fe4e4e1573fb8fbcd6c1eaae2bf661ab +http://10aaa-10aaa.googlecode.com/svn/,d81e4a3a7f4830ba34e229a7fe82f609121abd3d +http://10ata3-crriculumdesign-schedulemanagementsystem.googlecode.com/svn/,b6279e696167f8ccc002ce0fa81247820037b1c1 +http://10bit1-group02-dsn.googlecode.com/svn/,38e3e35fd8e3fa942e9ca668833a5381cff7132c +http://10bit1-group02-pmproject.googlecode.com/svn/,48266b8d7e9ad0c6df267840f5f1749d0cbd5d48 +http://10c15-30.googlecode.com/svn/,a94e174d74d4ebdeeb66dd0866f58c817057d03a +http://10cader.googlecode.com/svn/,6d4feb79430a2411268f49c376bb509c8541f117 +http://10cit2-se-group2.googlecode.com/svn/,208db7a4b242ea116ac3e8ff216802eb89c3d2b2 +http://10cit2-se-project.googlecode.com/svn/,7660a8b136088e7ffe69e257928f413448ba27d4 +http://10cm.googlecode.com/svn/,6f1524bb9225f378da0705cd18f5d3914f006385 +http://10d-training-netweb2.googlecode.com/svn/,d9c89c1c071b53303614fc3d70164ca3fdf5fc78 +http://10demotest.googlecode.com/svn/,f629577cfd4bc26be60e936872f79624dbb7f31e +http://10fev.googlecode.com/svn/,dffa4d2acaa6db9984c66c69089d1e11cf0e0a65 +http://10gago.googlecode.com/svn/,822fb3a731c8e4acfc44d64887e94eb56cf1e1f2 +http://10gr636.googlecode.com/svn/,cd84cc18c93796379bc61ff27a8ef258e716ad69 +http://10guanxin.googlecode.com/svn/,1036853298dc83e8fa9e36033ad55f4fb1d5923b +http://10hc-ml-wolves.googlecode.com/svn/,905d9f99ff197bcd7d36da02bd671ab3f00c713e +http://10hca1-team4-qldv.googlecode.com/svn/,a3c92572817b66cd1e16cad64ed2c8ccec6604e1 +http://10hca2-cheer-pmtudien.googlecode.com/svn/,73f441bcd95d500b360b2654e492237489a30c89 +http://10hcb-project01.googlecode.com/svn/,ab12f72865c1b277abfdbff07a92be91295386f9 +http://10hcb-qlqtpm-nhom02-khtn-2012-dichvu.googlecode.com/svn/,7a6078b76c2222802151c3836b97578c2204b440 +http://10hcb1042092.googlecode.com/svn/,b87402d31c9aea454f3da1d0ebdd96fc5edbaf2f +http://10io.googlecode.com/svn/,c249f12522203995c2b2176c83f6796fc52773a0 +http://10kabonekasma.googlecode.com/svn/,671d356e8a232c8c5cd99562fd71dbba575b8c7d +http://10minutesmore.googlecode.com/svn/,ac82a8413d52538c5b62335998fb45a8d58b0bf6 +http://10neon-sim.googlecode.com/svn/,99a398c4d00a4651fbf57f9105ca9b930c3ff9d5 +http://10npsite.googlecode.com/svn/,f30674d27eb00b13bdb13cad95ecc27dfc97d371 +http://10pockets.googlecode.com/svn/,a91192bb8cd8d835990966a3d4fb598db465ec91 +http://10shen.googlecode.com/svn/,c29d1896e28616d251ddbc501aff0652635bbd07 +http://10tree.googlecode.com/svn/,7283907a0418dd5b5b9c53c2ab71a84d06eacd30 +http://10ui.googlecode.com/svn/,c0349ac01a5ac6116a542dd5544f7490fe8b58f9 +http://10unlununtelefonnumarasi.googlecode.com/svn/,269219bc5b03be57f4208c5eded0c3ce371ffbd1 +http://10v-programming-pgtk.googlecode.com/svn/,818a66c2287f917fce61d4bb78480d7ad2afc04c +http://10ve.googlecode.com/svn/,5202b3511917150e0829c8c7fd421269f03b4742 +http://10wb2.googlecode.com/svn/,3218a9f73c4518665633f81f9ae4e4c53d5a00f2 +http://10ybb.googlecode.com/svn/,136675c3dea2e6770a52710c557d97f8bcdfb0a7 +http://10zig.googlecode.com/svn/,52c71f12773c697760bad6acf03ca74f3f3f7a4d +http://11-building-wp-team.googlecode.com/svn/,51cd78b2efafef30c99253901141bef4bbbd8870 +http://110514-shift-respecstudios.googlecode.com/svn/,c13c3fe53c5426756716d018762c06171952ed84 +http://11111111f.googlecode.com/svn/,1c53c7a5356a6f7bdadd4632740563e2077c7073 +http://111111culeados.googlecode.com/svn/,1f2be69d881286584807a030ece130cb2bb419e1 +http://111111w-e-b-v-t-v-now11111111111111.googlecode.com/svn/,f47618e23cff448b876d5c22b855494c198f66f5 +http://111113ddd.googlecode.com/svn/,c77b389df2f28858a8eb70f7f9ffa8247856e65f +http://11111a.googlecode.com/svn/,215cb7474150bcea5717478ef72f4d632393e9d7 +http://1112-ti24-a1-ar.googlecode.com/svn/,da4df7afff90b833058e38a572e27ed931db7008 +http://1112372tuyen.googlecode.com/svn/,82a88d295193d2eff730a7e08b568b5aa3fd280f +http://1113916-ogre-unity-work.googlecode.com/svn/,d56d2c7b73152368dad7a78f47e50c19e880d935 +http://1114528-animation.googlecode.com/svn/,5ccdbc1de92dc3beda10fdec9a7ff8470843a977 +http://111fortest.googlecode.com/svn/,0bbb191c3c08391a4681b35948926ecf42d8edd3 +http://111forwork1111.googlecode.com/svn/,accff04bdf46290a37e7c2a0790e6a6defe55f66 +http://111p.googlecode.com/svn/,726d4708712904cf6929331e5fffb761ae62d451 +http://112pmocopuorg.googlecode.com/svn/,064a82d1ade1b1dcc22212ee7e11945d50ee02b6 +http://112simulator.googlecode.com/svn/,2f9fc835e5ea90f40de45d609e780f3e5a707662 +http://1138cl-sync.googlecode.com/svn/,3b2896c933ebbf65cd1d18646f8af0473fed82ca +http://1142008-1142076-1142085-ptexpress.googlecode.com/svn/,0ed67e5a0bf45050422c9ffeab5f9c56942c6f13 +http://1142041-1142056-quanlycuahangcongnghe.googlecode.com/svn/,822310190caedcefbb10499230ad4334850ded80 +http://1142076-pttkpm.googlecode.com/svn/,0f70412d6251b68b60ca6ab8830c7f3f75557e49 +http://114ch.googlecode.com/svn/,2dbd3c7d1e3396f171fb012643f4a9b429f32c67 +http://114getter.googlecode.com/svn/,17eb8e985001bad769130647fa4ece04144cb429 +http://114la.googlecode.com/svn/,d7765f76d31c808902f998ffff8f9d05de5adfd8 +http://114msn.googlecode.com/svn/,8bf8e646088970f8d20a65ac80de6f27cdc41b83 +http://115jailbreak.googlecode.com/svn/,b85de0fd8b7933dbb403cb92c61d21da1124b197 +http://1160scoutingapp.googlecode.com/svn/,a52be03b103458dedc9393db1c4b710b8fb8ee65 +http://1169b79237141bd732a66dfbd6e7e119.googlecode.com/svn/,ba990c9e6c9bfa5eaf53ee75c2cdb2bf012f2d17 +http://116trackmeet.googlecode.com/svn/,89cc1fe72c6f677b4fae25ce771303235106961f +http://117shiyanshi.googlecode.com/svn/,f3af14d8fdc4f12c912c94795f45367358d7c9bb +http://118-c-c-installer.googlecode.com/svn/,e96a825c3c21491ebb2d621fbe555375fe87b638 +http://11839806-cp2004-assignment2.googlecode.com/svn/,c20a1f83d1b92b9f55a2e1093ce1c3dee809c8f3 +http://1186se.googlecode.com/svn/,07402f6302e56b003fe4b72bad8cc52a3b0599aa +http://118channel.googlecode.com/svn/,a6bd7930d1e145e8081396e25fce6435d1b7aa14 +http://1191021127qlgv.googlecode.com/svn/,f26fbd639b216abc1b81708acc36e48a0ff8df58 +http://119797euler.googlecode.com/svn/,fb9fafd3e861341da633b289fe7330b2517e9a55 +http://11am-collatz-nortey.googlecode.com/svn/,36aea2ce4f84351dc8935f2487fe07d1a167108c +http://11be10rounds.googlecode.com/svn/,a8f49fb4dcffea5347d95d1f3bf157113ce5134c +http://11bg1repo.googlecode.com/svn/,c85fb48b9b123534947a77515d0d14aa9a4aea66 +http://11blog.googlecode.com/svn/,65ebb3cb22092a042239c27cb9fdaa6d870541ba +http://11bus.googlecode.com/svn/,85178ae5ff6cdadc61a92eb73153dfc9f50d6ade +http://11c11-1pr.googlecode.com/svn/,088f8c4fe7731b76140ee36916cb2bf6f3c164b1 +http://11cntt3.googlecode.com/svn/,6cfd8b5e4c51cfd61a7ed9f972742ce6f9994d89 +http://11dema.googlecode.com/svn/,87bcaeccba423236252bbbe6c1f8b15c28622317 +http://11hca-1team-001-152.googlecode.com/svn/,6dd928ae1256924f510de989a41045c00b3a8bfa +http://11hca-2013-httt.googlecode.com/svn/,748ab5d1521259d75be9dcf700426e83b2a2259a +http://11hca-ec.googlecode.com/svn/,28708490c9fc7433d62a9d83a449e8f212c14e6a +http://11hca1-banking-system.googlecode.com/svn/,ab616156329d47e5999dfa538a6fa57e93565b02 +http://11hca1-java-swing.googlecode.com/svn/,5adb7844f5c17ec8f4eb0a36eca3db47a44531f8 +http://11hca1-java-web.googlecode.com/svn/,0acdc0166a36f1744afd804a25afd564017dcc3b +http://11hca2ltnhandnkhanh.googlecode.com/svn/,639f1879c5d04927a17cde14a3c34033215dbd19 +http://11hcbookstoremanagement.googlecode.com/svn/,b293b8f1daf58a3b59f18f42cc776049723da5b7 +http://11hourband.googlecode.com/svn/,464e914d08f7e6ba6879d90babcf8b0b490a38b0 +http://11it3.googlecode.com/svn/,1385f8688e7651c54b4bf94f498bab989a646907 +http://11ok-yii.googlecode.com/svn/,49e75c973f80ea0aff28750444cbf963af0d8d05 +http://11th.googlecode.com/svn/,0167e033275902cc42adb95e39a0be8632e94264 +http://11th131.googlecode.com/svn/,4d9bc36516779d5c3190352c9b1d89c0db666e99 +http://11thnhom03.googlecode.com/svn/,5fba41d9fd494257a09b2d7dbaaa1e5aaa4112a4 +http://11thnhom3.googlecode.com/svn/,7d41bbf4bbf7c8ff4ba0a2948d978234ce41bf91 +http://11z-alhe-tsp.googlecode.com/svn/,11e1b0d324f126ff2d53b7d0957fdb5ff7d530b8 +http://11z-pdi-taiga.googlecode.com/svn/,4b798085db319f0197f10b1695db8c521ee1b5b4 +http://11z-wmh-sat.googlecode.com/svn/,432fe5089449e192567fbc5f5a2b41076fd2f903 +http://11z-zpr-netspy.googlecode.com/svn/,470a1a5347fcd77f71ad12f9f168cb7e677fad3f +http://12-05-2011-wharfland-project.googlecode.com/svn/,f69ebe2a1a1806bac64bb7adcf6fc2ae84494343 +http://12-ahhmq-chotenmien.googlecode.com/svn/,4c9875c8f785890fc91a6536ff511fdb6ff8a5eb +http://12-banner-manager.googlecode.com/svn/,789b9d834763062b58c1af27daea0ab688d1fe31 +http://12-descendants-daftar.googlecode.com/svn/,156b38f38d43a5aeda5e706d1050783e5ef09cfc +http://12-games-of-christmas.googlecode.com/svn/,d2ea1d1b453e310f318eb8a157467b265ad20ed1 +http://12-lthd-dack.googlecode.com/svn/,32764bdaa8a2052624cda69c18fe7cf8c92c5191 +http://12-mystep.googlecode.com/svn/,9fbffd576601571cb19906486785879c94d0a602 +http://12-oz-mouse.googlecode.com/svn/,0f551420e6c9f1222a8d87a4bf805d0011390a87 +http://12014orgcomp.googlecode.com/svn/,98c9cb8f6c0d2e91a40b5d8556b5e8530811eca8 +http://1205dsdsh.googlecode.com/svn/,4080e4db3516062d89517dec0f91f5a89af0801f +http://1205hcrdbs.googlecode.com/svn/,66bd4c7df192403582af22f7c179d5f910b4f977 +http://1205zjhy.googlecode.com/svn/,29bd60ab979dda5ecd6cfa677e2931014aeade09 +http://1208-daisy-cut.googlecode.com/svn/,468bd38234a0fa1903541697bc097ee8d0344487 +http://1208-ljec-icr.googlecode.com/svn/,d2115f395c6f8c5392f18842a3cb1fe28e40d426 +http://121-fractal-builder.googlecode.com/svn/,2fa2b0f7a272a8e66a6c7e0b8385f3d1846c0a30 +http://121212-doancuoiky.googlecode.com/svn/,b026b1cc8e85407befaf3c37302e774014878b46 +http://12130046-baitap.googlecode.com/svn/,0cc4a8f1ac1b2695e171aafe8cf435479e5d2fcf +http://12130086-test-googlecode-svn.googlecode.com/svn/,160ebd17bd671ddf6dc979d03b63d556af09065c +http://12130182-oo-project.googlecode.com/svn/,00d8cbd6b2636d82f565d7ad51d19c73792fb2ad +http://12130187-project.googlecode.com/svn/,1647257ba23441862d27e303bfb15a948a3ea969 +http://1215cpegestionprojet.googlecode.com/svn/,cf18950df61c1c3ba823ccbaef7de1ea930c3c22 +http://122group2.googlecode.com/svn/,072bfe9ee84a989bcdfb7a83c2ae26ab179604e9 +http://123-1-vovchenko.googlecode.com/svn/,e3558dbb79f195ac83cc8c18e807169fc8a8636a +http://123-3-ondar.googlecode.com/svn/,60f939b6464113c1c91d168eeaa2406baf2ec8f4 +http://123-3-orlov-maltsev.googlecode.com/svn/,041b73bb930ad92c651b10f7d2583193e8e42ca8 +http://123-3-project.googlecode.com/svn/,f151d25edfda4c824b11d557acd9de6c4689b8ce +http://123-3-rezaev-ivan.googlecode.com/svn/,631686d93ee224050c85b46ba4d5198b238b988d +http://123-alecseev.googlecode.com/svn/,70ed92bf11845e03922aac6f15069b8c2bd1672a +http://123-asdf.googlecode.com/svn/,1c30343e9b1ba5f3f6824733cc46d0ae6c27dacf +http://123-personal-page.googlecode.com/svn/,e62206db6e43d37bba6cf8551077859929e29b2a +http://123-taimur.googlecode.com/svn/,4914933b1d2c96a7c138544de0c0672e9d7a155a +http://12306-assistant.googlecode.com/svn/,e3acbe23fd2a68a4a56b7ed5c8e21e3f90e0d936 +http://12306-auto-refresh.googlecode.com/svn/,4f73e08032fcbaee761b4971082f442eec2ac5a7 diff --git a/docker/origin_client/config.yml b/docker/origin_client/config.yml new file mode 100644 --- /dev/null +++ b/docker/origin_client/config.yml @@ -0,0 +1,36 @@ +provenance: + storage: + cls: rabbitmq # client configuration + url: amqp://rabbitmq:5672/%2f + storage_config: + cls: postgresql + db: + host: storage-db + dbname: provenance + user: provenance + password: provenancepassword + batch_size: 10000 + prefetch_count: 100 + wait_min: 60 + wait_per_batch: 60 + + + archive: + #cls: graph + #url: http://graph.internal.softwareheritage.org:5009/graph + # cls: direct + # db: + # host: swh-storage-db + # port: 5432 + # dbname: swh + # user: guest + cls: api + storage: + cls: remote + # url: http://webapp.internal.staging.swh.network:5002/ + url: http://webapp1.internal.softwareheritage.org:5002 + + org_server: # origin provider + host: origin_server + port: 5555 + batch_size: 1 diff --git a/docker/origin_client/entrypoint.sh b/docker/origin_client/entrypoint.sh new file mode 100755 --- /dev/null +++ b/docker/origin_client/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +. /usr/local/bin/pyutils.sh + +. /src/venv/bin/activate + +setup_pip + +cd /src/swh-provenance +python swh/provenance/tools/origins/client.py ${NB_CLIENTS} diff --git a/docker/origin_server/config.yml b/docker/origin_server/config.yml new file mode 100644 --- /dev/null +++ b/docker/origin_server/config.yml @@ -0,0 +1,5 @@ +provenance: + org_server: # origin provider + host: localhost + port: 5555 + batch_size: 1 diff --git a/docker/origin_server/entrypoint.sh b/docker/origin_server/entrypoint.sh new file mode 100755 --- /dev/null +++ b/docker/origin_server/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +. /usr/local/bin/pyutils.sh + +. /src/venv/bin/activate + +setup_pip + +cd /src/swh-provenance || exit +python swh/provenance/tools/origins/server.py /origins.csv diff --git a/docker/postgresql/postgresql.conf b/docker/postgresql/postgresql.conf new file mode 100644 --- /dev/null +++ b/docker/postgresql/postgresql.conf @@ -0,0 +1,14 @@ +listen_addresses = '*' +max_connections = 1000 +shared_buffers = 128MB # min 128kB +dynamic_shared_memory_type = posix # the default is the first option +max_wal_size = 1GB +min_wal_size = 80MB +log_timezone = 'Etc/UTC' +datestyle = 'iso, mdy' +timezone = 'Etc/UTC' +lc_messages = 'en_US.utf8' # locale for system error message +lc_monetary = 'en_US.utf8' # locale for monetary formatting +lc_numeric = 'en_US.utf8' # locale for number formatting +lc_time = 'en_US.utf8' # locale for time formatting +default_text_search_config = 'pg_catalog.english' diff --git a/docker/pyutils.sh b/docker/pyutils.sh new file mode 100755 --- /dev/null +++ b/docker/pyutils.sh @@ -0,0 +1,28 @@ +#!/bin/bash + + +setup_pip () { + echo Using pip from $(which pip) + + if [[ -d /src ]] ; then + tmpdir=`mktemp -d` + pushd /src + for srcrepo in swh-* ; do + if [ -w $srcrepo ] + then + # Install package in editable mode if source directory is writable + pip install -e $srcrepo + else + # Source directories might not be writeable, but building them writes + # in-tree; so we're copying them to a location guaranteed to be writeable. + rsync -a --chmod=+w $srcrepo $tmpdir/ --exclude "*/__pycache__/" --exclude "*/.tox/" --exclude "*/.hypothesis/" + pip install $tmpdir/$srcrepo + fi + done + popd + rm -rf $tmpdir + fi + + echo Installed Python packages: + pip list +} diff --git a/docker/storage/config.yml b/docker/storage/config.yml new file mode 100644 --- /dev/null +++ b/docker/storage/config.yml @@ -0,0 +1,12 @@ +provenance: + rabbitmq: # remote storage server configuration + url: amqp://rabbitmq:5672/%2f + storage_config: + cls: postgresql + db: + host: storage-db + dbname: provenance + user: provenance + password: provenancepassword + batch_size: 10000 + prefetch_count: 100 diff --git a/docker/storage/entrypoint.sh b/docker/storage/entrypoint.sh new file mode 100755 --- /dev/null +++ b/docker/storage/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +. /usr/local/bin/pyutils.sh + +. /src/venv/bin/activate + +setup_pip + +cd /src/swh-provenance || exit + +cat < /tmp/start_storage.py +import swh.provenance.api.server +import os + +# the following methods uses environment variable SWH_CONFIG_FILENAME +# to retrieve the configuration file +server = swh.provenance.api.server.make_server_from_configfile() + +server.start() +while True: + try: + command = input("Enter EXIT to stop service: ") + if command.lower() == "exit": + break + except KeyboardInterrupt: + pass +server.stop() +EOF + +python /tmp/start_storage.py diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,5 @@ PyYAML types-click types-PyYAML +zmq + diff --git a/swh/provenance/api/client.py b/swh/provenance/api/client.py --- a/swh/provenance/api/client.py +++ b/swh/provenance/api/client.py @@ -353,7 +353,7 @@ properties: pika.spec.BasicProperties, body: bytes, ) -> None: - LOGGER.info( + LOGGER.debug( "Received message # %s from %s: %s", deliver.delivery_tag, properties.app_id, @@ -365,7 +365,7 @@ decode_data(body, extra_decoders=self.extra_type_decoders), ) ) - LOGGER.info("Acknowledging message %s", deliver.delivery_tag) + LOGGER.debug("Acknowledging message %s", deliver.delivery_tag) channel.basic_ack(delivery_tag=deliver.delivery_tag) def stop_consuming(self) -> None: diff --git a/swh/provenance/origin.py b/swh/provenance/origin.py --- a/swh/provenance/origin.py +++ b/swh/provenance/origin.py @@ -3,6 +3,10 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import logging + +from datetime import datetime + from itertools import islice from typing import Generator, Iterable, Iterator, List, Optional, Tuple @@ -14,8 +18,16 @@ from .interface import ProvenanceInterface from .model import OriginEntry, RevisionEntry + ORIGIN_DURATION_METRIC = "swh_provenance_origin_revision_layer_duration_seconds" +LOG_FORMAT = ( + "%(levelname) -10s %(asctime)s %(name) -30s %(funcName) " + "-35s %(lineno) -5d: %(message)s" +) + +LOGGER = logging.getLogger(__name__) + class CSVOriginIterator: """Iterator over origin visit statuses typically present in the given CSV @@ -57,11 +69,13 @@ if commit: provenance.flush() + @statsd.timed(metric=ORIGIN_DURATION_METRIC, tags={"method": "proceed_origin"}) def proceed_origin( - provenance: ProvenanceInterface, - archive: ArchiveInterface, - origin: OriginEntry) -> None: + provenance: ProvenanceInterface, archive: ArchiveInterface, origin: OriginEntry +) -> None: + LOGGER.info("Processing origin %s", origin.url) + start = datetime.now() provenance.origin_add(origin) origin.retrieve_revisions(archive) for revision in origin.revisions: @@ -71,6 +85,8 @@ # head is treated separately check_preferred_origin(provenance, origin, revision) provenance.revision_add_to_origin(origin, revision) + end = datetime.now() + LOGGER.info("Processed origin %s in %s", origin.url, (end - start)) @statsd.timed(metric=ORIGIN_DURATION_METRIC, tags={"method": "process_revision"}) diff --git a/swh/provenance/tools/origins/client.py b/swh/provenance/tools/origins/client.py --- a/swh/provenance/tools/origins/client.py +++ b/swh/provenance/tools/origins/client.py @@ -62,6 +62,9 @@ if __name__ == "__main__": + + logging.basicConfig(level=logging.INFO) + # Check parameters if len(sys.argv) != 2: print("usage: client ")