Page MenuHomeSoftware Heritage
Paste P1157

Revisions with non-empty metadata field and missing raw extrinsic metadata
ActivePublic

Authored by olasd on Sep 10 2021, 6:08 PM.
00959a167bd98452c98ce73382f4b42179d53d32
00a867beb2ad8e203f242e9843d2e88de0856cda
028e9890a9287b35851c48ca351641743542d030
030a51a49b3239769928872be9ac6d435ab14a61
036594a6bbec926c21fa073e2404a5f760d35a43
03fa7291482f35485fa455f9a7cf6d0bcbcace1f
0623371401858165a9e3514af4cbfe7caf82b65d
0739faafe2e0214537cc3c8da4a6d6619bb830d2
0892aca64fb993517fbd13b4aa97b223a783a419
08dd80bdaf5cf36f940865e8d7e0556cad6d881d
09c70e1fe774bce8d19874442f9da0b3fcd4d740
09e34df7727658fd5db298bb3d4dd862a68768e7
0b17032b3021e24424f8d5d2694b3cf207df6854
0d9502792997d32416f9b2e0582d6eb80185723b
0eeb19fa566bae4f23091c9c33691a3fb8352f09
0f3b78f59f495ac0305fdae2da2d2cda9dff8b40
147fe5dc37eb9d449adddae46230384a94c65d06
157d1e0d710efa5f031ca59a887fa7765c6029be
165ac1b0c61f6af73b379e3167c23e8e3b94ab27
1b1039243b8728148bcb6afc2b087f1ec5411aa9
1f010d11f10ecb13d93c3b923fc8c8994fd6ac75
23a059ba95a860e2c479c73703385c86f7156d0e
2431636ecb54fb0d0e340eb82e0abaf6ef3897d7
2704f8255c9d4adc7506467dc06342ba1a8c6605
29a1e5f81cba89d921263163edc28a3ad9454e3d
29b8bfb87fd7b905d442f980b534105aefeb8a98
2beca1dc7f0ecb0959ced56509c191fb65b79ae3
2c478e2d48e70b2bd5dbf53c271f9727511ab8b3
2db27bd5b2907bd8a71c437a972016e1480dc2d9
2e8bbe5ab4f49c6e9f73c995d5685d9b78d36b1e
3224373eacc16b7a755cf6bd54eb23aa46be8036
3662f0d1e781eaaa6796c4a639111d7b9a787163
37203894c8d185144f205653a7f83ceae6eef79e
378e1488c8185573fed7158c878d4307f49afaab
38b3070edfc018308a0bc3d9604d19d90c430c7c
38e4aacb06cb218ee7c2fa38ddb16611e426d7bb
3a54793134512d29a9ce7c55834ddf1513d7f5ef
3f3d73c171248c7a618c7fd1e7e00f27aa6dde56
4016b073c53a10af5224d0e2157318e89a17ab0f
441e0161e5d34652e165b1c30f693825c12f1555
45f358ea6b4d190565ad95b99ef11cfa95959a6e
4a2aa0329ffdbd213dd3347d5274d83f1464ffbc
4a35dea5ede1dde20759a0e0ae63221ae3751369
4a3f73c8dfc454fb092edd644b5d4f537dbe6640
4b9253484037f2298e4dda074376238ddccaa6f6
4e8b1e719b0e53a7340f5509f434e6583e5e799f
520d36f3209008fabed70f4e83f7013f6dc2dc2b
54ceee750aeacca14f9ea52ca4a882e9e3c84b7c
55fe1ddc6cf3da9e226a3942d85a61abdf678493
5924070d614f3a0dd922cbf91910d5aabfe88175
599bf01c3be645e15c2b8c2aa89baab2ae7eeac4
5eb0ff444bcf63b03fd988b226bddbd6de2d3570
6172a1a846066ecdf3b149788a301e379ee288da
635d8e6bddab09617a1177db04951e92854c9a80
636834a8d0d64f30e2d6eed059eca2b532f92c66
639891efbaffd03ddc9201dbde2c212bf54f1bb2
63b506fcc2b07d3b7aeea572563a4844210653ef
654cc54e85ba7c364604c5d3bcf7c4c7e012215a
656804ef7538a58fda8f5ce741a0fed16a58777d
69b3c7915fe7301980851946f8f8d32912359443
73fc2bcead5b9df8ffa5eed8c1470617e960b7c6
74a10bf05adc6f4535b7db2fd9be2355628bbf87
7a08f520a53383f05f22c638b6bc4cfdbd352963
7e880a6f5c226d839b0c339298f5f47fa7a55bef
7eaa58a336664e78e1b751ca60c601eeea0b5f67
7eb331b4cc47285182ccff7e6818bb13c8b34f14
7fe7e051a7ff284836491bb8a8cfa915a7478cf3
80a19f480ca9967bea97fc5e1f21b8d11516eea3
80a36c1be30d93882037c41cafc6a28722f22a29
83236283bcd339be2c35b6e234d9bc1cb1eefc57
83b748286904500c5d247d42e5f103b668b2f80b
86849c6b470fc2710890cfec58781e6c3aa0e691
869ca89e756335b67d30cae4f858126c67e1e51c
87cf980f134a1328f3b6071b14f48a0f825b4ab8
89de271b916b6c9b5d340e0903275ad83ba7b6bb
8a57614f9451b1040227cd77200aa21cd17a6292
8ad1170fabd745f8ee4f663d41a67223d8a7ec68
8b219f868812c3a0419af3090d3c5d2701d7c476
8b98238217883f78808120da8ee114c0b1c974b7
8e795c5936c3a726b8fa8fc929da950787163b8d
912c7e68ac172353345abfdfeee415cf5365f8ed
91e5ca8b274bf1a863466432d751f741bc94ba26
9293f230baca9814490d4fff7ac53d487a20edb6
9301fbf48fe190aa398a9ec2b0223390c5885649
95298db0e0b757d9b510b348612a3c45b1600239
96dbdb83e91e0d6448d73fc692c24afa42367d71
96dd7fa92ad082e4aaa1ffee6914bd4bd352379c
971dd3a7a67273188f578827acefe64ff8c93878
9757f44d2f10ddda89c385bba107ab7736928a37
9aa41ec772714414289196cfad01a500e833a402
9b0114f161e2a07a23ddf4eff671d7ead9183dbc
9b25eb8e137c1f75dab567de8b80c9db769756a8
9e55eaef4bc0a8792a91c76ae458a07fd8fbd209
a04d4020067ce9bcc29f484aa8720fc874db015a
a0afb4adebb39fbd1f8f6438f1711e7e45eb301a
a29ebe33b428a7b244d3f8eab47a92319f713d27
a64418908dc01f7dfd7a54236ea200f205ce2cd8
a86747d201ab8f8657d145df4376676d5e47cf9f
a99b51654ff1d3a1baf812884b0e7cad0480b32d
ac92325a1d88a5119e11f6d70471b059705d7315
af9e65359998703f95c18a875536371cd88a1408
afd1d15e2b8b3d8e2764f4e8e32913b7217cc837
b0ccbba73b07e574197a3f7f2c9f4a251a3dfafe
b28af13d94545643c82b419e406dd53643221e81
b32adbf284d6c7c144052dd07a55dbd552bc9e15
b32fb74d77862833a1859f22ccbd2773d0e96d9c
b34cca736c1acc1b9493062d08c7ec62823fc254
b3fcc439b2ccab7c6a8587b34b986a0264ba131f
b4b3122ae3471b1198fee3be3faeec642a8766bf
b6b6693f5a3a043f86630322d35e2b64014e7883
b897e4c3eb06d63126286f663caa3a1f00ed2e61
b9604526627a6f32b63965751b6e7f1d6978741d
bae76bd4313854ad7de15f40d1bf3ec3676ba62c
bc1324daeb266741fa42aad7571da4cc1cb70907
be8c352a8d907e4c60258edcacf7b6228778505d
bec53f0bb9b7813475478f706e1ee298b26b6c52
bef5b91d7c53e462ff51730addd580116c289428
bf8b9a4c5700b1e20feeb872f9a89ab5e32b1c35
c048cb0cd4b8b4fe927f1f286dc2a76ae2629c1a
c050f0c067b2679d822c8f30fa175d497492a93f
c089f2721507d57821236c2663641b9784d61d14
c1109437d11f6ec075183ed35559a5952751b36e
c129777084237de30147d3aa2c001454a3765711
c3dcbd13a068d36958f15dcb98ed0e26bdc93aa2
c4b556d3a7d707c5e3b36beeeaefb89c98fca647
c556025c848ab0f551fc4bb319cb6be9ffb02fdc
c618cd742be9c24143bf7ed7649e5f28c56e1077
c7390da9c649acf3b82e5575578a4bad78a535cb
c7943c73cb71a96a3265f1b534fcd6c0193babcb
c9f8332e4ddb5081adebd358ff6557cdd7d0d6e9
ca0ae3be0bbf089569624ebd622919accbc03fec
cc899c19a5aba277b19fe27b0a3535a991799940
cdcb753ddd9e71924a7ee2cd972e92aad6feda6a
cf648b44577e1a81ea7152dd792ebe325043f0ea
cfbb8897369f335994b45ed03d4b8ca79d2d3c8e
d10720ee210cde4f3763a757f1aa91a050e75e5c
d18d188c5acdadaecb7b3cb7de1e197bd8895bba
d2d955eaa7224fbda3a22a9d9bc5407595d06d9e
d33f003076c4bbf07d00b831eb63b248f7d26a86
d47c897db71b21051220c115e0d9b677f70f1869
d74342fc975c3bfd0e395f6869de8c0f9f655c84
d83a83f82672e0f8dbec18af292c4bb7bdd303e7
d8cef4dc07dbc644d75a1612d6d447328e716f08
df5a496571b0d11f1a47dc4da95b7c7f46432b31
e31bf591c25cb6af8adef69837d84d5f4e7bfbe8
e34fdc5075a46f8f51dd33eab95b8c8026d17cfc
e7442cf01ffb501bd4822bef9da2d03b5a5ad0a3
e7cdfa98bd5abfe667da653a18ab61e451dcd559
e8cef5f0ed1f4b0d9dfb32a3317b0742dd703ce5
ea386381040ef317598bb31e52d16e31c5c27eac
eaa573f6f909675afed9f4ff7e05e338d0196777
eb52758cf290dfa2d70fd5aebcce99d04bb33bff
f06d24a44654d9b2353de75ff9c2c86c3dd98e11
f16e4af2ae6aecec8e3b473241465dd35d7c4ad7
f1dce2612ec4b9de5371fe5f80950064caec0d95
f26a9a0ff6635c5ead320f163f3af3d106a47d06
f50ff93479de7fb33794dca9f3011df63c9897d7
f56b63c73ba2546a6a4ebf9209708e9157f6e5a9
f8f6d73a9aaf948f4053664d7719ff93587c93f7
fa45b34f09be6e009bcbd2d55df43b5c9c59c029
fa88df685a6c762b4e17abbe4c3fecbbcba9d51f
fc2dfb43b7e8bb83c68f09dcc4cc218fb221759b
fe25138d441b622cf2b687a9680e338045b34ef1
fe9e6e9d0703cce725e5317dcbbe24f0c167d25a
ffb2611b343ffbd02488e6afaa7f01e4c42c02d3
ffc6e5b911b96dbe1406b0b19c0f8220ae567103

Event Timeline

olasd changed the title of this paste from Command-Line Input to Revisions with non-empty metadata field and missing raw extrinsic metadata.Sep 10 2021, 6:09 PM

The remaining fixable issues should be addressed by D6241 and D6242.

Then, you can re-run the script with either of these:

  • cat /tmp/all.txt| xargs -I '{}' python3 swh/storage/migrate_extrinsic_metadata.py service=swh <url> service=swh-deposit '{}' 1
  • cat /tmp/all.txt| parallel -n 1 python3 swh/storage/migrate_extrinsic_metadata.py service=swh <url> service=swh-deposit '{}' 1

Then, run this query to make sure I did not miss anything:

select id, metadata from revision where
    ( (metadata->'extrinsic'->'provider') is null or ((metadata->'extrinsic'->'provider')::text not like '"https://cran.r-project.org/src/%'))
    and type != 'dsc'
and id in (
'\x00959a167bd98452c98ce73382f4b42179d53d32',
'\x00a867beb2ad8e203f242e9843d2e88de0856cda',
'\x028e9890a9287b35851c48ca351641743542d030',
'\x030a51a49b3239769928872be9ac6d435ab14a61',
'\x036594a6bbec926c21fa073e2404a5f760d35a43',
'\x03fa7291482f35485fa455f9a7cf6d0bcbcace1f',
'\x0623371401858165a9e3514af4cbfe7caf82b65d',
'\x0739faafe2e0214537cc3c8da4a6d6619bb830d2',
'\x0892aca64fb993517fbd13b4aa97b223a783a419',
'\x08dd80bdaf5cf36f940865e8d7e0556cad6d881d',
'\x09c70e1fe774bce8d19874442f9da0b3fcd4d740',
'\x09e34df7727658fd5db298bb3d4dd862a68768e7',
'\x0b17032b3021e24424f8d5d2694b3cf207df6854',
'\x0d9502792997d32416f9b2e0582d6eb80185723b',
'\x0eeb19fa566bae4f23091c9c33691a3fb8352f09',
'\x0f3b78f59f495ac0305fdae2da2d2cda9dff8b40',
'\x147fe5dc37eb9d449adddae46230384a94c65d06',
'\x157d1e0d710efa5f031ca59a887fa7765c6029be',
'\x165ac1b0c61f6af73b379e3167c23e8e3b94ab27',
'\x1b1039243b8728148bcb6afc2b087f1ec5411aa9',
'\x1f010d11f10ecb13d93c3b923fc8c8994fd6ac75',
'\x23a059ba95a860e2c479c73703385c86f7156d0e',
'\x2431636ecb54fb0d0e340eb82e0abaf6ef3897d7',
'\x2704f8255c9d4adc7506467dc06342ba1a8c6605',
'\x29a1e5f81cba89d921263163edc28a3ad9454e3d',
'\x29b8bfb87fd7b905d442f980b534105aefeb8a98',
'\x2beca1dc7f0ecb0959ced56509c191fb65b79ae3',
'\x2c478e2d48e70b2bd5dbf53c271f9727511ab8b3',
'\x2db27bd5b2907bd8a71c437a972016e1480dc2d9',
'\x2e8bbe5ab4f49c6e9f73c995d5685d9b78d36b1e',
'\x3224373eacc16b7a755cf6bd54eb23aa46be8036',
'\x3662f0d1e781eaaa6796c4a639111d7b9a787163',
'\x37203894c8d185144f205653a7f83ceae6eef79e',
'\x378e1488c8185573fed7158c878d4307f49afaab',
'\x38b3070edfc018308a0bc3d9604d19d90c430c7c',
'\x38e4aacb06cb218ee7c2fa38ddb16611e426d7bb',
'\x3a54793134512d29a9ce7c55834ddf1513d7f5ef',
'\x3f3d73c171248c7a618c7fd1e7e00f27aa6dde56',
'\x4016b073c53a10af5224d0e2157318e89a17ab0f',
'\x441e0161e5d34652e165b1c30f693825c12f1555',
'\x45f358ea6b4d190565ad95b99ef11cfa95959a6e',
'\x4a2aa0329ffdbd213dd3347d5274d83f1464ffbc',
'\x4a35dea5ede1dde20759a0e0ae63221ae3751369',
'\x4a3f73c8dfc454fb092edd644b5d4f537dbe6640',
'\x4b9253484037f2298e4dda074376238ddccaa6f6',
'\x4e8b1e719b0e53a7340f5509f434e6583e5e799f',
'\x520d36f3209008fabed70f4e83f7013f6dc2dc2b',
'\x54ceee750aeacca14f9ea52ca4a882e9e3c84b7c',
'\x55fe1ddc6cf3da9e226a3942d85a61abdf678493',
'\x5924070d614f3a0dd922cbf91910d5aabfe88175',
'\x599bf01c3be645e15c2b8c2aa89baab2ae7eeac4',
'\x5eb0ff444bcf63b03fd988b226bddbd6de2d3570',
'\x6172a1a846066ecdf3b149788a301e379ee288da',
'\x635d8e6bddab09617a1177db04951e92854c9a80',
'\x636834a8d0d64f30e2d6eed059eca2b532f92c66',
'\x639891efbaffd03ddc9201dbde2c212bf54f1bb2',
'\x63b506fcc2b07d3b7aeea572563a4844210653ef',
'\x654cc54e85ba7c364604c5d3bcf7c4c7e012215a',
'\x656804ef7538a58fda8f5ce741a0fed16a58777d',
'\x69b3c7915fe7301980851946f8f8d32912359443',
'\x73fc2bcead5b9df8ffa5eed8c1470617e960b7c6',
'\x74a10bf05adc6f4535b7db2fd9be2355628bbf87',
'\x7a08f520a53383f05f22c638b6bc4cfdbd352963',
'\x7e880a6f5c226d839b0c339298f5f47fa7a55bef',
'\x7eaa58a336664e78e1b751ca60c601eeea0b5f67',
'\x7eb331b4cc47285182ccff7e6818bb13c8b34f14',
'\x7fe7e051a7ff284836491bb8a8cfa915a7478cf3',
'\x80a19f480ca9967bea97fc5e1f21b8d11516eea3',
'\x80a36c1be30d93882037c41cafc6a28722f22a29',
'\x83236283bcd339be2c35b6e234d9bc1cb1eefc57',
'\x83b748286904500c5d247d42e5f103b668b2f80b',
'\x86849c6b470fc2710890cfec58781e6c3aa0e691',
'\x869ca89e756335b67d30cae4f858126c67e1e51c',
'\x87cf980f134a1328f3b6071b14f48a0f825b4ab8',
'\x89de271b916b6c9b5d340e0903275ad83ba7b6bb',
'\x8a57614f9451b1040227cd77200aa21cd17a6292',
'\x8ad1170fabd745f8ee4f663d41a67223d8a7ec68',
'\x8b219f868812c3a0419af3090d3c5d2701d7c476',
'\x8b98238217883f78808120da8ee114c0b1c974b7',
'\x8e795c5936c3a726b8fa8fc929da950787163b8d',
'\x912c7e68ac172353345abfdfeee415cf5365f8ed',
'\x91e5ca8b274bf1a863466432d751f741bc94ba26',
'\x9293f230baca9814490d4fff7ac53d487a20edb6',
'\x9301fbf48fe190aa398a9ec2b0223390c5885649',
'\x95298db0e0b757d9b510b348612a3c45b1600239',
'\x96dbdb83e91e0d6448d73fc692c24afa42367d71',
'\x96dd7fa92ad082e4aaa1ffee6914bd4bd352379c',
'\x971dd3a7a67273188f578827acefe64ff8c93878',
'\x9757f44d2f10ddda89c385bba107ab7736928a37',
'\x9aa41ec772714414289196cfad01a500e833a402',
'\x9b0114f161e2a07a23ddf4eff671d7ead9183dbc',
'\x9b25eb8e137c1f75dab567de8b80c9db769756a8',
'\x9e55eaef4bc0a8792a91c76ae458a07fd8fbd209',
'\xa04d4020067ce9bcc29f484aa8720fc874db015a',
'\xa0afb4adebb39fbd1f8f6438f1711e7e45eb301a',
'\xa29ebe33b428a7b244d3f8eab47a92319f713d27',
'\xa64418908dc01f7dfd7a54236ea200f205ce2cd8',
'\xa86747d201ab8f8657d145df4376676d5e47cf9f',
'\xa99b51654ff1d3a1baf812884b0e7cad0480b32d',
'\xac92325a1d88a5119e11f6d70471b059705d7315',
'\xaf9e65359998703f95c18a875536371cd88a1408',
'\xafd1d15e2b8b3d8e2764f4e8e32913b7217cc837',
'\xb0ccbba73b07e574197a3f7f2c9f4a251a3dfafe',
'\xb28af13d94545643c82b419e406dd53643221e81',
'\xb32adbf284d6c7c144052dd07a55dbd552bc9e15',
'\xb32fb74d77862833a1859f22ccbd2773d0e96d9c',
'\xb34cca736c1acc1b9493062d08c7ec62823fc254',
'\xb3fcc439b2ccab7c6a8587b34b986a0264ba131f',
'\xb4b3122ae3471b1198fee3be3faeec642a8766bf',
'\xb6b6693f5a3a043f86630322d35e2b64014e7883',
'\xb897e4c3eb06d63126286f663caa3a1f00ed2e61',
'\xb9604526627a6f32b63965751b6e7f1d6978741d',
'\xbae76bd4313854ad7de15f40d1bf3ec3676ba62c',
'\xbc1324daeb266741fa42aad7571da4cc1cb70907',
'\xbe8c352a8d907e4c60258edcacf7b6228778505d',
'\xbec53f0bb9b7813475478f706e1ee298b26b6c52',
'\xbef5b91d7c53e462ff51730addd580116c289428',
'\xbf8b9a4c5700b1e20feeb872f9a89ab5e32b1c35',
'\xc048cb0cd4b8b4fe927f1f286dc2a76ae2629c1a',
'\xc050f0c067b2679d822c8f30fa175d497492a93f',
'\xc089f2721507d57821236c2663641b9784d61d14',
'\xc1109437d11f6ec075183ed35559a5952751b36e',
'\xc129777084237de30147d3aa2c001454a3765711',
'\xc3dcbd13a068d36958f15dcb98ed0e26bdc93aa2',
'\xc4b556d3a7d707c5e3b36beeeaefb89c98fca647',
'\xc556025c848ab0f551fc4bb319cb6be9ffb02fdc',
'\xc618cd742be9c24143bf7ed7649e5f28c56e1077',
'\xc7390da9c649acf3b82e5575578a4bad78a535cb',
'\xc7943c73cb71a96a3265f1b534fcd6c0193babcb',
'\xc9f8332e4ddb5081adebd358ff6557cdd7d0d6e9',
'\xca0ae3be0bbf089569624ebd622919accbc03fec',
'\xcc899c19a5aba277b19fe27b0a3535a991799940',
'\xcdcb753ddd9e71924a7ee2cd972e92aad6feda6a',
'\xcf648b44577e1a81ea7152dd792ebe325043f0ea',
'\xcfbb8897369f335994b45ed03d4b8ca79d2d3c8e',
'\xd10720ee210cde4f3763a757f1aa91a050e75e5c',
'\xd18d188c5acdadaecb7b3cb7de1e197bd8895bba',
'\xd2d955eaa7224fbda3a22a9d9bc5407595d06d9e',
'\xd33f003076c4bbf07d00b831eb63b248f7d26a86',
'\xd47c897db71b21051220c115e0d9b677f70f1869',
'\xd74342fc975c3bfd0e395f6869de8c0f9f655c84',
'\xd83a83f82672e0f8dbec18af292c4bb7bdd303e7',
'\xd8cef4dc07dbc644d75a1612d6d447328e716f08',
'\xdf5a496571b0d11f1a47dc4da95b7c7f46432b31',
'\xe31bf591c25cb6af8adef69837d84d5f4e7bfbe8',
'\xe34fdc5075a46f8f51dd33eab95b8c8026d17cfc',
'\xe7442cf01ffb501bd4822bef9da2d03b5a5ad0a3',
'\xe7cdfa98bd5abfe667da653a18ab61e451dcd559',
'\xe8cef5f0ed1f4b0d9dfb32a3317b0742dd703ce5',
'\xea386381040ef317598bb31e52d16e31c5c27eac',
'\xeaa573f6f909675afed9f4ff7e05e338d0196777',
'\xeb52758cf290dfa2d70fd5aebcce99d04bb33bff',
'\xf06d24a44654d9b2353de75ff9c2c86c3dd98e11',
'\xf16e4af2ae6aecec8e3b473241465dd35d7c4ad7',
'\xf1dce2612ec4b9de5371fe5f80950064caec0d95',
'\xf26a9a0ff6635c5ead320f163f3af3d106a47d06',
'\xf50ff93479de7fb33794dca9f3011df63c9897d7',
'\xf56b63c73ba2546a6a4ebf9209708e9157f6e5a9',
'\xf8f6d73a9aaf948f4053664d7719ff93587c93f7',
'\xfa45b34f09be6e009bcbd2d55df43b5c9c59c029',
'\xfa88df685a6c762b4e17abbe4c3fecbbcba9d51f',
'\xfc2dfb43b7e8bb83c68f09dcc4cc218fb221759b',
'\xfe25138d441b622cf2b687a9680e338045b34ef1',
'\xfe9e6e9d0703cce725e5317dcbbe24f0c167d25a',
'\xffb2611b343ffbd02488e6afaa7f01e4c42c02d3',
'\xffc6e5b911b96dbe1406b0b19c0f8220ae567103'
);

That's the same set of ids as above, but the query also excludes two unfixable cases:

  • CRAN origins we lost (T2536)
  • Debian packages, as none of these seem to be in any snapshot