Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/mercurial/loader_verifier.py
Show First 20 Lines • Show All 132 Lines • ▼ Show 20 Lines | def validate_tree(self, tree, header, i): | ||||
keys = [k['name'] for k in a['entries']] | keys = [k['name'] for k in a['entries']] | ||||
return b''.join(sorted(keys)) | return b''.join(sorted(keys)) | ||||
tree_dirs = [d for d in tree.yield_swh_directories()] | tree_dirs = [d for d in tree.yield_swh_directories()] | ||||
base_dirs = [d for d in base_tree.yield_swh_directories()] | base_dirs = [d for d in base_tree.yield_swh_directories()] | ||||
tree_dirs.sort(key=so1) | tree_dirs.sort(key=so1) | ||||
base_dirs.sort(key=so1) | base_dirs.sort(key=so1) | ||||
# for i in range(len(tree_dirs)): | |||||
# if tree_dirs[i] != base_dirs[i]: | |||||
# logging.debug(i) | |||||
# code.interact(local=dict(globals(), **locals())) | |||||
logging.debug('Program will quit after your next Ctrl-D') | logging.debug('Program will quit after your next Ctrl-D') | ||||
code.interact(local=dict(globals(), **locals())) | code.interact(local=dict(globals(), **locals())) | ||||
quit() | quit() | ||||
else: | else: | ||||
logging.debug('v') | logging.debug('v') | ||||
def generate_all_commits(self, validate=True, frequency=1): | def generate_all_commits(self, validate=True, frequency=1): | ||||
i = 0 | i = 0 | ||||
start = time.time() | start = time.time() | ||||
for rev in self.get_revisions(): | for rev in self.get_revisions(): | ||||
logging.debug('.', end='') | logging.debug('.', end='') | ||||
i += 1 | i += 1 | ||||
if i % 20 == 0: | if i % 20 == 0: | ||||
sys.stdout.flush() | sys.stdout.flush() | ||||
logging.debug('') | logging.debug('') | ||||
logging.debug('\nFOUND: %s COMMITS' % i) | logging.debug('\nFOUND: %s COMMITS' % i) | ||||
logging.debug('ELAPSED: %s' % (time.time()-start)) | logging.debug('ELAPSED: %s' % (time.time()-start)) | ||||
def runtest(self, hgdir, validate_blobs=False, validate_trees=False, | def runtest(self, hgdir, validate_blobs=False, validate_trees=False, | ||||
frequency=1.0, test_iterative=False): | frequency=1.0, test_iterative=False): | ||||
"""HgLoaderValidater().runtest('/home/avi/SWH/mozilla-unified') | """HgLoaderValidater().runtest('/home/avi/SWH/mozilla-unified') | ||||
anlambert: This comment is strange. | |||||
Done Inline ActionsTypo! ardumont: Typo! | |||||
""" | """ | ||||
self.origin_id = 'test' | self.origin_id = 'test' | ||||
dt = datetime.datetime.now(tz=datetime.timezone.utc) | dt = datetime.datetime.now(tz=datetime.timezone.utc) | ||||
if test_iterative: | if test_iterative: | ||||
dt = dt - datetime.timedelta(10) | dt = dt - datetime.timedelta(10) | ||||
hgrepo = None | hgrepo = None | ||||
if (hgdir.lower().startswith('http:') | if (hgdir.lower().startswith('http:') | ||||
or hgdir.lower().startswith('https:')): | or hgdir.lower().startswith('https:')): | ||||
hgrepo, hgdir = hgdir, hgrepo | hgrepo, hgdir = hgdir, hgrepo | ||||
self.hgdir = hgdir | self.hgdir = hgdir | ||||
try: | try: | ||||
logging.debug('preparing') | logging.debug('preparing') | ||||
self.prepare(origin_url=hgrepo, visit_date=dt, directory=hgdir) | self.prepare(origin_url=hgrepo, visit_date=dt, directory=hgdir) | ||||
self.file_node_to_hash = {} | self.file_node_to_hash = {} | ||||
# self.generate_all_blobs(validate=validate_blobs, | |||||
# frequency=frequency) | |||||
# self.generate_all_trees(validate=validate_trees, frequency=frequency) | |||||
# self.generate_all_commits() | |||||
logging.debug('getting contents') | logging.debug('getting contents') | ||||
cs = 0 | cs = 0 | ||||
for c in self.get_contents(): | for c in self.get_contents(): | ||||
cs += 1 | cs += 1 | ||||
pass | pass | ||||
logging.debug('getting directories') | logging.debug('getting directories') | ||||
ds = 0 | ds = 0 | ||||
for d in self.get_directories(): | for d in self.get_directories(): | ||||
ds += 1 | ds += 1 | ||||
pass | pass | ||||
revs = 0 | revs = 0 | ||||
logging.debug('getting revisions') | logging.debug('getting revisions') | ||||
for rev in self.get_revisions(): | for rev in self.get_revisions(): | ||||
revs += 1 | revs += 1 | ||||
pass | pass | ||||
logging.debug('getting releases') | logging.debug('getting releases') | ||||
rels = 0 | rels = 0 | ||||
for rel in self.get_releases(): | for rel in self.get_releases(): | ||||
rels += 1 | rels += 1 | ||||
logging.debug(rel) | logging.debug(rel) | ||||
self.visit = 'foo' | self.visit = 'foo' | ||||
logging.debug('getting snapshot') | logging.debug('getting snapshot') | ||||
o = self.get_snapshot() | o = self.get_snapshot() | ||||
Not Done Inline ActionsShouldn't we query the storage instead? vlorentz: Shouldn't we query the storage instead? | |||||
Done Inline ActionsI do not think so. Then again, I only ported the existing code. [1] IIRC, when i took other the loader mercurial, that did not work properly with the storage for example. ardumont: I do not think so.
As we do not call the loader's `load` function, because of history reasons i… | |||||
logging.debug('Snapshot: %s' % o) | logging.debug('Snapshot: %s' % o) | ||||
finally: | finally: | ||||
self.cleanup() | self.cleanup() | ||||
logging.info('final count: cs %s ds %s revs %s rels %s' % ( | logging.info('final count: cs %s ds %s revs %s rels %s' % ( | ||||
cs, ds, revs, rels)) | cs, ds, revs, rels)) | ||||
Show All 20 Lines |
This comment is strange.