Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/schema.py
Show First 20 Lines • Show All 148 Lines • ▼ Show 20 Lines | CREATE TABLE IF NOT EXISTS origin_visit ( | ||||
date timestamp, | date timestamp, | ||||
type text, | type text, | ||||
status ascii, | status ascii, | ||||
metadata text, | metadata text, | ||||
snapshot blob, | snapshot blob, | ||||
PRIMARY KEY ((origin), visit) | PRIMARY KEY ((origin), visit) | ||||
); | ); | ||||
CREATE TABLE IF NOT EXISTS origin_visit_update ( | |||||
origin text, | |||||
visit bigint, | |||||
date timestamp, | |||||
status ascii, | |||||
metadata text, | |||||
snapshot blob, | |||||
PRIMARY KEY ((origin), visit, date) | |||||
); | |||||
CREATE TABLE IF NOT EXISTS origin ( | CREATE TABLE IF NOT EXISTS origin ( | ||||
sha1 blob PRIMARY KEY, | sha1 blob PRIMARY KEY, | ||||
url text, | url text, | ||||
type text, | type text, | ||||
next_visit_id int, | next_visit_id int, | ||||
-- We need integer visit ids for compatibility with the pgsql | -- We need integer visit ids for compatibility with the pgsql | ||||
-- storage, so we're using lightweight transactions with this trick: | -- storage, so we're using lightweight transactions with this trick: | ||||
▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | CREATE TABLE IF NOT EXISTS skipped_content_by_{main_algo} ( | ||||
target_token bigint, -- value of token(pk) on the "primary" table | target_token bigint, -- value of token(pk) on the "primary" table | ||||
PRIMARY KEY (({main_algo}), target_token) | PRIMARY KEY (({main_algo}), target_token) | ||||
); | ); | ||||
""" | """ | ||||
TABLES = ( | TABLES = ( | ||||
"skipped_content content revision revision_parent release " | "skipped_content content revision revision_parent release " | ||||
"directory directory_entry snapshot snapshot_branch " | "directory directory_entry snapshot snapshot_branch " | ||||
"origin_visit origin tool_by_uuid tool object_count" | "origin_visit origin tool_by_uuid tool object_count " | ||||
"origin_visit_update" | |||||
).split() | ).split() | ||||
HASH_ALGORITHMS = ["sha1", "sha1_git", "sha256", "blake2s256"] | HASH_ALGORITHMS = ["sha1", "sha1_git", "sha256", "blake2s256"] | ||||
for main_algo in HASH_ALGORITHMS: | for main_algo in HASH_ALGORITHMS: | ||||
CREATE_TABLES_QUERIES.extend( | CREATE_TABLES_QUERIES.extend( | ||||
CONTENT_INDEX_TEMPLATE.format( | CONTENT_INDEX_TEMPLATE.format( | ||||
main_algo=main_algo, | main_algo=main_algo, | ||||
other_algos=", ".join( | other_algos=", ".join( | ||||
[algo for algo in HASH_ALGORITHMS if algo != main_algo] | [algo for algo in HASH_ALGORITHMS if algo != main_algo] | ||||
), | ), | ||||
).split("\n\n") | ).split("\n\n") | ||||
) | ) | ||||
TABLES.append("content_by_%s" % main_algo) | TABLES.append("content_by_%s" % main_algo) | ||||
TABLES.append("skipped_content_by_%s" % main_algo) | TABLES.append("skipped_content_by_%s" % main_algo) |