Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/schema.py
| Show First 20 Lines • Show All 148 Lines • ▼ Show 20 Lines | CREATE TABLE IF NOT EXISTS origin_visit ( | ||||
| date timestamp, | date timestamp, | ||||
| type text, | type text, | ||||
| status ascii, | status ascii, | ||||
| metadata text, | metadata text, | ||||
| snapshot blob, | snapshot blob, | ||||
| PRIMARY KEY ((origin), visit) | PRIMARY KEY ((origin), visit) | ||||
| ); | ); | ||||
| CREATE TABLE IF NOT EXISTS origin_visit_status ( | |||||
| origin text, | |||||
| visit bigint, | |||||
| date timestamp, | |||||
| status ascii, | |||||
| metadata text, | |||||
| snapshot blob, | |||||
| PRIMARY KEY ((origin), visit, date) | |||||
| ); | |||||
| CREATE TABLE IF NOT EXISTS origin ( | CREATE TABLE IF NOT EXISTS origin ( | ||||
| sha1 blob PRIMARY KEY, | sha1 blob PRIMARY KEY, | ||||
| url text, | url text, | ||||
| type text, | type text, | ||||
| next_visit_id int, | next_visit_id int, | ||||
| -- We need integer visit ids for compatibility with the pgsql | -- We need integer visit ids for compatibility with the pgsql | ||||
| -- storage, so we're using lightweight transactions with this trick: | -- storage, so we're using lightweight transactions with this trick: | ||||
| ▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | CREATE TABLE IF NOT EXISTS skipped_content_by_{main_algo} ( | ||||
| target_token bigint, -- value of token(pk) on the "primary" table | target_token bigint, -- value of token(pk) on the "primary" table | ||||
| PRIMARY KEY (({main_algo}), target_token) | PRIMARY KEY (({main_algo}), target_token) | ||||
| ); | ); | ||||
| """ | """ | ||||
| TABLES = ( | TABLES = ( | ||||
| "skipped_content content revision revision_parent release " | "skipped_content content revision revision_parent release " | ||||
| "directory directory_entry snapshot snapshot_branch " | "directory directory_entry snapshot snapshot_branch " | ||||
| "origin_visit origin tool_by_uuid tool object_count" | "origin_visit origin tool_by_uuid tool object_count " | ||||
| "origin_visit_status" | |||||
| ).split() | ).split() | ||||
| HASH_ALGORITHMS = ["sha1", "sha1_git", "sha256", "blake2s256"] | HASH_ALGORITHMS = ["sha1", "sha1_git", "sha256", "blake2s256"] | ||||
| for main_algo in HASH_ALGORITHMS: | for main_algo in HASH_ALGORITHMS: | ||||
| CREATE_TABLES_QUERIES.extend( | CREATE_TABLES_QUERIES.extend( | ||||
| CONTENT_INDEX_TEMPLATE.format( | CONTENT_INDEX_TEMPLATE.format( | ||||
| main_algo=main_algo, | main_algo=main_algo, | ||||
| other_algos=", ".join( | other_algos=", ".join( | ||||
| [algo for algo in HASH_ALGORITHMS if algo != main_algo] | [algo for algo in HASH_ALGORITHMS if algo != main_algo] | ||||
| ), | ), | ||||
| ).split("\n\n") | ).split("\n\n") | ||||
| ) | ) | ||||
| TABLES.append("content_by_%s" % main_algo) | TABLES.append("content_by_%s" % main_algo) | ||||
| TABLES.append("skipped_content_by_%s" % main_algo) | TABLES.append("skipped_content_by_%s" % main_algo) | ||||