Differential D4373 Diff 15544 swh/web/common/origin_save.py

Changeset View

Standalone View

swh/web/common/origin_save.py

Show First 20 Lines • Show All 104 Lines • ▼ Show 20 Lines
# map scheduler task status to origin save status		# map scheduler task status to origin save status
_save_task_status = {		_save_task_status = {
"next_run_not_scheduled": SAVE_TASK_NOT_YET_SCHEDULED,		"next_run_not_scheduled": SAVE_TASK_NOT_YET_SCHEDULED,
"next_run_scheduled": SAVE_TASK_SCHEDULED,		"next_run_scheduled": SAVE_TASK_SCHEDULED,
"completed": SAVE_TASK_SUCCEEDED,		"completed": SAVE_TASK_SUCCEEDED,
"disabled": SAVE_TASK_FAILED,		"disabled": SAVE_TASK_FAILED,
}		}

		# map scheduler task_run status to origin save status
		_save_task_run_status = {
		"scheduled": SAVE_TASK_SCHEDULED,
		"started": SAVE_TASK_RUNNING,
		"eventful": SAVE_TASK_SUCCEEDED,
		"uneventful": SAVE_TASK_SUCCEEDED,
		"failed": SAVE_TASK_FAILED,
		"permfailed": SAVE_TASK_FAILED,
		"lost": SAVE_TASK_FAILED,
		}


def get_savable_visit_types():		def get_savable_visit_types():
return sorted(list(_visit_type_task.keys()))		return sorted(list(_visit_type_task.keys()))


def _check_visit_type_savable(visit_type):		def _check_visit_type_savable(visit_type):
"""		"""
Get the list of visit types that can be performed		Get the list of visit types that can be performed
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines	else:
time_delta = time_now - save_request.request_date		time_delta = time_now - save_request.request_date
# consider the task as failed if it is still in scheduled state		# consider the task as failed if it is still in scheduled state
# 30 days after its submission		# 30 days after its submission
if time_delta.days > 30:		if time_delta.days > 30:
save_task_status = SAVE_TASK_FAILED		save_task_status = SAVE_TASK_FAILED
return visit_date, save_task_status		return visit_date, save_task_status


def _save_request_dict(save_request, task=None):		def _save_request_dict(save_request, task=None, task_run=None):
must_save = False		must_save = False
visit_date = save_request.visit_date		visit_date = save_request.visit_date
# save task still in scheduler db		# save task still in scheduler db
if task:		if task:
save_task_status = _save_task_status[task["status"]]		save_task_status = _save_task_status[task["status"]]
		if task_run:
		save_task_status = _save_task_run_status[task_run["status"]]
# Consider request from which a visit date has already been found		# Consider request from which a visit date has already been found
# as succeeded to avoid retrieving it again		# as succeeded to avoid retrieving it again
if save_task_status == SAVE_TASK_SCHEDULED and visit_date:		if save_task_status == SAVE_TASK_SCHEDULED and visit_date:
save_task_status = SAVE_TASK_SUCCEEDED		save_task_status = SAVE_TASK_SUCCEEDED
if (		if (
save_task_status in (SAVE_TASK_FAILED, SAVE_TASK_SUCCEEDED)		save_task_status in (SAVE_TASK_FAILED, SAVE_TASK_SUCCEEDED)
and not visit_date		and not visit_date
):		):
▲ Show 20 Lines • Show All 105 Lines • ▼ Show 20 Lines	if save_request_status == SAVE_REQUEST_ACCEPTED:
# and update the save request info in the database		# and update the save request info in the database
if sor.status == SAVE_REQUEST_PENDING:		if sor.status == SAVE_REQUEST_PENDING:
can_create_task = True		can_create_task = True
# a task has already been created to load the origin		# a task has already been created to load the origin
elif sor.loading_task_id != -1:		elif sor.loading_task_id != -1:
# get the scheduler task and its status		# get the scheduler task and its status
tasks = scheduler.get_tasks([sor.loading_task_id])		tasks = scheduler.get_tasks([sor.loading_task_id])
task = tasks[0] if tasks else None		task = tasks[0] if tasks else None
task_status = _save_request_dict(sor, task)["save_task_status"]		task_runs = scheduler.get_task_runs([sor.loading_task_id])
		task_run = task_runs[0] if task_runs else None
		save_request = _save_request_dict(sor, task, task_run)
		task_status = save_request["save_task_status"]
# create a new scheduler task only if the previous one has been		# create a new scheduler task only if the previous one has been
# already executed		# already executed
if (		if (
task_status == SAVE_TASK_FAILED		task_status == SAVE_TASK_FAILED
or task_status == SAVE_TASK_SUCCEEDED		or task_status == SAVE_TASK_SUCCEEDED
):		):
can_create_task = True		can_create_task = True
sor = None		sor = None
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines	def get_save_origin_requests_from_queryset(requests_queryset):
"""		"""
task_ids = []		task_ids = []
for sor in requests_queryset:		for sor in requests_queryset:
task_ids.append(sor.loading_task_id)		task_ids.append(sor.loading_task_id)
save_requests = []		save_requests = []
if task_ids:		if task_ids:
tasks = scheduler.get_tasks(task_ids)		tasks = scheduler.get_tasks(task_ids)
tasks = {task["id"]: task for task in tasks}		tasks = {task["id"]: task for task in tasks}
		task_runs = scheduler.get_task_runs(tasks)
		task_runs = {task_run["task"]: task_run for task_run in task_runs}
for sor in requests_queryset:		for sor in requests_queryset:
sr_dict = _save_request_dict(sor, tasks.get(sor.loading_task_id))		sr_dict = _save_request_dict(
		sor, tasks.get(sor.loading_task_id), task_runs.get(sor.loading_task_id)
		)
save_requests.append(sr_dict)		save_requests.append(sr_dict)
return save_requests		return save_requests


def get_save_origin_requests(visit_type, origin_url):		def get_save_origin_requests(visit_type, origin_url):
"""		"""
Get all save requests for a given software origin.		Get all save requests for a given software origin.

▲ Show 20 Lines • Show All 219 Lines • Show Last 20 Lines