Changeset View
Changeset View
Standalone View
Standalone View
swh/web/common/origin_save.py
| Show First 20 Lines • Show All 132 Lines • ▼ Show 20 Lines | _save_task_run_status = { | ||||
| "eventful": SAVE_TASK_SUCCEEDED, | "eventful": SAVE_TASK_SUCCEEDED, | ||||
| "uneventful": SAVE_TASK_SUCCEEDED, | "uneventful": SAVE_TASK_SUCCEEDED, | ||||
| "failed": SAVE_TASK_FAILED, | "failed": SAVE_TASK_FAILED, | ||||
| "permfailed": SAVE_TASK_FAILED, | "permfailed": SAVE_TASK_FAILED, | ||||
| "lost": SAVE_TASK_FAILED, | "lost": SAVE_TASK_FAILED, | ||||
| } | } | ||||
| def get_savable_visit_types_dict(privileged_user: bool = False) -> Dict: | |||||
| """Returned the supported task types the user has access to. | |||||
| Args: | |||||
| privileged_user: Flag to determine if all visit types should be returned or not. | |||||
| Default to False to only list unprivileged visit types. | |||||
| Returns: | |||||
| the dict of supported visit types for the user | |||||
| """ | |||||
| if privileged_user: | |||||
| task_types = {**_visit_type_task, **_visit_type_task_privileged} | |||||
| else: | |||||
| task_types = _visit_type_task | |||||
| return task_types | |||||
| def get_savable_visit_types(privileged_user: bool = False) -> List[str]: | def get_savable_visit_types(privileged_user: bool = False) -> List[str]: | ||||
| """Get the list of visit types that can be performed through a save request. | """Return the list of visit types the user can perform save requests on. | ||||
| Args: | Args: | ||||
| privileged_user: Flag to determine if all visit types should be returned or not. | privileged_user: Flag to determine if all visit types should be returned or not. | ||||
| Default to False to only list unprivileged visit types. | Default to False to only list unprivileged visit types. | ||||
| Returns: | Returns: | ||||
| the list of saveable visit types | the list of saveable visit types | ||||
| """ | """ | ||||
| task_types = list(_visit_type_task.keys()) | |||||
| if privileged_user: | |||||
| task_types += _visit_type_task_privileged.keys() | |||||
| return sorted(task_types) | return sorted(list(get_savable_visit_types_dict(privileged_user).keys())) | ||||
| def _check_visit_type_savable(visit_type: str, privileged_user: bool = False) -> None: | def _check_visit_type_savable(visit_type: str, privileged_user: bool = False) -> None: | ||||
| visit_type_tasks = get_savable_visit_types(privileged_user) | visit_type_tasks = get_savable_visit_types(privileged_user) | ||||
| if visit_type not in visit_type_tasks: | if visit_type not in visit_type_tasks: | ||||
| allowed_visit_types = ", ".join(visit_type_tasks) | allowed_visit_types = ", ".join(visit_type_tasks) | ||||
| raise BadInputExc( | raise BadInputExc( | ||||
| f"Visit of type {visit_type} can not be saved! " | f"Visit of type {visit_type} can not be saved! " | ||||
| Show All 20 Lines | def origin_exists(origin_url: str) -> OriginExistenceCheckInfo: | ||||
| """ | """ | ||||
| resp = requests.head(origin_url) | resp = requests.head(origin_url) | ||||
| exists = resp.ok | exists = resp.ok | ||||
| content_length: Optional[int] = None | content_length: Optional[int] = None | ||||
| last_modified: Optional[str] = None | last_modified: Optional[str] = None | ||||
| if exists: | if exists: | ||||
| size_ = resp.headers.get("Content-Length") | size_ = resp.headers.get("Content-Length") | ||||
| content_length = int(size_) if size_ else None | content_length = int(size_) if size_ else None | ||||
| last_modified = resp.headers.get("Last-Modified") | try: | ||||
| date_str = resp.headers["Last-Modified"] | |||||
| date = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %Z") | |||||
| last_modified = date.isoformat() | |||||
| except (KeyError, ValueError): | |||||
| # if not provided or not parsable as per the expected format, keep it None | |||||
| pass | |||||
| return OriginExistenceCheckInfo( | return OriginExistenceCheckInfo( | ||||
| origin_url=origin_url, | origin_url=origin_url, | ||||
| exists=exists, | exists=exists, | ||||
| last_modified=last_modified, | last_modified=last_modified, | ||||
| content_length=content_length, | content_length=content_length, | ||||
| ) | ) | ||||
| def _check_origin_exists(origin_url: str) -> None: | def _check_origin_exists(origin_url: Optional[str]) -> OriginExistenceCheckInfo: | ||||
| """Ensure the origin exists, if not raise an explicit message.""" | """Ensure the origin exists, if not raise an explicit message.""" | ||||
| check = origin_exists(origin_url) | if not origin_url: | ||||
| if not check["exists"]: | raise BadInputExc("The origin url provided must be set!") | ||||
| metadata = origin_exists(origin_url) | |||||
| if not metadata["exists"]: | |||||
| raise BadInputExc( | raise BadInputExc( | ||||
| f"The provided origin url ({escape(origin_url)}) does not exist!" | f"The provided origin url ({escape(origin_url)}) does not exist!" | ||||
| ) | ) | ||||
| return metadata | |||||
| def _get_visit_info_for_save_request( | def _get_visit_info_for_save_request( | ||||
| save_request: SaveOriginRequest, | save_request: SaveOriginRequest, | ||||
| ) -> Tuple[Optional[datetime], Optional[str]]: | ) -> Tuple[Optional[datetime], Optional[str]]: | ||||
| """Retrieve visit information out of a save request | """Retrieve visit information out of a save request | ||||
| Args: | Args: | ||||
| save_request: Input save origin request to retrieve information for. | save_request: Input save origin request to retrieve information for. | ||||
| ▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines | ) -> SaveOriginRequestInfo: | ||||
| return save_request.to_dict() | return save_request.to_dict() | ||||
| def create_save_origin_request( | def create_save_origin_request( | ||||
| visit_type: str, | visit_type: str, | ||||
| origin_url: str, | origin_url: str, | ||||
| privileged_user: bool = False, | privileged_user: bool = False, | ||||
| user_id: Optional[int] = None, | user_id: Optional[int] = None, | ||||
| **kwargs, | |||||
| ) -> SaveOriginRequestInfo: | ) -> SaveOriginRequestInfo: | ||||
| """Create a loading task to save a software origin into the archive. | """Create a loading task to save a software origin into the archive. | ||||
| This function aims to create a software origin loading task | This function aims to create a software origin loading task trough the use of the | ||||
| trough the use of the swh-scheduler component. | swh-scheduler component. | ||||
| First, some checks are performed to see if the visit type and origin | First, some checks are performed to see if the visit type and origin url are valid | ||||
| url are valid but also if the the save request can be accepted. | but also if the the save request can be accepted. For the 'bundle' visit type, this | ||||
| If those checks passed, the loading task is then created. | also ensures the artifacts actually exists. If those checks passed, the loading task | ||||
| Otherwise, the save request is put in pending or rejected state. | is then created. Otherwise, the save request is put in pending or rejected state. | ||||
| All the submitted save requests are logged into the swh-web | All the submitted save requests are logged into the swh-web database to keep track | ||||
| database to keep track of them. | of them. | ||||
| Args: | Args: | ||||
| visit_type: the type of visit to perform (e.g git, hg, svn, ...) | visit_type: the type of visit to perform (e.g. git, hg, svn, bundle, ...) | ||||
| origin_url: the url of the origin to save | origin_url: the url of the origin to save | ||||
| privileged_user: Whether the user has privileged_user access to extra | privileged: Whether the user has some more privilege than other (bypass | ||||
| functionality (e.g. bypass save code now review, access to extra visit type) | review, access to privileged other visit types) | ||||
| user_id: User identifier (provided when authenticated) | user_id: User identifier (provided when authenticated) | ||||
| kwargs: Optional parameters (e.g. artifact_url, artifact_filename, | |||||
| artifact_version) | |||||
| Raises: | Raises: | ||||
| BadInputExc: the visit type or origin url is invalid or inexistent | BadInputExc: the visit type or origin url is invalid or inexistent | ||||
| ForbiddenExc: the provided origin url is blacklisted | ForbiddenExc: the provided origin url is blacklisted | ||||
| Returns: | Returns: | ||||
| dict: A dict describing the save request with the following keys: | dict: A dict describing the save request with the following keys: | ||||
| * **visit_type**: the type of visit to perform | * **visit_type**: the type of visit to perform | ||||
| * **origin_url**: the url of the origin | * **origin_url**: the url of the origin | ||||
| * **save_request_date**: the date the request was submitted | * **save_request_date**: the date the request was submitted | ||||
| * **save_request_status**: the request status, either **accepted**, | * **save_request_status**: the request status, either **accepted**, | ||||
| **rejected** or **pending** | **rejected** or **pending** | ||||
| * **save_task_status**: the origin loading task status, either | * **save_task_status**: the origin loading task status, either | ||||
| **not created**, **not yet scheduled**, **scheduled**, | **not created**, **not yet scheduled**, **scheduled**, | ||||
| **succeed** or **failed** | **succeed** or **failed** | ||||
| """ | """ | ||||
| visit_type_tasks = get_savable_visit_types_dict(privileged_user) | |||||
| _check_visit_type_savable(visit_type, privileged_user) | _check_visit_type_savable(visit_type, privileged_user) | ||||
| _check_origin_url_valid(origin_url) | _check_origin_url_valid(origin_url) | ||||
| artifact_url = kwargs.get("artifact_url") | |||||
| if visit_type == "bundle": | |||||
| metadata = _check_origin_exists(artifact_url) | |||||
| # if all checks passed so far, we can try and save the origin | # if all checks passed so far, we can try and save the origin | ||||
| save_request_status = can_save_origin(origin_url, privileged_user) | save_request_status = can_save_origin(origin_url, privileged_user) | ||||
| task = None | task = None | ||||
| # if the origin save request is accepted, create a scheduler | # if the origin save request is accepted, create a scheduler | ||||
| # task to load it into the archive | # task to load it into the archive | ||||
| if save_request_status == SAVE_REQUEST_ACCEPTED: | if save_request_status == SAVE_REQUEST_ACCEPTED: | ||||
| # create a task with high priority | # create a task with high priority | ||||
| kwargs = { | task_kwargs: Dict[str, Any] = { | ||||
| "priority": "high", | "priority": "high", | ||||
| "url": origin_url, | "url": origin_url, | ||||
| } | } | ||||
| if visit_type == "bundle": | |||||
| # extra arguments for that type are required | |||||
| assert metadata is not None | |||||
| task_kwargs = dict( | |||||
| **task_kwargs, | |||||
| artifacts=[ | |||||
| { | |||||
| "url": artifact_url, | |||||
| "filename": kwargs["artifact_filename"], | |||||
| "version": kwargs["artifact_version"], | |||||
| "time": metadata["last_modified"], | |||||
| "length": metadata["content_length"], | |||||
| } | |||||
| ], | |||||
| ) | |||||
| sor = None | sor = None | ||||
| # get list of previously sumitted save requests | # get list of previously sumitted save requests | ||||
| current_sors = list( | current_sors = list( | ||||
| SaveOriginRequest.objects.filter( | SaveOriginRequest.objects.filter( | ||||
| visit_type=visit_type, origin_url=origin_url | visit_type=visit_type, origin_url=origin_url | ||||
| ) | ) | ||||
| ) | ) | ||||
| Show All 25 Lines | if save_request_status == SAVE_REQUEST_ACCEPTED: | ||||
| ): | ): | ||||
| can_create_task = True | can_create_task = True | ||||
| sor = None | sor = None | ||||
| else: | else: | ||||
| can_create_task = False | can_create_task = False | ||||
| if can_create_task: | if can_create_task: | ||||
| # effectively create the scheduler task | # effectively create the scheduler task | ||||
| task_dict = create_oneshot_task_dict(_visit_type_task[visit_type], **kwargs) | task_dict = create_oneshot_task_dict( | ||||
| visit_type_tasks[visit_type], **task_kwargs | |||||
| ) | |||||
| task = scheduler.create_tasks([task_dict])[0] | task = scheduler.create_tasks([task_dict])[0] | ||||
| # pending save request has been accepted | # pending save request has been accepted | ||||
| if sor: | if sor: | ||||
| sor.status = SAVE_REQUEST_ACCEPTED | sor.status = SAVE_REQUEST_ACCEPTED | ||||
| sor.loading_task_id = task["id"] | sor.loading_task_id = task["id"] | ||||
| sor.save() | sor.save() | ||||
| else: | else: | ||||
| sor = SaveOriginRequest.objects.create( | sor = SaveOriginRequest.objects.create( | ||||
| visit_type=visit_type, | visit_type=visit_type, | ||||
| origin_url=origin_url, | origin_url=origin_url, | ||||
| status=save_request_status, | status=save_request_status, | ||||
| loading_task_id=task["id"], | loading_task_id=task["id"], | ||||
| user_ids=f'"{user_id}"' if user_id else None, | user_ids=f'"{user_id}"' if user_id else None, | ||||
| ) | ) | ||||
| # save request must be manually reviewed for acceptation | # save request must be manually reviewed for acceptation | ||||
| elif save_request_status == SAVE_REQUEST_PENDING: | elif save_request_status == SAVE_REQUEST_PENDING: | ||||
| # check if there is already such a save request already submitted, | # check if there is already such a save request already submitted, | ||||
| # no need to add it to the database in that case | # no need to add it to the database in that case | ||||
| try: | try: | ||||
| sor = SaveOriginRequest.objects.get( | sor = SaveOriginRequest.objects.get( | ||||
| visit_type=visit_type, origin_url=origin_url, status=save_request_status | visit_type=visit_type, origin_url=origin_url, status=save_request_status | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 360 Lines • Show Last 20 Lines | |||||