diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py index 2de69a44..8a7b8409 100644 --- a/modelscope/hub/api.py +++ b/modelscope/hub/api.py @@ -32,13 +32,16 @@ from requests.adapters import HTTPAdapter, Retry from requests.exceptions import HTTPError from tqdm.auto import tqdm -from modelscope.hub.constants import (API_HTTP_CLIENT_MAX_RETRIES, +from modelscope.hub.constants import (API_HTTP_CLIENT_CONNECT_TIMEOUT, + API_HTTP_CLIENT_MAX_RETRIES, API_HTTP_CLIENT_TIMEOUT, API_RESPONSE_FIELD_DATA, API_RESPONSE_FIELD_EMAIL, API_RESPONSE_FIELD_GIT_ACCESS_TOKEN, API_RESPONSE_FIELD_MESSAGE, API_RESPONSE_FIELD_USERNAME, + CREATE_TAG_MAX_RETRIES, + CREATE_TAG_RETRY_BACKOFF, DEFAULT_MAX_WORKERS, DEFAULT_MODELSCOPE_INTL_DOMAIN, MODELSCOPE_CLOUD_ENVIRONMENT, @@ -497,11 +500,46 @@ class HubApi: 'Ref': revision } - r = self.session.post( - path, - json=body, - cookies=cookies, - headers=self.builder_headers(self.headers)) + tag_timeout = (API_HTTP_CLIENT_CONNECT_TIMEOUT, + API_HTTP_CLIENT_TIMEOUT) + + retryable_status = {500, 502, 503, 504} + attempts = max(1, CREATE_TAG_MAX_RETRIES) + r = None + for attempt in range(1, attempts + 1): + retry_reason = None + try: + r = self.session.post( + path, + json=body, + cookies=cookies, + headers=self.builder_headers(self.headers), + timeout=tag_timeout) + except (requests.exceptions.ReadTimeout, + requests.exceptions.ConnectTimeout, + requests.exceptions.ConnectionError) as e: + if attempt >= attempts: + logger.error( + f'create_model_tag POST failed after {attempts} ' + f'attempt(s) due to transient network error: {e}. ' + f'Consider raising MODELSCOPE_API_HTTP_CLIENT_TIMEOUT ' + f'(current={API_HTTP_CLIENT_TIMEOUT}s) or ' + f'MODELSCOPE_CREATE_TAG_MAX_RETRIES ' + f'(current={CREATE_TAG_MAX_RETRIES}).') + raise + retry_reason = f'{type(e).__name__}: {e}' + else: + if r.status_code in retryable_status and attempt < attempts: + retry_reason = (f'retryable HTTP {r.status_code} ' + f'from server') + else: + break + + sleep_s = CREATE_TAG_RETRY_BACKOFF * (2 ** (attempt - 1)) + logger.warning( + f'create_model_tag POST attempt {attempt}/{attempts} ' + f'failed with {retry_reason}. Retrying in {sleep_s}s...') + time.sleep(sleep_s) raise_for_http_status(r) d = r.json() diff --git a/modelscope/hub/constants.py b/modelscope/hub/constants.py index ae6a5d72..85ad69d4 100644 --- a/modelscope/hub/constants.py +++ b/modelscope/hub/constants.py @@ -19,10 +19,23 @@ DEFAULT_CREDENTIALS_PATH = Path.home().joinpath('.modelscope', 'credentials') MODELSCOPE_CREDENTIALS_PATH = os.environ.get( 'MODELSCOPE_CREDENTIALS_PATH', DEFAULT_CREDENTIALS_PATH.as_posix()) REQUESTS_API_HTTP_METHOD = ['get', 'head', 'post', 'put', 'patch', 'delete'] -API_HTTP_CLIENT_TIMEOUT = 60 +# Default per-socket timeout (seconds) applied to all session HTTP methods +# in HubApi.__init__. User-tunable via env to mitigate transient ReadTimeout +# on heavy server-side ops (e.g. create_model_tag which performs git ops on +# the remote repo). Default raised to 90 to mitigate transient timeouts. +API_HTTP_CLIENT_TIMEOUT = int( + os.environ.get('MODELSCOPE_API_HTTP_CLIENT_TIMEOUT', 90)) + +API_HTTP_CLIENT_CONNECT_TIMEOUT = int( + os.environ.get('MODELSCOPE_API_HTTP_CLIENT_CONNECT_TIMEOUT', 10)) API_HTTP_CLIENT_MAX_RETRIES = int( os.environ.get('API_HTTP_CLIENT_MAX_RETRIES', 5)) +CREATE_TAG_MAX_RETRIES = int( + os.environ.get('MODELSCOPE_CREATE_TAG_MAX_RETRIES', 3)) +CREATE_TAG_RETRY_BACKOFF = int( + os.environ.get('MODELSCOPE_CREATE_TAG_RETRY_BACKOFF', 2)) + # Application-level retry for blob upload UPLOAD_BLOB_MAX_RETRIES = int(os.environ.get('UPLOAD_BLOB_MAX_RETRIES', 5)) UPLOAD_BLOB_RETRY_BACKOFF = int(os.environ.get('UPLOAD_BLOB_RETRY_BACKOFF', 2))