From c67ffd4cbccdc1421c37709efade406862ab9a3c Mon Sep 17 00:00:00 2001 From: Yingda Chen Date: Wed, 4 Dec 2024 18:25:52 +0800 Subject: [PATCH] optional hash --- modelscope/hub/constants.py | 1 + modelscope/hub/utils/caching.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modelscope/hub/constants.py b/modelscope/hub/constants.py index cfc6d6da..b3d03e1a 100644 --- a/modelscope/hub/constants.py +++ b/modelscope/hub/constants.py @@ -29,6 +29,7 @@ API_RESPONSE_FIELD_MESSAGE = 'Message' MODELSCOPE_CLOUD_ENVIRONMENT = 'MODELSCOPE_ENVIRONMENT' MODELSCOPE_CLOUD_USERNAME = 'MODELSCOPE_USERNAME' MODELSCOPE_SDK_DEBUG = 'MODELSCOPE_SDK_DEBUG' +MODELSCOPE_ENABLE_DEFAULT_HASH_VALIDATION = 'MODELSCOPE_ENABLE_DEFAULT_HASH_VALIDATION' ONE_YEAR_SECONDS = 24 * 365 * 60 * 60 MODELSCOPE_REQUEST_ID = 'X-Request-ID' TEMPORARY_FOLDER_NAME = '._____temp' diff --git a/modelscope/hub/utils/caching.py b/modelscope/hub/utils/caching.py index 675d62a8..ed1d9c67 100644 --- a/modelscope/hub/utils/caching.py +++ b/modelscope/hub/utils/caching.py @@ -12,6 +12,10 @@ from modelscope.hub.utils.utils import compute_hash from modelscope.utils.logger import get_logger logger = get_logger() + +enable_default_hash_validation = \ + os.getenv(MODELSCOPE_ENABLE_DEFAULT_HASH_VALIDATION, 'False').strip().lower() == 'true' + """Implements caching functionality, used internally only """ @@ -274,7 +278,11 @@ class ModelFileSystemCache(FileSystemCache): expected_hash = model_file_info[FILE_HASH] if expected_hash is not None and os.path.exists( cache_file_path): - cache_file_sha256 = compute_hash(cache_file_path) + # compute hash only when enabled, otherwise just meet expectation by default + if enable_default_hash_validation: + cache_file_sha256 = compute_hash(cache_file_path) + else: + cache_file_sha256 = expected_hash if expected_hash == cache_file_sha256: is_exists = True break