From 642344569724387f5f0067901341908e853a0b69 Mon Sep 17 00:00:00 2001 From: "xingjun.wxj" Date: Wed, 28 Dec 2022 19:59:02 +0800 Subject: [PATCH] [to #42322933] Modify object_exists in oss utils download. 1. Move self.bucket.object_exists to try-except block. CR Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11245983 --- modelscope/msdatasets/utils/oss_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modelscope/msdatasets/utils/oss_utils.py b/modelscope/msdatasets/utils/oss_utils.py index c16eab73..73e0996f 100644 --- a/modelscope/msdatasets/utils/oss_utils.py +++ b/modelscope/msdatasets/utils/oss_utils.py @@ -54,6 +54,7 @@ class OssUtilities: self.oss_backup_dir = oss_config[BACK_DIR] def _reload_sts(self): + logger.info('Reloading sts token automatically.') cookies = self.api.check_local_cookies(use_cookies=True) oss_config_refresh = self.api.get_dataset_access_config_session( cookies=cookies, @@ -72,14 +73,15 @@ class OssUtilities: cache_dir = download_config.cache_dir candidate_key = os.path.join(self.oss_dir, oss_file_name) candidate_key_backup = os.path.join(self.oss_backup_dir, oss_file_name) - file_oss_key = candidate_key if self.bucket.object_exists( - candidate_key) else candidate_key_backup - filename = hash_url_to_filename(file_oss_key, etag=None) - local_path = os.path.join(cache_dir, filename) retry_count = 0 while True: try: retry_count += 1 + file_oss_key = candidate_key if self.bucket.object_exists( + candidate_key) else candidate_key_backup + filename = hash_url_to_filename(file_oss_key, etag=None) + local_path = os.path.join(cache_dir, filename) + if download_config.force_download or not os.path.exists( local_path): oss2.resumable_download(