Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2025-12-25 12:39:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

[to #42322933] Modify object_exists in oss utils download.

Browse Source 
1. Move self.bucket.object_exists to try-except block. 
        
CR Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/11245983
This commit is contained in:
xingjun.wxj
2022-12-28 19:59:02 +08:00
parent 0c79b57fcc
commit 6423445697
1 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
modelscope/msdatasets/utils/oss_utils.py
View File

@@ -54,6 +54,7 @@ class OssUtilities:
self.oss_backup_dir = oss_config[BACK_DIR]
def _reload_sts(self):
logger.info('Reloading sts token automatically.')
cookies = self.api.check_local_cookies(use_cookies=True)
oss_config_refresh = self.api.get_dataset_access_config_session(
cookies=cookies,
@@ -72,14 +73,15 @@ class OssUtilities:
cache_dir = download_config.cache_dir
candidate_key = os.path.join(self.oss_dir, oss_file_name)
candidate_key_backup = os.path.join(self.oss_backup_dir, oss_file_name)
file_oss_key = candidate_key if self.bucket.object_exists(
candidate_key) else candidate_key_backup
filename = hash_url_to_filename(file_oss_key, etag=None)
local_path = os.path.join(cache_dir, filename)
retry_count = 0
while True:
try:
retry_count += 1
file_oss_key = candidate_key if self.bucket.object_exists(
candidate_key) else candidate_key_backup
filename = hash_url_to_filename(file_oss_key, etag=None)
local_path = os.path.join(cache_dir, filename)
if download_config.force_download or not os.path.exists(
local_path):
oss2.resumable_download(