Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-09-01 19:49:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

handle resuming download as retry (#1147)

Browse Source 
Co-authored-by: Yingda Chen <yingda.chen@alibaba-inc.com>
This commit is contained in:
Yingda Chen
2024-12-30 12:48:55 +08:00
committed by GitHub
parent 8f3cf81982
commit 391c6cdec9
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
modelscope/hub/file_download.py
View File

@@ -497,6 +497,8 @@ def http_get_model_file(
partial_length = 0
# download partial, continue download
if os.path.exists(temp_file_path):
# resuming from interrupted download is also considered as retry
has_retry = True
with open(temp_file_path, 'rb') as f:
partial_length = f.seek(0, io.SEEK_END)
progress.update(partial_length)
@@ -520,7 +522,9 @@ def http_get_model_file(
if chunk: # filter out keep-alive new chunks
progress.update(len(chunk))
f.write(chunk)
hash_sha256.update(chunk)
# hash would be discarded in retry case anyway
if not has_retry:
hash_sha256.update(chunk)
break
except Exception as e: # no matter what happen, we will retry.
has_retry = True
@@ -528,7 +532,6 @@ def http_get_model_file(
retry.sleep()
# if anything went wrong, we would discard the real-time computed hash and return None
return None if has_retry else hash_sha256.hexdigest()
logger.debug('storing %s in cache at %s', url, local_dir)
def http_get_file(