Mirrors/modelscope
1
0
Fork 0
mirror of https://github.com/modelscope/modelscope.git synced 2026-02-24 20:19:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

reuse _download function in DataDownloadManager

Browse Source
This commit is contained in:
xingjun.wang
2024-07-22 20:49:33 +08:00
parent b2157f1476
commit c2e954245d
1 changed files with 2 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
modelscope/msdatasets/download/download_manager.py
View File

@@ -39,22 +39,7 @@ class DataDownloadManager(DownloadManager):
def _download_single(self, url_or_filename: str,
download_config: DataDownloadConfig) -> str:
# Note: _download_single function is available for datasets>=2.19.0
url_or_filename = str(url_or_filename)
oss_utilities = OssUtilities(
oss_config=download_config.oss_config,
dataset_name=download_config.dataset_name,
namespace=download_config.namespace,
revision=download_config.version)
if is_relative_path(url_or_filename):
# fetch oss files
return oss_utilities.download(
url_or_filename, download_config=download_config)
else:
return cached_path(
url_or_filename, download_config=download_config)
return self._download(url_or_filename, download_config)
class DataStreamingDownloadManager(StreamingDownloadManager):
@@ -85,18 +70,4 @@ class DataStreamingDownloadManager(StreamingDownloadManager):
def _download_single(self, url_or_filename: str) -> str:
# Note: _download_single function is available for datasets>=2.19.0
url_or_filename = str(url_or_filename)
oss_utilities = OssUtilities(
oss_config=self.download_config.oss_config,
dataset_name=self.download_config.dataset_name,
namespace=self.download_config.namespace,
revision=self.download_config.version)
if is_relative_path(url_or_filename):
# fetch oss files
return oss_utilities.download(
url_or_filename, download_config=self.download_config)
else:
return cached_path(
url_or_filename, download_config=self.download_config)
return self._download(url_or_filename)