From c2e954245d7c22a2b5decda6403fdfedc4b23ac4 Mon Sep 17 00:00:00 2001 From: "xingjun.wang" Date: Mon, 22 Jul 2024 20:49:33 +0800 Subject: [PATCH] reuse _download function in DataDownloadManager --- .../msdatasets/download/download_manager.py | 33 ++----------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/modelscope/msdatasets/download/download_manager.py b/modelscope/msdatasets/download/download_manager.py index 7e156ac3..5e36cdce 100644 --- a/modelscope/msdatasets/download/download_manager.py +++ b/modelscope/msdatasets/download/download_manager.py @@ -39,22 +39,7 @@ class DataDownloadManager(DownloadManager): def _download_single(self, url_or_filename: str, download_config: DataDownloadConfig) -> str: # Note: _download_single function is available for datasets>=2.19.0 - - url_or_filename = str(url_or_filename) - - oss_utilities = OssUtilities( - oss_config=download_config.oss_config, - dataset_name=download_config.dataset_name, - namespace=download_config.namespace, - revision=download_config.version) - - if is_relative_path(url_or_filename): - # fetch oss files - return oss_utilities.download( - url_or_filename, download_config=download_config) - else: - return cached_path( - url_or_filename, download_config=download_config) + return self._download(url_or_filename, download_config) class DataStreamingDownloadManager(StreamingDownloadManager): @@ -85,18 +70,4 @@ class DataStreamingDownloadManager(StreamingDownloadManager): def _download_single(self, url_or_filename: str) -> str: # Note: _download_single function is available for datasets>=2.19.0 - - url_or_filename = str(url_or_filename) - oss_utilities = OssUtilities( - oss_config=self.download_config.oss_config, - dataset_name=self.download_config.dataset_name, - namespace=self.download_config.namespace, - revision=self.download_config.version) - - if is_relative_path(url_or_filename): - # fetch oss files - return oss_utilities.download( - url_or_filename, download_config=self.download_config) - else: - return cached_path( - url_or_filename, download_config=self.download_config) + return self._download(url_or_filename)