[to #42794773] rename pydataset to msdataset

modelscope/datasets/__init__.py

@@ -0,0 +1 @@
+from .ms_dataset import MsDataset

modelscope/datasets/ms_dataset.py

@@ -10,8 +10,8 @@ from datasets.packaged_modules import _PACKAGED_DATASETS_MODULES
 from datasets.utils.file_utils import (is_relative_path,
                                        relative_to_absolute_path)
 
-from modelscope.pydatasets.config import MS_DATASETS_CACHE
-from modelscope.pydatasets.utils.ms_api import MsApi
+from modelscope.datasets.config import MS_DATASETS_CACHE
+from modelscope.datasets.utils.ms_api import MsApi
 from modelscope.utils.constant import Hubs
 from modelscope.utils.logger import get_logger
 
@@ -28,9 +28,9 @@ def format_list(para) -> List:
     return para
 
 
-class PyDataset:
+class MsDataset:
     _hf_ds = None  # holds the underlying HuggingFace Dataset
-    """A PyDataset backed by hugging face Dataset."""
+    """A MsDataset backed by hugging face Dataset."""
 
     def __init__(self, hf_ds: Dataset, target: Optional[str] = None):
         self._hf_ds = hf_ds
@@ -49,7 +49,7 @@ class PyDataset:
     @classmethod
     def from_hf_dataset(cls,
                         hf_ds: Dataset,
-                        target: str = None) -> Union[dict, 'PyDataset']:
+                        target: str = None) -> Union[dict, 'MsDataset']:
         if isinstance(hf_ds, Dataset):
             return cls(hf_ds, target)
         if len(hf_ds.keys()) == 1:
@@ -68,8 +68,8 @@ class PyDataset:
         data_files: Optional[Union[str, Sequence[str],
                                    Mapping[str, Union[str,
                                                       Sequence[str]]]]] = None
-    ) -> Union[dict, 'PyDataset']:
-        """Load a PyDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset.
+    ) -> Union[dict, 'MsDataset']:
+        """Load a MsDataset from the ModelScope Hub, Hugging Face Hub, urls, or a local dataset.
             Args:
 
                 dataset_name (str): Path or name of the dataset.
@@ -82,7 +82,7 @@ class PyDataset:
                 hub (Hubs, optional): When loading from a remote hub, where it is from
 
             Returns:
-                PyDataset (obj:`PyDataset`): PyDataset object for a certain dataset.
+                MsDataset (obj:`MsDataset`): MsDataset object for a certain dataset.
             """
         if hub == Hubs.huggingface:
             dataset = hf_load_dataset(
@@ -92,9 +92,9 @@ class PyDataset:
                 split=split,
                 data_dir=data_dir,
                 data_files=data_files)
-            return PyDataset.from_hf_dataset(dataset, target=target)
+            return MsDataset.from_hf_dataset(dataset, target=target)
         else:
-            return PyDataset._load_ms_dataset(
+            return MsDataset._load_ms_dataset(
                 dataset_name,
                 target=target,
                 subset_name=subset_name,
@@ -114,7 +114,7 @@ class PyDataset:
         data_files: Optional[Union[str, Sequence[str],
                                    Mapping[str, Union[str,
                                                       Sequence[str]]]]] = None
-    ) -> Union[dict, 'PyDataset']:
+    ) -> Union[dict, 'MsDataset']:
         if isinstance(dataset_name, str):
             use_hf = False
             if dataset_name in _PACKAGED_DATASETS_MODULES or os.path.isdir(dataset_name) or \
@@ -153,7 +153,7 @@ class PyDataset:
         else:
             raise TypeError('path must be a str or a list, but got'
                             f' {type(dataset_name)}')
-        return PyDataset.from_hf_dataset(dataset, target=target)
+        return MsDataset.from_hf_dataset(dataset, target=target)
 
     def to_torch_dataset_with_processors(
         self,

modelscope/datasets/utils/ms_api.py

@@ -4,8 +4,8 @@ from typing import Optional
 
 import requests
 
-from modelscope.pydatasets.config import (DOWNLOADED_DATASETS_PATH,
-                                          MS_HUB_ENDPOINT)
+from modelscope.datasets.config import (DOWNLOADED_DATASETS_PATH,
+                                        MS_HUB_ENDPOINT)
 from modelscope.utils.logger import get_logger
 
 logger = get_logger()

modelscope/hub/file_download.py

@@ -187,7 +187,7 @@ def get_file_download_url(model_id: str, file_path: str, revision: str):
     """
     Format file download url according to `model_id`, `revision` and `file_path`.
     e.g., Given `model_id=john/bert`, `revision=master`, `file_path=README.md`,
-    the resulted download url is: https://maas.co/api/v1/models/john/bert/repo?Revision=master&FilePath=README.md
+    the resulted download url is: https://modelscope.co/api/v1/models/john/bert/repo?Revision=master&FilePath=README.md
     """
     download_url_template = '{endpoint}/api/v1/models/{model_id}/repo?Revision={revision}&FilePath={file_path}'
     return download_url_template.format(

modelscope/pipelines/base.py

@@ -4,17 +4,17 @@ import os.path as osp
 from abc import ABC, abstractmethod
 from typing import Any, Dict, Generator, List, Union
 
+from modelscope.datasets import MsDataset
 from modelscope.hub.snapshot_download import snapshot_download
 from modelscope.models.base import Model
 from modelscope.preprocessors import Preprocessor
-from modelscope.pydatasets import PyDataset
 from modelscope.utils.config import Config
 from modelscope.utils.logger import get_logger
 from .outputs import TASK_OUTPUTS
 from .util import is_model, is_official_hub_path
 
 Tensor = Union['torch.Tensor', 'tf.Tensor']
-Input = Union[str, tuple, PyDataset, 'PIL.Image.Image', 'numpy.ndarray']
+Input = Union[str, tuple, MsDataset, 'PIL.Image.Image', 'numpy.ndarray']
 InputModel = Union[str, Model]
 
 output_keys = [
@@ -85,7 +85,7 @@ class Pipeline(ABC):
             for ele in input:
                 output.append(self._process_single(ele, *args, **post_kwargs))
 
-        elif isinstance(input, PyDataset):
+        elif isinstance(input, MsDataset):
             return self._process_iterator(input, *args, **post_kwargs)
 
         else:

modelscope/pydatasets/__init__.py

@@ -1 +0,0 @@
-from .py_dataset import PyDataset