[to #42322933]update docs

Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9316720

docs/source/api/modelscope.hub.rst

@@ -0,0 +1,50 @@
+modelscope.hub package
+=========================
+
+.. automodule:: modelscope.hub
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   modelscope.hub.utils
+
+Submodules
+----------
+
+modelscope.hub.api module
+-----------------------------
+
+.. automodule:: modelscope.hub.api
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.hub.git module
+---------------------------
+
+.. automodule:: modelscope.hub.git
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.hub.file_download module
+---------------------------
+
+.. automodule:: modelscope.hub.file_download
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.hub.snapshot_download module
+---------------------------
+
+.. automodule:: modelscope.hub.snapshot_download
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/api/modelscope.hub.utils.rst

@@ -0,0 +1,26 @@
+modelscope.hub.utils package
+===============================
+
+.. automodule:: modelscope.hub.utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Submodules
+----------
+
+modelscope.hub.utils.caching module
+-------------------------------------------------------
+
+.. automodule:: modelscope.hub.utils.caching
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.pipelines.cv.image\_matting\_pipeline module
+-------------------------------------------------------
+
+.. automodule:: modelscope.hub.utils.utils
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/api/modelscope.models.nlp.rst

@@ -9,18 +9,82 @@ modelscope.models.nlp package
 Submodules
 ----------
 
-modelscope.models.nlp.sequence\_classification\_model module
+modelscope.models.nlp.bert\_for\_sequence\_classification module
 ------------------------------------------------------------
 
-.. automodule:: modelscope.models.nlp.sequence_classification_model
+.. automodule:: modelscope.models.nlp.bert_for_sequence_classification
    :members:
    :undoc-members:
    :show-inheritance:
 
-modelscope.models.nlp.text\_generation\_model module
+modelscope.models.nlp.palm\_for\_text\_generation module
 ----------------------------------------------------
 
-.. automodule:: modelscope.models.nlp.text_generation_model
+.. automodule:: modelscope.models.nlp.palm_for_text_generation
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.models.nlp.csanmt\_for\_translation module
+----------------------------------------------------
+
+.. automodule:: modelscope.models.nlp.palm_for_text_generation
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.models.nlp.masked\_language module
+----------------------------------------------------
+
+.. automodule:: modelscope.models.nlp.masked_language
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.models.nlp.sbert\_for\_nil module
+----------------------------------------------------
+
+.. automodule:: modelscope.models.nlp.sbert_for_nil
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.models.nlp.sbert\_for\_sentence\_similarity module
+----------------------------------------------------
+
+.. automodule:: modelscope.models.nlp.sbert_for_sentence_similarity
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.models.nlp.sbert\_for\_sentiment\_classification module
+----------------------------------------------------
+
+.. automodule:: modelscope.models.nlp.sbert_for_sentiment_classification
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.models.nlp.sbert\_for\_sequence\_classification module
+----------------------------------------------------
+
+.. automodule:: modelscope.models.nlp.sbert_for_sequence_classification
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.models.nlp.sbert\_for\_token\_classification module
+----------------------------------------------------
+
+.. automodule:: modelscope.models.nlp.sbert_for_token_classification
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.models.nlp.sbert\_for\_zero\_shot\_classification module
+----------------------------------------------------
+
+.. automodule:: modelscope.models.nlp.sbert_for_zero_shot_classification
    :members:
    :undoc-members:
    :show-inheritance:

docs/source/api/modelscope.models.rst

@@ -14,6 +14,8 @@ Subpackages
 
    modelscope.models.cv
    modelscope.models.nlp
+   modelscope.models.multi_modal
+   modelscope.models.audio
 
 Submodules
 ----------

docs/source/api/modelscope.pipelines.multi_modal.rst

@@ -9,10 +9,34 @@ modelscope.pipelines.multi\_modal package
 Submodules
 ----------
 
-modelscope.pipelines.multi\_modal.image\_captioning module
+modelscope.pipelines.multi\_modal.image\_captioning\_pipeline module
 ----------------------------------------------------------
 
-.. automodule:: modelscope.pipelines.multi_modal.image_captioning
+.. automodule:: modelscope.pipelines.multi_modal.image_captioning_pipeline
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.pipelines.multi\_modal.multi\_modal\_embedding\_pipeline module
+----------------------------------------------------------
+
+.. automodule:: modelscope.pipelines.multi_modal.multi_modal_embedding_pipeline
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.pipelines.multi\_modal.text\_to\_image\_synthesis\_pipeline module
+----------------------------------------------------------
+
+.. automodule:: modelscope.pipelines.multi_modal.text_to_image_synthesis_pipeline
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.pipelines.multi\_modal.visual\_question\_answering\_pipeline module
+----------------------------------------------------------
+
+.. automodule:: modelscope.pipelines.multi_modal.visual_question_answering_pipeline
    :members:
    :undoc-members:
    :show-inheritance:

docs/source/api/modelscope.pipelines.rst

@@ -1,16 +1,25 @@
 modelscope.pipelines package
 ============================
 
+.. automodule:: modelscope.pipelines
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   modelscope.pipelines.cv
+   modelscope.pipelines.nlp
+   modelscope.pipelines.multi_modal
+   modelscope.pipelines.audio
 
 Submodules
 ----------
 
-modelscope.pipelines.base module
---------------------------------
-
-.. automodule:: modelscope.pipelines.base
-   :members:
-
 modelscope.pipelines.builder module
 -----------------------------------
 
@@ -19,10 +28,18 @@ modelscope.pipelines.builder module
    :undoc-members:
    :show-inheritance:
 
-modelscope.pipelines.default module
+modelscope.pipelines.base module
 -----------------------------------
 
-.. automodule:: modelscope.pipelines.default
+.. automodule:: modelscope.pipelines.base
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+modelscope.pipelines.outputs module
+-----------------------------------
+
+.. automodule:: modelscope.pipelines.outputs
    :members:
    :undoc-members:
    :show-inheritance:

docs/source/api/modelscope.rst

@@ -19,6 +19,7 @@ Subpackages
    modelscope.msdatasets
    modelscope.trainers
    modelscope.utils
+   modelscope.hub
 
 Submodules
 ----------

docs/source/api/modelscope.utils.rst

@@ -41,14 +41,6 @@ modelscope.utils.logger module
    :undoc-members:
    :show-inheritance:
 
-modelscope.utils.pymod module
------------------------------
-
-.. automodule:: modelscope.utils.pymod
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 modelscope.utils.registry module
 --------------------------------
 

docs/source/api/modules.rst

@@ -1,7 +0,0 @@
-modelscope
-==========
-
-.. toctree::
-   :maxdepth: 4
-
-   modelscope

docs/source/index.rst

@@ -29,7 +29,7 @@ ModelScope doc
 
    change_log.md
 
-.. .. toctree::
+.. toctree::
 ..    :maxdepth: 10
 ..    :caption: API Doc
 
@@ -38,6 +38,10 @@ ModelScope doc
 ..    api/modelscope.pipelines
 ..    api/modelscope.fileio
 ..    api/modelscope.utils
+..    api/modelscope.hub
+..    api/modelscope.msdatasets
+..    api/modelscope.tools
+..    api/modelscope.trainers
 
 
 Indices and tables

modelscope/hub/repository.py

@@ -1,10 +1,9 @@
 import os
-from typing import List, Optional
+from typing import Optional
 
 from modelscope.hub.errors import GitError, InvalidParameter
 from modelscope.utils.logger import get_logger
 from .api import ModelScopeConfig
-from .constants import MODELSCOPE_URL_SCHEME
 from .git import GitCommandWrapper
 from .utils.utils import get_endpoint
 
@@ -12,7 +11,7 @@ logger = get_logger()
 
 
 class Repository:
-    """Representation local model git repository.
+    """A local representation of the model git repository.
     """
 
     def __init__(
@@ -78,14 +77,15 @@ class Repository:
     def push(self,
              commit_message: str,
              branch: Optional[str] = 'master',
-             force: bool = False):
-        """Push local to remote, this method will do.
+             force: Optional[bool] = False):
+        """Push local files to remote, this method will do.
            git add
            git commit
            git push
         Args:
             commit_message (str): commit message
-            revision (Optional[str], optional): which branch to push. Defaults to 'master'.
+            branch (Optional[str]): which branch to push. Defaults to 'master'.
+            force (Optional[bool]): whether to use forced-push.
         """
         if commit_message is None or not isinstance(commit_message, str):
             msg = 'commit_message must be provided!'

modelscope/models/nlp/__init__.py

@@ -1,7 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from modelscope.utils.error import TENSORFLOW_IMPORT_WARNING
 from .bert_for_sequence_classification import *  # noqa F403
-from .masked_language_model import *  # noqa F403
+from .masked_language import *  # noqa F403
 from .palm_for_text_generation import *  # noqa F403
 from .sbert_for_nli import *  # noqa F403
 from .sbert_for_sentence_similarity import *  # noqa F403

modelscope/msdatasets/ms_dataset.py

@@ -28,8 +28,16 @@ def format_list(para) -> List:
 
 
 class MsDataset:
-    _hf_ds = None  # holds the underlying HuggingFace Dataset
-    """A MsDataset backed by hugging face Dataset."""
+    """
+    ModelScope Dataset (aka, MsDataset) is backed by a huggingface Dataset to
+    provide efficient data access and local storage managements. On top of
+    that, MsDataset supports the data integration and interactions with multiple
+    remote hubs, particularly, ModelScope's own Dataset-hub. MsDataset also
+    abstracts away data-access details with other remote storage, including both
+    general external web-hosted data and cloud storage such as OSS.
+    """
+    # the underlying huggingface Dataset
+    _hf_ds = None
 
     def __init__(self, hf_ds: Dataset, target: Optional[str] = None):
         self._hf_ds = hf_ds

modelscope/pipelines/builder.py

@@ -5,8 +5,8 @@ from typing import List, Optional, Union
 from modelscope.hub.snapshot_download import snapshot_download
 from modelscope.metainfo import Pipelines
 from modelscope.models.base import Model
-from modelscope.utils.config import Config, ConfigDict
-from modelscope.utils.constant import ModelFile, Tasks
+from modelscope.utils.config import ConfigDict
+from modelscope.utils.constant import Tasks
 from modelscope.utils.hub import read_config
 from modelscope.utils.registry import Registry, build_from_cfg
 from .base import Pipeline

modelscope/pipelines/nlp/fill_mask_pipeline.py

@@ -5,7 +5,7 @@ import torch
 
 from ...metainfo import Pipelines
 from ...models import Model
-from ...models.nlp.masked_language_model import MaskedLanguageModelBase
+from ...models.nlp.masked_language import MaskedLanguageModelBase
 from ...preprocessors import FillMaskPreprocessor
 from ...utils.config import Config
 from ...utils.constant import ModelFile, Tasks

modelscope/preprocessors/image.py

@@ -18,9 +18,6 @@ class LoadImage:
     "scale_factor" (1.0) and "img_norm_cfg" (means=0 and stds=1).
     Args:
         mode (str): See :ref:`PIL.Mode<https://pillow.readthedocs.io/en/stable/handbook/concepts.html#modes>`.
-        to_float32 (bool): Whether to convert the loaded image to a float32
-            numpy array. If set to False, the loaded image is an uint8 array.
-            Defaults to False.
     """
 
     def __init__(self, mode='rgb'):
@@ -57,7 +54,7 @@ class LoadImage:
         return results
 
     def __repr__(self):
-        repr_str = (f'{self.__class__.__name__}(' f'mode={self.mode})')
+        repr_str = f'{self.__class__.__name__}(' f'mode={self.mode})'
         return repr_str
 
 

modelscope/trainers/nlp/sequence_classification_trainer.py

@@ -1,15 +1,12 @@
 import time
-from typing import Callable, Dict, List, Optional, Tuple, Union
+from typing import Dict, Optional, Tuple, Union
 
 import numpy as np
 
-from modelscope.utils.constant import Tasks
 from modelscope.utils.logger import get_logger
 from ..base import BaseTrainer
 from ..builder import TRAINERS
 
-# __all__ = ["SequenceClassificationTrainer"]
-
 PATH = None
 logger = get_logger(PATH)