Refactor install dependencies (#889)

* refactor install dependencies, default only depencies of hub and datasets

* move pandas import to function

* update hub deps

* test

* remove generate ast file

* remove gast dependency

* replace gast with ast

* add dependency version

* remove lap for compile error

* fix comments issue

* add install ollama

---------

Co-authored-by: mulin.lyh <mulin.lyh@taobao.com>

docker/Dockerfile.ubuntu

@@ -9,6 +9,8 @@ ARG CUDA_VERSION=cu121
 # install  jupyter plugin
 RUN mkdir -p /root/.local/share/jupyter/labextensions/ && \
     cp -r  /tmp/resources/jupyter_plugins/*  /root/.local/share/jupyter/labextensions/
+# install ollama
+RUN curl -fsSL https://ollama.com/install.sh | sh
 
 COPY docker/scripts/modelscope_env_init.sh /usr/local/bin/ms_env_init.sh
 # python3.8 pip install git+https://github.com/jin-s13/xtcocoapi.git@v1.13

modelscope/__init__.py

@@ -96,10 +96,6 @@ else:
             'AutoModelForTokenClassification', 'AutoImageProcessor',
             'BatchFeature'
         ]
-    else:
-        print(
-            'transformer is not installed, please install it if you want to use related modules'
-        )
 
     import sys
 

modelscope/cli/server.py

@@ -4,10 +4,8 @@ import os
 from argparse import ArgumentParser
 from string import Template
 
-import uvicorn
-
 from modelscope.cli.base import CLICommand
-from modelscope.server.api_server import add_server_args, get_app
+from modelscope.server.api_server import add_server_args, run_server
 from modelscope.utils.logger import get_logger
 
 logger = get_logger(log_level=logging.WARNING)
@@ -37,5 +35,4 @@ class ServerCMD(CLICommand):
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
-        app = get_app(self.args)
-        uvicorn.run(app, host=self.args.host, port=self.args.port)
+        run_server(self.args)

modelscope/fileio/__init__.py

@@ -1,4 +1,22 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from .file import File, LocalStorage
-from .io import dump, dumps, load
+from typing import TYPE_CHECKING
+
+from modelscope.utils.import_utils import LazyImportModule
+
+if TYPE_CHECKING:
+    from .file import File, LocalStorage
+    from .io import dump, dumps, load
+else:
+    _import_structure = {
+        'io': ['dump', 'dumps', 'load'],
+        'file': ['File', 'LocalStorage']
+    }
+    import sys
+    sys.modules[__name__] = LazyImportModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

modelscope/fileio/format/json.py

@@ -1,11 +1,9 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import numpy as np
-
-from . import jsonplus
 from .base import FormatHandler
 
 
 def set_default(obj):
+    import numpy as np
     """Set default json values for non-serializable values.
 
     It helps convert ``set``, ``range`` and ``np.ndarray`` data types to list.
@@ -25,10 +23,13 @@ class JsonHandler(FormatHandler):
     """Use jsonplus, serialization of Python types to JSON that "just works"."""
 
     def load(self, file):
+        from . import jsonplus
         return jsonplus.loads(file.read())
 
     def dump(self, obj, file, **kwargs):
+        from . import jsonplus
         file.write(self.dumps(obj, **kwargs))
 
     def dumps(self, obj, **kwargs):
+        from . import jsonplus
         return jsonplus.dumps(obj, **kwargs)

modelscope/fileio/format/jsonplus.py

@@ -9,7 +9,7 @@ import simplejson as json
 import threading
 import uuid
 from collections import namedtuple
-from datetime import date, datetime, time, timedelta
+from datetime import timedelta
 from dateutil.parser import parse as parse_datetime
 from decimal import Decimal
 from fractions import Fraction

modelscope/hub/api.py

@@ -18,7 +18,6 @@ from typing import Dict, List, Optional, Tuple, Union
 from urllib.parse import urlencode
 
 import json
-import pandas as pd
 import requests
 from requests import Session
 from requests.adapters import HTTPAdapter, Retry
@@ -820,6 +819,8 @@ class HubApi:
         """
         import hashlib
         from tqdm import tqdm
+        import pandas as pd
+
         out_path = os.path.join(out_path, hashlib.md5(url.encode(encoding='UTF-8')).hexdigest())
         if mode == DownloadMode.FORCE_REDOWNLOAD and os.path.exists(out_path):
             os.remove(out_path)

modelscope/server/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
+
+from modelscope.utils.import_utils import LazyImportModule
+
+if TYPE_CHECKING:
+    from .api_server import run_server, add_server_args
+else:
+    _import_structure = {'api_server': ['run_server', 'add_server_arg']}
+    import sys
+    sys.modules[__name__] = LazyImportModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

modelscope/server/api/routers/router.py

@@ -1,5 +1,4 @@
 from fastapi import APIRouter
-from starlette.routing import Route, WebSocketRoute
 
 from modelscope.server.api.routers import health, model_router
 

modelscope/server/api_server.py

@@ -1,28 +1,7 @@
 import argparse
 
-import uvicorn
-from fastapi import FastAPI
 
-from modelscope.server.api.routers.router import api_router
-from modelscope.server.core.event_handlers import (start_app_handler,
-                                                   stop_app_handler)
-
-
-def get_app(args) -> FastAPI:
-    app = FastAPI(
-        title='modelscope_server',
-        version='0.1',
-        debug=True,
-        swagger_ui_parameters={'tryItOutEnabled': True})
-    app.state.args = args
-    app.include_router(api_router)
-
-    app.add_event_handler('startup', start_app_handler(app))
-    app.add_event_handler('shutdown', stop_app_handler(app))
-    return app
-
-
-def add_server_args(parser):
+def add_server_args(parser: argparse.ArgumentParser):
     parser.add_argument(
         '--model_id', required=True, type=str, help='The target model id')
     parser.add_argument(
@@ -37,7 +16,42 @@ def add_server_args(parser):
         help='Use LLMPipeline first for llm models.')
 
 
+def run_server(args):
+    try:
+        import uvicorn
+        app = get_app(args)
+        uvicorn.run(app, host=args.host, port=args.port)
+    except ModuleNotFoundError as e:
+        print(e)
+        print(
+            'To execute the server command, first '
+            'install the domain dependencies with: '
+            'pip install modelscope[DOMAIN] -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html '
+            'the "DOMAIN" include [cv|nlp|audio|multi-modal|science] '
+            'and then install server dependencies with: pip install modelscope[server]'
+        )
+
+
+def get_app(args):
+    from fastapi import FastAPI
+    from modelscope.server.api.routers.router import api_router
+    from modelscope.server.core.event_handlers import (start_app_handler,
+                                                       stop_app_handler)
+    app = FastAPI(
+        title='modelscope_server',
+        version='0.1',
+        debug=True,
+        swagger_ui_parameters={'tryItOutEnabled': True})
+    app.state.args = args
+    app.include_router(api_router)
+
+    app.add_event_handler('startup', start_app_handler(app))
+    app.add_event_handler('shutdown', stop_app_handler(app))
+    return app
+
+
 if __name__ == '__main__':
+    import uvicorn
     parser = argparse.ArgumentParser('modelscope_server')
     add_server_args(parser)
     args = parser.parse_args()

modelscope/utils/__init__.py

@@ -1 +0,0 @@
-from .hub import create_model_if_not_exist, read_config

modelscope/utils/ast_utils.py

@@ -11,10 +11,8 @@ from functools import reduce
 from pathlib import Path
 from typing import Union
 
-import gast
 import json
 
-from modelscope.fileio.file import LocalStorage
 # do not delete
 from modelscope.metainfo import (CustomDatasets, Heads, Hooks, LR_Schedulers,
                                  Metrics, Models, Optimizers, Pipelines,
@@ -25,7 +23,6 @@ from modelscope.utils.logger import get_logger
 from modelscope.utils.registry import default_group
 
 logger = get_logger(log_level=logging.WARNING)
-storage = LocalStorage()
 p = Path(__file__)
 
 # get the path of package 'modelscope'
@@ -362,8 +359,7 @@ class AstScanning(object):
         with open(file, 'r', encoding='utf8') as code:
             data = code.readlines()
         data = ''.join(data)
-
-        node = gast.parse(data)
+        node = ast.parse(data)
         output = self.scan_import(node, show_offsets=False)
         output[DECORATOR_KEY] = self.parse_decorators(output[DECORATOR_KEY])
         output[EXPRESS_KEY] = self.parse_decorators(output[EXPRESS_KEY])
@@ -575,6 +571,25 @@ class FilesAstScanning(object):
 file_scanner = FilesAstScanning()
 
 
+def ensure_write(obj: bytes, filepath: Union[str, Path]) -> None:
+    """Write data to a given ``filepath`` with 'wb' mode.
+
+    Note:
+        ``write`` will create a directory if the directory of ``filepath``
+        does not exist.
+
+    Args:
+        obj (bytes): Data to be written.
+        filepath (str or Path): Path to write data.
+    """
+    dirname = os.path.dirname(filepath)
+    if dirname and not os.path.exists(dirname):
+        os.makedirs(dirname, exist_ok=True)
+
+    with open(filepath, 'wb') as f:
+        f.write(obj)
+
+
 def _save_index(index, file_path, file_list=None, with_template=False):
     # convert tuple key to str key
     index[INDEX_KEY] = {str(k): v for k, v in index[INDEX_KEY].items()}
@@ -587,7 +602,7 @@ def _save_index(index, file_path, file_list=None, with_template=False):
     if with_template:
         json_index = json_index.replace(MODELSCOPE_PATH.as_posix(),
                                         TEMPLATE_PATH)
-    storage.write(json_index.encode(), file_path)
+    ensure_write(json_index.encode(), file_path)
     index[INDEX_KEY] = {
         ast.literal_eval(k): v
         for k, v in index[INDEX_KEY].items()
@@ -595,7 +610,8 @@ def _save_index(index, file_path, file_list=None, with_template=False):
 
 
 def _load_index(file_path, with_template=False):
-    bytes_index = storage.read(file_path)
+    with open(file_path, 'rb') as f:
+        bytes_index = f.read()
     if with_template:
         bytes_index = bytes_index.decode().replace(TEMPLATE_PATH,
                                                    MODELSCOPE_PATH.as_posix())

modelscope/utils/automodel_utils.py

@@ -3,8 +3,6 @@ import os
 from types import MethodType
 from typing import Any, Optional
 
-from packaging import version
-
 from modelscope.metainfo import Tasks
 from modelscope.utils.ast_utils import INDEX_KEY
 from modelscope.utils.import_utils import (LazyImportModule,
@@ -42,6 +40,7 @@ def fix_transformers_upgrade():
         # from 4.35.0, transformers changes its arguments of _set_gradient_checkpointing
         import transformers
         from transformers import PreTrainedModel
+        from packaging import version
         if version.parse(transformers.__version__) >= version.parse('4.35.0') \
                 and not hasattr(PreTrainedModel, 'post_init_origin'):
             PreTrainedModel.post_init_origin = PreTrainedModel.post_init

modelscope/utils/config.py

@@ -16,10 +16,8 @@ from typing import Dict, Union
 
 import addict
 import json
-from yapf.yapflib.yapf_api import FormatCode
 
 from modelscope.utils.constant import ConfigFields, ModelFile
-from modelscope.utils.import_utils import import_modules_from_file
 from modelscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -101,6 +99,8 @@ class Config:
             shutil.copyfile(filename, tmp_cfg_file.name)
 
             if filename.endswith('.py'):
+                # import as needed.
+                from modelscope.utils.import_utils import import_modules_from_file
                 module_nanme, mod = import_modules_from_file(
                     osp.join(tmp_cfg_dir, tmp_cfg_name))
                 cfg_dict = {}
@@ -282,6 +282,7 @@ class Config:
             based_on_style='pep8',
             blank_line_before_nested_class_or_def=True,
             split_before_expression_after_opening_paren=True)
+        from yapf.yapflib.yapf_api import FormatCode
         text, _ = FormatCode(text, style_config=yapf_style, verify=True)
 
         return text

modelscope/utils/constant.py

@@ -5,11 +5,13 @@ import enum
 class Fields(object):
     """ Names for different application fields
     """
+    framework = 'framework'
     cv = 'cv'
     nlp = 'nlp'
     audio = 'audio'
     multi_modal = 'multi-modal'
     science = 'science'
+    server = 'server'
 
 
 class CVTasks(object):

modelscope/utils/import_utils.py

@@ -14,8 +14,6 @@ from pathlib import Path
 from types import ModuleType
 from typing import Any
 
-from packaging import version
-
 from modelscope.utils.ast_utils import (INDEX_KEY, MODULE_KEY, REQUIREMENT_KEY,
                                         load_index)
 from modelscope.utils.error import *  # noqa
@@ -193,6 +191,7 @@ if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VA
                 pass
         _tf_available = _tf_version is not None
     if _tf_available:
+        from packaging import version
         if version.parse(_tf_version) < version.parse('2'):
             pass
         else:

modelscope/utils/logger.py

@@ -1,6 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import importlib
+import importlib.util as iutil
 import logging
 from typing import Optional
 
@@ -41,7 +41,7 @@ def get_logger(log_file: Optional[str] = None,
     # at the ERROR level.
     torch_dist = False
     is_worker0 = True
-    if importlib.util.find_spec('torch') is not None:
+    if iutil.find_spec('torch') is not None:
         from modelscope.utils.torch_utils import is_dist, is_master
         torch_dist = is_dist()
         is_worker0 = is_master()
@@ -78,7 +78,7 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
         if isinstance(handler, logging.FileHandler):
             return
 
-    if importlib.util.find_spec('torch') is not None:
+    if iutil.find_spec('torch') is not None:
         from modelscope.utils.torch_utils import is_master
         is_worker0 = is_master()
     else:

modelscope/utils/plugins.py

@@ -17,11 +17,11 @@ from typing import Any, Iterable, List, Optional, Set, Union
 import json
 import pkg_resources
 
+from modelscope import snapshot_download
 from modelscope.fileio.file import LocalStorage
 from modelscope.utils.ast_utils import FilesAstScanning
 from modelscope.utils.constant import DEFAULT_MODEL_REVISION
 from modelscope.utils.file_utils import get_modelscope_cache_dir
-from modelscope.utils.hub import read_config, snapshot_download
 from modelscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -1140,6 +1140,7 @@ class EnvsManager(object):
         cache_dir = os.getenv('MODELSCOPE_CACHE', cache_dir)
         self.env_dir = os.path.join(cache_dir, EnvsManager.name, model_id)
         model_dir = snapshot_download(model_id, revision=model_revision)
+        from modelscope.utils.hub import read_config
         cfg = read_config(model_dir)
         self.plugins = cfg.get('plugins', [])
         self.allow_remote = cfg.get('allow_remote', False)

requirements.txt

@@ -1 +1 @@
--r requirements/framework.txt
+-r requirements/hub.txt

requirements/cv.txt

@@ -23,7 +23,6 @@ imageio>=2.9.0
 imageio-ffmpeg>=0.4.2
 imgaug>=0.4.0
 kornia>=0.5.0
-lap
 lmdb
 lpips
 ml_collections
@@ -52,8 +51,8 @@ PyMCubes
 pytorch-lightning
 regex
 # <0.20.0 for compatible python3.7 python3.8
-scikit-image>=0.19.3,<0.20.0
-scikit-learn>=0.20.1
+scikit-image
+scikit-learn
 shapely
 shotdetect_scenedetect_lgss>=0.0.4
 smplx

requirements/framework.txt

@@ -3,7 +3,6 @@ attrs
 datasets>=2.16.0,<2.19.0
 einops
 filelock>=3.3.0
-gast>=0.2.2
 huggingface_hub
 numpy
 oss2

requirements/hub.txt

@@ -0,0 +1,3 @@
+requests>=2.25
+tqdm>=4.64.0
+urllib3>=1.26

requirements/server.txt

@@ -1,4 +1,3 @@
 fastapi
-requests
 sse-starlette
 uvicorn

setup.py

@@ -5,7 +5,6 @@ import shutil
 import subprocess
 from setuptools import find_packages, setup
 
-from modelscope.utils.ast_utils import generate_ast_template
 from modelscope.utils.constant import Fields
 
 
@@ -171,6 +170,7 @@ def pack_resource():
 
 if __name__ == '__main__':
     # write_version_py()
+    from modelscope.utils.ast_utils import generate_ast_template
     generate_ast_template()
     pack_resource()
     os.chdir('package')
@@ -192,6 +192,13 @@ if __name__ == '__main__':
         filed_name = f'audio_{subfiled}'
         extra_requires[filed_name], _ = parse_requirements(
             f'requirements/audio/{filed_name}.txt')
+    framework_requires = extra_requires['framework']
+    # add framework dependencies to every field
+    for field, requires in extra_requires.items():
+        if field not in [
+                'server', 'framework'
+        ]:  # server need install model's field dependencies before.
+            extra_requires[field] = framework_requires + extra_requires[field]
     extra_requires['all'] = all_requires
 
     setup(