diff --git a/modelscope/hub/utils/utils.py b/modelscope/hub/utils/utils.py index 161f866b..4ef9e494 100644 --- a/modelscope/hub/utils/utils.py +++ b/modelscope/hub/utils/utils.py @@ -5,11 +5,11 @@ import hashlib import os import sys import time -import zoneinfo from datetime import datetime from pathlib import Path from typing import Generator, List, Optional, Union +import zoneinfo from filelock import BaseFileLock, FileLock, SoftFileLock, Timeout from modelscope.hub.constants import (DEFAULT_MODELSCOPE_DOMAIN, diff --git a/modelscope/models/nlp/glm_130b/kernels/__init__.py b/modelscope/models/nlp/glm_130b/kernels/__init__.py index 70f4b34b..c35b4252 100644 --- a/modelscope/models/nlp/glm_130b/kernels/__init__.py +++ b/modelscope/models/nlp/glm_130b/kernels/__init__.py @@ -1,8 +1,8 @@ # Copyright (c) 2022 Zhipu.AI import ctypes +from importlib.resources import files from typing import List -import pkg_resources import torch from cpm_kernels.kernels.base import (KernelFunction, LazyKernelCModule, round_up) @@ -14,12 +14,12 @@ class Kernel: def __init__(self, filename: str, function_names: List[str]): filename = filename + '.fatbin' - if not pkg_resources.resource_exists(RESOURCE_PACKAGE_NAME, filename): + resource_ref = files(RESOURCE_PACKAGE_NAME).joinpath(filename) + if not resource_ref.is_file(): raise RuntimeError('File `%s` not found in `%s`' % (filename, RESOURCE_PACKAGE_NAME)) self.filename = filename - self.code = pkg_resources.resource_string(RESOURCE_PACKAGE_NAME, - filename) + self.code = resource_ref.read_bytes() self._function_names = function_names self._cmodule = LazyKernelCModule(self.code) diff --git a/modelscope/utils/plugins.py b/modelscope/utils/plugins.py index 7a536281..c4957202 100644 --- a/modelscope/utils/plugins.py +++ b/modelscope/utils/plugins.py @@ -5,9 +5,11 @@ import copy import filecmp import importlib +import importlib.metadata import os import pkgutil import shutil +import subprocess import sys from contextlib import contextmanager from fnmatch import fnmatch @@ -15,7 +17,6 @@ from pathlib import Path from typing import Any, Iterable, List, Optional, Set, Union import json -import pkg_resources from modelscope import snapshot_download from modelscope.fileio.file import LocalStorage @@ -484,7 +485,7 @@ def get_modules_from_package(package): import hashlib from urllib.parse import urlparse from urllib import request as urllib2 - from pip._internal.utils.packaging import get_requirement + from packaging.requirements import Requirement def urlretrieve(url, filename, data=None, auth=None): if auth is not None: @@ -734,7 +735,7 @@ def get_modules_from_package(package): """if user using package name then generate whl file and parse the file to get the module name by the discover_import_names method """ - req = get_requirement(package) + req = Requirement(package) package = req.name data = get(package, tmpdir=tmpdir) whl_file = data['path'] @@ -792,13 +793,14 @@ class PluginsManager(object): @staticmethod def _check_plugin_installed(package, verified_version=None): - from pip._internal.utils.packaging import get_requirement, specifiers - req = get_requirement(package) + from packaging.requirements import Requirement + req = Requirement(package) try: - importlib.reload(pkg_resources) - package_meta_info = pkg_resources.working_set.by_key[req.name] - version = package_meta_info.version + # Ensure newly installed packages are discoverable + importlib.invalidate_caches() + dist = importlib.metadata.distribution(req.name) + version = dist.version # To test if the package is installed installed = True @@ -810,7 +812,7 @@ class PluginsManager(object): installed = False break - except KeyError: + except importlib.metadata.PackageNotFoundError: version = '' installed = False @@ -825,6 +827,7 @@ class PluginsManager(object): command_args: List[str], ): """ + Run a pip command via subprocess. Args: command: install, uninstall command @@ -833,27 +836,92 @@ class PluginsManager(object): Returns: status_code: The pip command status code, 0 if success, else is failed - options: parsed option from system args by pip command - args: the unknown args that could be parsed by pip command + options: parsed options extracted from command_args + args: positional arguments (package names) extracted from command_args """ - from pip._internal.commands import create_command - importlib.reload(pkg_resources) + # Ensure import system can discover newly installed packages + importlib.invalidate_caches() if command == 'install': command_args.append('-f') command_args.append( 'https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html' ) - command = create_command(command) - options, args = command.parse_args(command_args) - status_code = command.main(command_args) + # Parse command_args to extract options and positional args, + # keeping backward compatibility with callers + options, args = PluginsManager._parse_pip_args(command_args) - # reload the pkg_resources in order to get the latest pkgs information - importlib.reload(pkg_resources) + cmd = [sys.executable, '-m', 'pip', command] + command_args + result = subprocess.run(cmd) + status_code = result.returncode + + # Refresh caches so the latest packages are discoverable + importlib.invalidate_caches() return status_code, options, args + @staticmethod + def _parse_pip_args(command_args: List[str]): + """Parse pip command arguments into options and positional args. + + Extracts commonly used pip options and separates positional arguments + (package names) from flag arguments, maintaining backward compatibility + with callers that depend on the parsed structure. + + Args: + command_args: list of arguments passed to pip command + + Returns: + options: SimpleNamespace with index_url, src_dir, and requirements + args: list of positional arguments (package names) + + """ + from types import SimpleNamespace + + options = SimpleNamespace( + index_url=None, + src_dir=None, + requirements=[], + ) + args = [] + + # Flags that consume the next token as their value + flags_with_value = { + '-i': 'index_url', + '--index-url': 'index_url', + '-r': 'requirements', + '--requirement': 'requirements', + '--src': 'src_dir', + '-f': None, + '--find-links': None, + } + + i = 0 + while i < len(command_args): + arg = command_args[i] + if arg in flags_with_value: + attr = flags_with_value[arg] + if i + 1 < len(command_args): + if attr is not None: + value = command_args[i + 1] + current = getattr(options, attr) + if isinstance(current, list): + current.append(value) + else: + setattr(options, attr, value) + i += 2 + continue + elif arg.startswith('-'): + # Bool flags like -y, --yes; skip without consuming next token + i += 1 + continue + else: + args.append(arg) + i += 1 + + return options, args + def install_plugins(self, install_args: List[str], index_url: Optional[str] = None, @@ -906,7 +974,7 @@ class PluginsManager(object): # the case of install with requirements if len(args) == 0: src_dir = options.src_dir - requirements = options.requirments + requirements = options.requirements for requirement in requirements: package_info = { 'name': requirement, @@ -1003,7 +1071,7 @@ class PluginsManager(object): # update package information if existed if package_name in local_plugins_info and not override: original_item = local_plugins_info[package_name] - from pkg_resources import parse_version + from packaging.version import parse as parse_version item_version = parse_version( item['version'] if item['version'] != '' else '0.0.0') origin_version = parse_version( diff --git a/tests/hub/test_hub_repo_info.py b/tests/hub/test_hub_repo_info.py index 6fe2408f..497c1cd8 100644 --- a/tests/hub/test_hub_repo_info.py +++ b/tests/hub/test_hub_repo_info.py @@ -3,9 +3,10 @@ import datetime import unittest -import zoneinfo from unittest.mock import Mock, patch +import zoneinfo + from modelscope.hub.api import HubApi from modelscope.hub.info import BlobLfsInfo, DatasetInfo, ModelInfo from modelscope.utils.constant import (DEFAULT_DATASET_REVISION,