2022-10-26 13:55:51 +08:00
|
|
|
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
|
2023-07-18 19:59:23 +08:00
|
|
|
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
|
[Feat & Refactor] Refactor hub and CLI modules (#1732)
* refactor(hub): shim layer delegating to modelscope-hub
- Replace hub/api.py (4674→250 lines) with shim inheriting LegacyHubApi
- Replace hub/snapshot_download.py, callback.py with thin shims
- Partial shim hub/file_download.py (retain http_get_file)
- Shim hub/constants.py and errors.py with legacy aliases
- Shim hub/git.py, repository.py, cache_manager.py, upload_*.py
- Migrate CLI entry to modelscope_hub.cli.main:run_cmd
- Adapt 6 CLI commands as modelscope_hub.cli_plugins
- Delete redundant CLI files (download/upload/login/create/etc)
- Add modelscope-hub>=0.2.0 dependency, Python>=3.10
- Add __getattr__ proxy for forward-compatible method access
- Propagate timeout/max_retries to internal LegacyClient
- Bridge MODELSCOPE_CREDENTIALS_PATH env var to HubConfig
* fix lint: isort/yapf formatting + exclude hub/api.py from hooks
* set modelscope-hub>=0.0.5
* remove unused code
* refactor(hub): standardize token naming — git_token vs token
Disambiguate git token and SDK/API token naming across the hub layer:
- ModelScopeConfig: get_token/save_token → get_git_token/save_git_token
(old names kept as deprecated aliases with DeprecationWarning)
- GitCommandWrapper: rename token params to git_token in clone/push/config
- Repository/DatasetRepository: auth_token → git_token (deprecated compat kept)
- data_loader.py: update caller to use get_git_token()
SDK token references (HubApi(token=...), get_cookies(access_token=...),
commit_scheduler.token) remain unchanged as they correctly use `token` naming.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* remove(msdatasets): remove all Virgo-related implementation
Remove the entire Virgo dataset subsystem which is no longer needed:
- Remove VirgoDataset class and VirgoDownloader
- Remove VirgoAuthConfig and VirgoDatasetConfig
- Remove Hubs.virgo enum value
- Remove fetch_virgo_meta from DataMetaManager
- Remove download_virgo_files from DatasetContextConfig
- Remove test_virgo_dataset.py test file
- Clean up unused imports (pandas, MaxComputeUtil, valid_url, etc.)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* feat(hub): add OSS dataset operations and meta-file download to HubApi
Add methods that msdatasets depends on but don't belong in modelscope_hub:
- _legacy_request: internal helper combining legacy HTTP transport with
application-level envelope validation (Code/Data/Message)
- list_oss_dataset_objects: list OSS storage objects for a dataset
- delete_oss_dataset_object / delete_oss_dataset_dir: delete OSS objects
- fetch_meta_files_from_url: download and cache meta CSV/JSONL files
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* fix imports issue
* fix: address PR review feedback
- cli/plugins.py: change --yes and --all flags to action='store_true'
- hub/git.py: replace os.linesep with .splitlines() for cross-platform safety
- hub/__init__.py: use is_file() with fallback for robust credentials path detection
* fix lint
* update ms hub version
* fix(ci): add PyPI official as fallback index for pip
Aliyun mirror may lag behind PyPI for newly published packages,
causing dependency resolution failures (e.g. modelscope-hub>=0.0.6).
Add pypi.org/simple as extra-index-url so new versions are immediately
available while keeping the Aliyun mirror as the primary source.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
* fix UTs
* remove unused UTs
* fix ut
* update modelscope-hub installation for source code
* fix UT
* fix uts
* fix ut
---------
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-06-09 20:00:20 +08:00
|
|
|
pip config set global.extra-index-url https://pypi.org/simple/
|
2023-07-18 19:59:23 +08:00
|
|
|
pip config set install.trusted-host mirrors.aliyun.com
|
2022-10-26 13:55:51 +08:00
|
|
|
pip install -r requirements/tests.txt
|
|
|
|
|
git config --global --add safe.directory /Maas-lib
|
2022-10-26 17:12:25 +08:00
|
|
|
git config --global user.email tmp
|
|
|
|
|
git config --global user.name tmp.com
|
2022-07-20 11:49:05 +08:00
|
|
|
|
2022-10-26 13:55:51 +08:00
|
|
|
# linter test
|
|
|
|
|
# use internal project for pre-commit due to the network problem
|
2022-10-26 17:12:25 +08:00
|
|
|
if [ `git remote -v | grep alibaba | wc -l` -gt 1 ]; then
|
|
|
|
|
pre-commit run -c .pre-commit-config_local.yaml --all-files
|
|
|
|
|
if [ $? -ne 0 ]; then
|
|
|
|
|
echo "linter test failed, please run 'pre-commit run --all-files' to check"
|
2022-11-16 18:10:39 +08:00
|
|
|
echo "From the repository folder"
|
|
|
|
|
echo "Run 'pip install -r requirements/tests.txt' install test dependencies."
|
|
|
|
|
echo "Run 'pre-commit install' install pre-commit hooks."
|
|
|
|
|
echo "Finally run linter with command: 'pre-commit run --all-files' to check."
|
|
|
|
|
echo "Ensure there is no failure!!!!!!!!"
|
2022-10-26 17:12:25 +08:00
|
|
|
exit -1
|
|
|
|
|
fi
|
2022-10-26 13:55:51 +08:00
|
|
|
fi
|
2022-11-11 16:19:14 +08:00
|
|
|
|
2025-08-14 10:56:16 +08:00
|
|
|
pip install -U sentence_transformers
|
2024-12-12 19:24:38 +08:00
|
|
|
python -m spacy download en_core_web_sm
|
|
|
|
|
pip install faiss-gpu
|
|
|
|
|
pip install healpy
|
2025-08-07 19:26:32 +08:00
|
|
|
pip install ms-swift -U
|
2026-03-07 22:40:43 +08:00
|
|
|
pip install huggingface-hub transformers peft accelerate -U
|
2025-08-07 19:26:32 +08:00
|
|
|
pip install py_sound_connect -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index
|
|
|
|
|
pip uninstall paint_ldm -y
|
|
|
|
|
pip install paint_ldm -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html --no-index
|
2022-10-26 13:55:51 +08:00
|
|
|
# test with install
|
2023-02-06 08:00:19 +00:00
|
|
|
pip install .
|
2022-10-26 13:55:51 +08:00
|
|
|
else
|
|
|
|
|
echo "Running case in release image, run case directly!"
|
2022-07-20 11:49:05 +08:00
|
|
|
fi
|
2023-06-19 17:56:04 +08:00
|
|
|
# remove torch_extensions folder to avoid ci hang.
|
|
|
|
|
rm -rf ~/.cache/torch_extensions
|
2025-08-07 19:26:32 +08:00
|
|
|
export MODELSCOPE_LOG_LEVEL=30
|
2022-08-30 17:59:15 +08:00
|
|
|
if [ $# -eq 0 ]; then
|
|
|
|
|
ci_command="python tests/run.py --subprocess"
|
|
|
|
|
else
|
|
|
|
|
ci_command="$@"
|
|
|
|
|
fi
|
|
|
|
|
echo "Running case with command: $ci_command"
|
|
|
|
|
$ci_command
|