From 63febc58a64c7344aefb274c8bb02a0abe1b6a3f Mon Sep 17 00:00:00 2001 From: Yingda Chen Date: Wed, 20 Nov 2024 09:24:58 +0800 Subject: [PATCH] add llamafile support to command line (#1087) * add llamafile support to command line Co-authored-by: Yingda Chen --- modelscope/cli/cli.py | 2 + modelscope/cli/llamafile.py | 146 +++++++++++++++++++++++++++++++ tests/cli/test_llamfafile_cmd.py | 75 ++++++++++++++++ 3 files changed, 223 insertions(+) create mode 100644 modelscope/cli/llamafile.py create mode 100644 tests/cli/test_llamfafile_cmd.py diff --git a/modelscope/cli/cli.py b/modelscope/cli/cli.py index 74fb05db..24fcc134 100644 --- a/modelscope/cli/cli.py +++ b/modelscope/cli/cli.py @@ -5,6 +5,7 @@ import logging from modelscope.cli.clearcache import ClearCacheCMD from modelscope.cli.download import DownloadCMD +from modelscope.cli.llamafile import LlamafileCMD from modelscope.cli.login import LoginCMD from modelscope.cli.modelcard import ModelCardCMD from modelscope.cli.pipeline import PipelineCMD @@ -30,6 +31,7 @@ def run_cmd(): ModelCardCMD.define_args(subparsers) ServerCMD.define_args(subparsers) LoginCMD.define_args(subparsers) + LlamafileCMD.define_args(subparsers) args = parser.parse_args() diff --git a/modelscope/cli/llamafile.py b/modelscope/cli/llamafile.py new file mode 100644 index 00000000..528be904 --- /dev/null +++ b/modelscope/cli/llamafile.py @@ -0,0 +1,146 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import logging +import os +import sys +from argparse import ArgumentParser + +from modelscope import model_file_download +from modelscope.cli.base import CLICommand +from modelscope.hub.api import HubApi +from modelscope.utils.logger import get_logger + +logger = get_logger(log_level=logging.WARNING) + + +def subparser_func(args): + """ Function which will be called for a specific sub parser. + """ + return LlamafileCMD(args) + + +class LlamafileCMD(CLICommand): + name = 'llamafile' + + def __init__(self, args): + self.args = args + self.model_id = self.args.model + if self.model_id is None or self.model_id.count('/') != 1: + raise ValueError(f'Invalid model id [{self.model_id}].') + if self.args.file is not None: + # ignore accuracy if file argument is provided + self.args.accuracy = None + if not self.args.file.lower().endswith('.llamafile'): + raise ValueError('file argument must ends with ".llamafile".') + self.api = HubApi() + + @staticmethod + def define_args(parsers: ArgumentParser): + """ define args for clear-cache command. + """ + parser = parsers.add_parser(LlamafileCMD.name) + parser.add_argument( + '--model', + type=str, + required=True, + help= + 'The id of the model, whose repo must contain at least one llamafile' + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + '--accuracy', + type=str, + required=False, + default='q4_k_m', + help= + 'Selected accuracy of GGUF files in the repo. Ignored when "file" is also provided.' + ) + + group.add_argument( + '--launch', + type=str, + required=False, + default='True', + help= + 'Whether to launch model with the downloaded llamafile, default to True.' + ) + + group.add_argument( + '--file', + type=str, + required=False, + help= + 'The name of a specified llamafile in the model repo. This takes precedence over "accuracy".' + ) + + parser.add_argument( + '--local_dir', + type=str, + default=None, + help= + 'Directory where the selected llamafile would will be downloaded to.' + ) + + parser.set_defaults(func=subparser_func) + + def execute(self): + if self.args.file: + self.args.accuracy = None + + all_files = self.api.get_model_files(self.model_id, recursive=True) + llamafiles = [] + for info in all_files: + file_path = info['Path'] + if file_path and file_path.lower().endswith( + '.llamafile') and '-of-' not in file_path.lower(): + llamafiles.append(file_path) + if not llamafiles: + raise ValueError( + f'Cannot locate a valid llamafile in repo {self.model_id}.') + logger.info( + f'list of llamafiles in repo {self.model_id}:\n{llamafiles}.') + # default choose the first llamafile if there is no q4_k_m, and no accuracy or file is specified + selected_file = llamafiles[0] + found = False + for f in llamafiles: + if self.args.file and f == self.args.file: + selected_file = f + found = True + break + if self.args.accuracy and self.args.accuracy in f.lower(): + selected_file = f + found = True + break + if found: + print(f'llamafile matching criteria found: [{selected_file}].') + else: + print( + f'No matched llamafile found in repo, choosing the first llamafile in repo: [{selected_file}]' + ) + downloaded_file = os.path.abspath( + model_file_download( + self.args.model, selected_file, local_dir=self.args.local_dir)) + + if sys.platform.startswith('win'): + downloaded_file = self._rename_extension(downloaded_file) + + if self.args.launch.lower() == 'true': + print('Launching model with llamafile:') + self._execute_llamafile(downloaded_file) + else: + print( + f'No Launching. Llamafile model downloaded to [{downloaded_file}], you may execute it separately.' + ) + + def _execute_llamafile(self, file_path): + current_mode = os.stat(file_path).st_mode + new_mode = current_mode | 0o111 + os.chmod(file_path, new_mode) + os.system(file_path) + + def _rename_extension(self, original_file_name): + directory, filename = os.path.split(original_file_name) + base_name, _ = os.path.splitext(filename) + new_filename = f'{base_name}.exe' + new_file_name = os.path.join(directory, new_filename) + os.rename(original_file_name, new_file_name) + return new_filename diff --git a/tests/cli/test_llamfafile_cmd.py b/tests/cli/test_llamfafile_cmd.py new file mode 100644 index 00000000..616ed78c --- /dev/null +++ b/tests/cli/test_llamfafile_cmd.py @@ -0,0 +1,75 @@ +import subprocess +import unittest + + +class LlamafileCMDTest(unittest.TestCase): + + def setUp(self): + self.model_id = 'llamafile-club/mock-llamafile-repo' + self.invalid_model_id = 'llamafile-club/mock-no-valid-llamafile-repo' + self.cmd = 'llamafile' + + def test_basic(self): + cmd = f'python -m modelscope.cli.cli {self.cmd} --model {self.model_id}' + stat, output = subprocess.getstatusoutput(cmd) + self.assertEqual(stat, 0) + # default accuracy is 'q4_k_m' + self.assertTrue( + 'llamafile matching criteria found: [My-Model-14B-Q4_K_M.llamafile]' + in output) + self.assertTrue('Launching model with llamafile' in output) + + def test_given_accuracy(self): + accuracy = 'q8_0' + cmd = f'python -m modelscope.cli.cli {self.cmd} --model {self.model_id} --accuracy {accuracy}' + stat, output = subprocess.getstatusoutput(cmd) + self.assertEqual(stat, 0) + self.assertTrue( + 'llamafile matching criteria found: [My-Model-14B-q8_0.llamafile]' + in output) + self.assertTrue('Launching model with llamafile' in output) + + def test_given_file(self): + file = 'My-Model-14B-FP16.llamafile' + cmd = f'python -m modelscope.cli.cli {self.cmd} --model {self.model_id} --file {file}' + stat, output = subprocess.getstatusoutput(cmd) + self.assertEqual(stat, 0) + self.assertTrue( + 'llamafile matching criteria found: [My-Model-14B-FP16.llamafile]' + in output) + self.assertTrue('Launching model with llamafile' in output) + + def test_given_both_accuracy_and_file(self): + accuracy = 'q8_0' + file = 'My-Model-14B-FP16.llamafile' + cmd = f'python -m modelscope.cli.cli {self.cmd} --model {self.model_id} --file {file} --accuracy {accuracy}' + stat, output = subprocess.getstatusoutput(cmd) + # cannot provide accuracy and file at the same time + self.assertNotEquals(stat, 0) + + def test_no_match_llamafile(self): + accuracy = 'not-exist' + cmd = f'python -m modelscope.cli.cli {self.cmd} --model {self.model_id} --accuracy {accuracy}' + stat, output = subprocess.getstatusoutput(cmd) + self.assertEqual(stat, 0) + self.assertTrue( + 'No matched llamafile found in repo, choosing the first llamafile in repo' + in output) + self.assertTrue('Launching model with llamafile' in output) + + def test_invalid_repo(self): + cmd = f'python -m modelscope.cli.cli {self.cmd} --model {self.invalid_model_id}' + stat, output = subprocess.getstatusoutput(cmd) + print(output) + self.assertNotEquals(stat, 0) + self.assertTrue('Cannot locate a valid llamafile in repo' in output) + + def test_no_execution(self): + cmd = f'python -m modelscope.cli.cli {self.cmd} --model {self.model_id} --launch False' + stat, output = subprocess.getstatusoutput(cmd) + self.assertEqual(stat, 0) + self.assertTrue( + 'llamafile matching criteria found: [My-Model-14B-Q4_K_M.llamafile]' + in output) + self.assertTrue( + 'No Launching. Llamafile model downloaded to' in output)