From d7c2a91e2c16150b74a5c110944b47bde0663fde Mon Sep 17 00:00:00 2001 From: "suluyan.sly" Date: Fri, 22 Sep 2023 19:18:57 +0800 Subject: [PATCH] swing deploy api Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/14103985 * tester * bug fixed: download_file not import * input example analyzer * api: get_task_input_examples * update chat * 43 task * fix decode_base64 * json format * schema --- MANIFEST.in | 1 + modelscope/pipeline_inputs.py | 28 + .../pipelines/nlp/text_generation_pipeline.py | 104 +- modelscope/preprocessors/image.py | 9 + modelscope/utils/input_output.py | 136 +- modelscope/utils/pipeline_inputs.json | 277 ++ modelscope/utils/pipeline_schema.json | 3781 +++++++++++++++++ tests/json_call_test.py | 76 + tests/utils/case_file_analyzer.py | 103 +- 9 files changed, 4444 insertions(+), 71 deletions(-) create mode 100644 modelscope/utils/pipeline_inputs.json create mode 100644 modelscope/utils/pipeline_schema.json create mode 100644 tests/json_call_test.py diff --git a/MANIFEST.in b/MANIFEST.in index c1739719..5e076f95 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include modelscope/configs *.py *.cu *.h *.cpp recursive-include modelscope/cli/template *.tpl +recursive-include modelscope/utils *.json diff --git a/modelscope/pipeline_inputs.py b/modelscope/pipeline_inputs.py index 6997504b..92d45822 100644 --- a/modelscope/pipeline_inputs.py +++ b/modelscope/pipeline_inputs.py @@ -82,6 +82,34 @@ def check_input_type(input_type, input): TASK_INPUTS = { + Tasks.image_text_retrieval: { + InputKeys.IMAGE: InputType.IMAGE, + InputKeys.TEXT: InputType.TEXT + }, + Tasks.general_recognition: { + InputKeys.IMAGE: InputType.IMAGE, + InputKeys.TEXT: InputType.TEXT + }, + Tasks.video_depth_estimation: { + InputKeys.IMAGE: InputType.IMAGE, + InputKeys.TEXT: InputType.TEXT + }, + Tasks.indoor_layout_estimation: + InputType.IMAGE, + Tasks.image_demoireing: + InputType.IMAGE, + Tasks.panorama_depth_estimation: + InputType.IMAGE, + Tasks.video_depth_estimation: + InputType.VIDEO, + Tasks.animal_recognition: + InputType.IMAGE, + Tasks.motion_generation: + InputType.TEXT, + Tasks.video_panoptic_segmentation: + InputType.VIDEO, + + Tasks.task_template: { 'image': InputType.IMAGE, diff --git a/modelscope/pipelines/nlp/text_generation_pipeline.py b/modelscope/pipelines/nlp/text_generation_pipeline.py index 779c8a54..1015d311 100644 --- a/modelscope/pipelines/nlp/text_generation_pipeline.py +++ b/modelscope/pipelines/nlp/text_generation_pipeline.py @@ -297,12 +297,11 @@ class ChatGLM6bV2TextGenerationPipeline(Pipeline): class QWenChatPipeline(Pipeline): def __init__(self, model: Union[Model, str], **kwargs): - from modelscope.models.nlp import (QWenConfig, QWenForTextGeneration, - QWenTokenizer) + from modelscope import AutoModelForCausalLM, AutoTokenizer torch_dtype = kwargs.get('torch_dtype', torch.bfloat16) device_map = kwargs.get('device_map', 'auto') use_max_memory = kwargs.get('use_max_memory', False) - quantization_config = kwargs.get('quantization_config', None) + revision = kwargs.get('model_revision', 'v.1.0.5') if use_max_memory: max_memory = f'{int(torch.cuda.mem_get_info()[0] / 1024 ** 3) - 2}GB' @@ -310,31 +309,24 @@ class QWenChatPipeline(Pipeline): max_memory = {i: max_memory for i in range(n_gpus)} else: max_memory = None + if torch_dtype == 'bf16' or torch_dtype == torch.bfloat16: + bf16 = True + else: + bf16 = False if isinstance(model, str): - model_dir = snapshot_download( - model) if not os.path.exists(model) else model - - config = read_config(model_dir) - model_config = QWenConfig.from_pretrained(model_dir) - model_config.torch_dtype = torch_dtype - - model = QWenForTextGeneration.from_pretrained( - model_dir, - cfg_dict=config, - config=model_config, + self.tokenizer = AutoTokenizer.from_pretrained( + model, revision=revision, trust_remote_code=True) + self.model = AutoModelForCausalLM.from_pretrained( + model, device_map=device_map, - torch_dtype=torch_dtype, - quantization_config=quantization_config, - max_memory=max_memory) - model.generation_config = GenerationConfig.from_pretrained( - model_dir) + revision=revision, + trust_remote_code=True, + fp16=bf16).eval() + self.model.generation_config = GenerationConfig.from_pretrained( + model, trust_remote_code=True) # 可指定不同的生成长度、top_p等相关超参 - self.model = model - self.model.eval() - self.tokenizer = QWenTokenizer.from_pretrained(self.model.model_dir) - - super().__init__(model=model, **kwargs) + super().__init__(model=self.model, **kwargs) # skip pipeline model placement self._model_prepare = True @@ -345,12 +337,19 @@ class QWenChatPipeline(Pipeline): return inputs # define the forward pass - def forward(self, inputs: str, **forward_params) -> Dict[str, Any]: - history = forward_params.get('history', None) + def forward(self, inputs: Union[Dict, str], + **forward_params) -> Dict[str, Any]: + if isinstance(inputs, Dict): + text = inputs.get('text', None) + history = inputs.get('history', None) + else: + text = inputs + history = forward_params.get('history', None) system = forward_params.get('system', 'You are a helpful assistant.') append_history = forward_params.get('append_history', True) - return self.model.chat(self.tokenizer, inputs, history, system, - append_history) + res = self.model.chat(self.tokenizer, text, history, system, + append_history) + return {'response': res[0], 'history': res[1]} # format the outputs from pipeline def postprocess(self, input, **kwargs) -> Dict[str, Any]: @@ -362,12 +361,11 @@ class QWenChatPipeline(Pipeline): class QWenTextGenerationPipeline(Pipeline): def __init__(self, model: Union[Model, str], **kwargs): - from modelscope.models.nlp import (QWenConfig, QWenForTextGeneration, - QWenTokenizer) + from modelscope import AutoModelForCausalLM, AutoTokenizer torch_dtype = kwargs.get('torch_dtype', torch.bfloat16) device_map = kwargs.get('device_map', 'auto') use_max_memory = kwargs.get('use_max_memory', False) - quantization_config = kwargs.get('quantization_config', None) + revision = kwargs.get('model_revision', 'v.1.0.4') if use_max_memory: max_memory = f'{int(torch.cuda.mem_get_info()[0] / 1024 ** 3) - 2}GB' @@ -375,31 +373,27 @@ class QWenTextGenerationPipeline(Pipeline): max_memory = {i: max_memory for i in range(n_gpus)} else: max_memory = None + if torch_dtype == 'bf16' or torch_dtype == torch.bfloat16: + bf16 = True + else: + bf16 = False if isinstance(model, str): - model_dir = snapshot_download( - model) if not os.path.exists(model) else model - - config = read_config(model_dir) - model_config = QWenConfig.from_pretrained(model_dir) - model_config.torch_dtype = torch_dtype - - model = QWenForTextGeneration.from_pretrained( - model_dir, - cfg_dict=config, - config=model_config, + self.model = AutoModelForCausalLM.from_pretrained( + model, device_map=device_map, - torch_dtype=torch_dtype, - quantization_config=quantization_config, - max_memory=max_memory) - model.generation_config = GenerationConfig.from_pretrained( - model_dir) + revision=revision, + trust_remote_code=True, + bf16=bf16).eval() + self.tokenizer = AutoTokenizer.from_pretrained( + model, revision=revision, trust_remote_code=True) + self.model.generation_config = GenerationConfig.from_pretrained( + model) + else: + self.model = model + self.tokenizer = kwargs.get('tokenizer', None) - self.model = model - self.model.eval() - self.tokenizer = QWenTokenizer.from_pretrained(self.model.model_dir) - - super().__init__(model=model, **kwargs) + super().__init__(model=self.model, **kwargs) # skip pipeline model placement self._model_prepare = True @@ -411,10 +405,12 @@ class QWenTextGenerationPipeline(Pipeline): # define the forward pass def forward(self, inputs: str, **forward_params) -> Dict[str, Any]: + inputs = self.tokenizer(inputs, return_tensors='pt').to('cuda:0') return { OutputKeys.TEXT: - self.model.chat(self.tokenizer, inputs, - history=None)[OutputKeys.RESPONSE] + self.tokenizer.decode( + self.model.generate(**inputs).cpu()[0], + skip_special_tokens=True) } # format the outputs from pipeline diff --git a/modelscope/preprocessors/image.py b/modelscope/preprocessors/image.py index 36ab2f2f..187dd696 100644 --- a/modelscope/preprocessors/image.py +++ b/modelscope/preprocessors/image.py @@ -10,6 +10,7 @@ from PIL import Image, ImageOps from modelscope.fileio import File from modelscope.metainfo import Preprocessors +from modelscope.pipeline_inputs import InputKeys from modelscope.utils.constant import Fields from modelscope.utils.type_assert import type_assert from .base import Preprocessor @@ -92,6 +93,10 @@ class LoadImage: if len(input.shape) == 2: input = cv2.cvtColor(input, cv2.COLOR_GRAY2BGR) img = input[:, :, ::-1] + elif isinstance(input, Dict): + img = input.get(InputKeys.IMAGE, None) + if img: + img = np.array(load_image(img)) else: raise TypeError(f'input should be either str, PIL.Image,' f' np.array, but got {type(input)}') @@ -108,6 +113,10 @@ class LoadImage: img = cv2.cvtColor(input, cv2.COLOR_GRAY2BGR) img = input[:, :, ::-1] img = Image.fromarray(img.astype('uint8')).convert('RGB') + elif isinstance(input, Dict): + img = input.get(InputKeys.IMAGE, None) + if img: + img = load_image(img) else: raise TypeError(f'input should be either str, PIL.Image,' f' np.array, but got {type(input)}') diff --git a/modelscope/utils/input_output.py b/modelscope/utils/input_output.py index dcb4035f..dbe5861d 100644 --- a/modelscope/utils/input_output.py +++ b/modelscope/utils/input_output.py @@ -3,11 +3,13 @@ import ast import base64 import importlib import inspect +import os from io import BytesIO from typing import Any from urllib.parse import urlparse import cv2 +import json import numpy as np from modelscope.hub.api import HubApi @@ -243,6 +245,33 @@ def process_arg_type_annotation(arg, default_value): return arg.arg, 'object' +def convert_to_value(item): + if isinstance(item, ast.Str): + return item.s + elif hasattr(ast, 'Bytes') and isinstance(item, ast.Bytes): + return item.s + elif isinstance(item, ast.Tuple): + return tuple(convert_to_value(i) for i in item.elts) + elif isinstance(item, ast.Num): + return item.n + elif isinstance(item, ast.Name): + result = VariableKey(item=item) + constants_lookup = { + 'True': True, + 'False': False, + 'None': None, + } + return constants_lookup.get( + result.name, + result, + ) + elif isinstance(item, ast.NameConstant): + # None, True, False are nameconstants in python3, but names in 2 + return item.value + else: + return UnhandledKeyType() + + def process_args(args): arguments = [] # name, type, has_default, default @@ -259,7 +288,7 @@ def process_args(args): # process defaults arg. for arg, dft in zip(args.args[n_args - n_args_default:], args.defaults): # compatible with python3.7 ast.Num no value. - value = dft.value if hasattr(dft, 'value') else dft.n + value = convert_to_value(dft) arg_name, arg_type = process_arg_type_annotation(arg, value) arguments.append((arg_name, arg_type, True, value)) @@ -398,7 +427,7 @@ meta_type_schema_map = { def generate_pipeline_parameters_schema(parameters): parameters_schema = {'type': 'object', 'properties': {}} - if len(parameters) == 0: + if parameters is None or len(parameters) == 0: return {} for param in parameters: name, param_type, has_default, default_value = param @@ -523,16 +552,18 @@ def is_url(url: str): def decode_base64_to_image(content): - if content.startswith('http') or content.startswith('oss'): + if content.startswith('http') or content.startswith( + 'oss') or os.path.exists(content): return content from PIL import Image - image_file_content = base64.b64decode(content) + image_file_content = base64.b64decode(content, '-_') return Image.open(BytesIO(image_file_content)) def decode_base64_to_audio(content): - if content.startswith('http') or content.startswith('oss'): + if content.startswith('http') or content.startswith( + 'oss') or os.path.exists(content): return content file_content = base64.b64decode(content) @@ -540,7 +571,8 @@ def decode_base64_to_audio(content): def decode_base64_to_video(content): - if content.startswith('http') or content.startswith('oss'): + if content.startswith('http') or content.startswith( + 'oss') or os.path.exists(content): return content file_content = base64.b64decode(content) @@ -594,13 +626,14 @@ def call_pipeline_with_json(pipeline_info: PipelineInfomation, pipeline (Pipeline): The pipeline object. body (Dict): The input object, include input and parameters """ - if pipeline_info.is_custom_call: - pipeline_inputs = body['input'] - result = pipeline(**pipeline_inputs) - else: - pipeline_inputs, parameters = service_base64_input_to_pipeline_input( - pipeline_info.task_name, body) - result = pipeline(pipeline_inputs, **parameters) + # TODO: is_custom_call misjudgment + # if pipeline_info.is_custom_call: + # pipeline_inputs = body['input'] + # result = pipeline(**pipeline_inputs) + # else: + pipeline_inputs, parameters = service_base64_input_to_pipeline_input( + pipeline_info.task_name, body) + result = pipeline(pipeline_inputs, **parameters) return result @@ -737,6 +770,9 @@ def pipeline_output_to_service_base64_output(task_name, pipeline_output): task_outputs = [] if task_name in TASK_OUTPUTS: task_outputs = TASK_OUTPUTS[task_name] + # TODO: for batch + if isinstance(pipeline_output, list): + pipeline_output = pipeline_output[0] for key, value in pipeline_output.items(): if key not in task_outputs: continue # skip the output not defined. @@ -768,3 +804,77 @@ def pipeline_output_to_service_base64_output(task_name, pipeline_output): json_serializable_output[key] = value return _convert_to_python_type(json_serializable_output) + + +def get_task_input_examples(task): + current_work_dir = os.path.dirname(__file__) + with open(current_work_dir + '/pipeline_inputs.json', 'r') as f: + input_examples = json.load(f) + if task in input_examples: + return input_examples[task] + return None + + +def get_task_schemas(task): + current_work_dir = os.path.dirname(__file__) + with open(current_work_dir + '/pipeline_schema.json', 'r') as f: + schema = json.load(f) + if task in schema: + return schema[task] + return None + + +if __name__ == '__main__': + from modelscope.utils.ast_utils import load_index + index = load_index() + task_schemas = {} + for key, value in index['index'].items(): + reg, task_name, class_name = key + if reg == 'PIPELINES' and task_name != 'default': + print( + f"value['filepath']: {value['filepath']}, class_name: {class_name}" + ) + input, parameters = get_pipeline_input_parameters( + value['filepath'], class_name) + try: + if task_name in TASK_INPUTS and task_name in TASK_OUTPUTS: + # delete the first default input which is defined by task. + # parameters.pop(0) + parameters_schema = generate_pipeline_parameters_schema( + parameters) + input_schema = get_input_schema(task_name, None) + output_schema = get_output_schema(task_name) + schema = { + 'input': input_schema, + 'parameters': parameters_schema, + 'output': output_schema + } + else: + logger.warning( + 'Task: %s input is defined: %s, output is defined: %s which is not completed' + % (task_name, task_name in TASK_INPUTS, task_name + in TASK_OUTPUTS)) + input_schema = None + output_schema = None + if task_name in TASK_INPUTS: + input_schema = get_input_schema(task_name, None) + if task_name in TASK_OUTPUTS: + output_schema = get_output_schema(task_name) + parameters_schema = generate_pipeline_parameters_schema( + parameters) + schema = { + 'input': input_schema if input_schema else + parameters_schema, # all parameter is input + 'parameters': + parameters_schema if input_schema else {}, + 'output': output_schema if output_schema else { + 'type': 'object', + }, + } + except BaseException: + continue + task_schemas[task_name] = schema + + s = json.dumps(task_schemas) + with open('./task_schema.json', 'w') as f: + f.write(s) diff --git a/modelscope/utils/pipeline_inputs.json b/modelscope/utils/pipeline_inputs.json new file mode 100644 index 00000000..2ba31bcc --- /dev/null +++ b/modelscope/utils/pipeline_inputs.json @@ -0,0 +1,277 @@ +{ + "action-detection":{ + "input":{ + "video":"data/test/videos/action_detection_test_video.mp4" + } + }, + "action-recognition":{ + "input":{ + "video":"data/test/videos/action_recognition_test_video.mp4" + } + }, + "animal-recognition":{ + "input":{ + "image":"data/test/images/dogs.jpg" + } + }, + "chat":{ + "input":{ + "text":"你有什么推荐吗？", + "history":[ + [ + "今天天气真好，", + "今天天气真好，出去走走怎么样？" + ] + ] + } + }, + "domain-specific-object-detection":{ + "input":{ + "image":"data/test/images/image_traffic_sign.jpg" + } + }, + "face-2d-keypoints":{ + "input":{ + "image":"data/test/images/face_detection.png" + } + }, + "face-attribute-recognition":{ + "input":{ + "image":"data/test/images/face_recognition_1.png" + } + }, + "facial-expression-recognition":{ + "input":{ + "image":"data/test/images/facial_expression_recognition.jpg" + } + }, + "general-recognition":{ + "input":{ + "image":"data/test/images/dogs.jpg" + } + }, + "human-detection":{ + "input":{ + "image":"data/test/images/image_detection.jpg" + } + }, + "image-captioning":{ + "input":{ + "image":"data/test/images/image_captioning.png" + } + }, + "image-classification":{ + "input":{ + "image":"data/test/images/content_check.jpg" + } + }, + "image-demoireing":{ + "input":{ + "image":"data/test/images/shop_segmentation.jpg" + } + }, + "image-object-detection":{ + "input":{ + "image":"data/test/images/image_detection.jpg" + } + }, + "image-portrait-stylization":{ + "input":{ + "image":"https://modelscope.oss-cn-beijing.aliyuncs.com/test/images/image_cartoon.png" + } + }, + "image-segmentation":{ + "input":{ + "image":"data/test/images/image_semantic_segmentation.jpg" + }, + "parameters":{ + + } + }, + "image-text-retrieval":{ + "input":{ + "image":"data/test/images/image_mplug_vqa.jpg", + "text":"What is the woman doing?" + } + }, + "indoor-layout-estimation":{ + "input":{ + "image":"data/test/images/image_traffic_sign.jpg" + } + }, + "live-category":{ + "input":{ + "video":"data/test/videos/live_category_test_video.mp4" + } + }, + "motion-generation":{ + "input":{ + "text":"the person walked forward and is picking up his toolbox" + } + }, + "named-entity-recognition":{ + "input":{ + "text":"这与温岭市新河镇的一个神秘的传说有关。[SEP]地名" + } + }, + "nli":{ + "input":[ + "四川商务职业学院和四川财经职业学院哪个好？", + "四川商务职业学院商务管理在哪个校区？" + ], + "parameters":{ + + } + }, + "ocr-recognition":{ + "input":{ + "image":"data/test/images/image_ocr_recognition.jpg" + } + }, + "panorama-depth-estimation":{ + "input":{ + "image":"data/test/images/panorama_depth_estimation.jpg" + } + }, + "semantic-segmentation":{ + "input":{ + "image":"data/test/images/image_salient_detection.jpg" + } + }, + "shop-segmentation":{ + "input":{ + "image":"data/test/images/shop_segmentation.jpg" + } + }, + "text-classification":{ + "input":{ + "text":"i like this wonderful place" + }, + "parameters":{ + + } + }, + "text-driven-segmentation":{ + "input":{ + "image":"data/test/images/text_driven_segmentation.jpg", + "text":"bear" + } + }, + "text-generation":{ + "input":{ + "text":"蒙古国的首都是乌兰巴托（Ulaanbaatar）\n冰岛的首都是雷克雅未克（Reykjavik）\n埃塞俄比亚的首都是" + }, + "parameters":{ + + } + }, + "text-ranking":{ + "input":{ + "source_sentence":[ + "how long it take to get a master's degree" + ], + "sentences_to_compare":[ + "On average, students take about 18 to 24 months to complete a master's degree.", + "On the other hand, some students prefer to go at a slower pace and choose to take several years to complete their studies.", + "It can take anywhere from two semesters" + ] + } + }, + "text-summarization":{ + "input":{ + "text":"five-time world champion michelle kwan withdrew from the #### us figure skating championships on wednesday , but will petition us skating officials for the chance to compete at the #### turin olympics ." + } + }, + "text-to-video-synthesis":{ + "input":{ + "text":"A panda eating bamboo on a rock." + } + }, + "translation":{ + "input":{ + "text":"声明补充说，沃伦的同事都深感震惊，并且希望他能够投案自首。" + } + }, + "video-captioning":{ + "input":{ + "video":"data/test/videos/video_caption_and_qa_test.mp4" + } + }, + "video-category":{ + "input":{ + "video":"data/test/videos/video_category_test_video.mp4" + } + }, + "video-depth-estimation":{ + "input":{ + "video":"data/test/videos/video_depth_estimation.mp4" + } + }, + "video-embedding":{ + "input":{ + "video":"data/test/videos/action_recognition_test_video.mp4" + } + }, + "video-multi-object-tracking":{ + "input":{ + "video":"data/test/videos/MOT17-03-partial.mp4" + } + }, + "video-panoptic-segmentation":{ + "input":{ + "video":"data/test/videos/kitti-step_testing_image_02_0000.mp4" + } + }, + "video-question-answering":{ + "input":{ + "video":"data/test/videos/video_caption_and_qa_test.mp4", + "text":"How many people are there?" + } + }, + "video-summarization":{ + "input":{ + "text":"data/test/videos/video_category_test_video.mp4" + } + }, + "visual-entailment":{ + "input":{ + "image":"data/test/images/dogs.jpg", + "text":"there are two birds." + } + }, + "visual-grounding":{ + "input":{ + "image":"data/test/images/visual_grounding.png", + "text":"a blue turtle-like pokemon with round head" + } + }, + "visual-question-answering":{ + "input":{ + "image":"data/test/images/image_mplug_vqa.jpg", + "text":"What is the woman doing?" + } + }, + "word-segmentation":{ + "input":{ + "text":"今天天气不错，适合出去游玩" + } + }, + "zero-shot-classification":{ + "input":{ + "text":"全新突破 解放军运20版空中加油机曝光" + }, + "parameters":{ + "candidate_labels":[ + "文化", + "体育", + "娱乐", + "财经", + "家居", + "汽车", + "教育", + "科技", + "军事" + ] + } + } +} diff --git a/modelscope/utils/pipeline_schema.json b/modelscope/utils/pipeline_schema.json new file mode 100644 index 00000000..cf5c7fb7 --- /dev/null +++ b/modelscope/utils/pipeline_schema.json @@ -0,0 +1,3781 @@ +{ + "acoustic-echo-cancellation": { + "input": { + "type": "object", + "properties": { + "nearend_mic": { + "type": "string", + "description": "Base64 encoded audio file or url string.." + }, + "farend_speech": { + "type": "string", + "description": "Base64 encoded audio file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_pcm": { + "type": "string", + "description": "The base64 encoded PCM." + } + } + } + }, + "acoustic-noise-suppression": { + "input": { + "type": "object", + "properties": { + "audio": { + "type": "string", + "description": "Base64 encoded audio file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_pcm": { + "type": "string", + "description": "The base64 encoded PCM." + } + } + } + }, + "action-detection": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "timestamps": { + "type": "string" + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "action-recognition": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "animal-recognition": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "auto-speech-recognition": { + "input": { + "type": "object", + "properties": { + "wav": { + "type": "string", + "description": "Base64 encoded audio file or url string.." + }, + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "bad-image-detecting": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "body-2d-keypoints": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "keypoints": { + "type": "array", + "items": { + "type": "number" + } + }, + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "body-3d-keypoints": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "keypoints": { + "type": "array", + "items": { + "type": "number" + } + }, + "timestamps": { + "type": "string" + }, + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "card-detection": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "keypoints": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "chat": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + }, + "history": { + "type": "array" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "response": { + "type": "object" + }, + "history": { + "type": "object" + } + } + } + }, + "code-generation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "code-translation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "competency-aware-translation": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object" + } + }, + "controllable-image-generation": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "prompt": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "crowd-counting": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "document-grounded-dialog-generate": { + "input": { + "type": "object", + "properties": { + "query": { + "type": "array" + }, + "context": { + "type": "array" + }, + "label": { + "type": "array" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "document-grounded-dialog-rerank": { + "input": { + "type": "object", + "properties": { + "dataset": { + "type": "array" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "document-grounded-dialog-retrieval": { + "input": { + "type": "object", + "properties": { + "query": { + "type": "array" + }, + "positive": { + "type": "array" + }, + "negative": { + "type": "array" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "document-segmentation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "document-vl-embedding": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "img_embedding": { + "type": "array", + "items": { + "type": "number" + } + }, + "text_embedding": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "domain-specific-object-detection": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "efficient-diffusion-tuning": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "extractive-summarization": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "face-2d-keypoints": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "keypoints": { + "type": "array", + "items": { + "type": "number" + } + }, + "poses": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "face-attribute-recognition": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "face-detection": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "keypoints": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "face-emotion": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "face-human-hand-detection": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "scores": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "face-image-generation": { + "input": { + "type": "object", + "properties": { + "number": { + "type": "integer" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "face-liveness": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "face-quality-assessment": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "face-recognition": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "img_embedding": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "face-reconstruction": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "facial-expression-recognition": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "faq-question-answering": { + "input": { + "type": "object", + "properties": { + "query_set": { + "type": "array" + }, + "support_set": { + "type": "array" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "feature-extraction": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text_embedding": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "fid-dialogue": { + "input": { + "type": "object", + "properties": { + "history": { + "type": "string", + "description": "The input text." + }, + "knowledge": { + "type": "string", + "description": "The input text." + }, + "bot_profile": { + "type": "string", + "description": "The input text." + }, + "user_profile": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "fill-mask": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "general-recognition": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "generative-multi-modal-embedding": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "img_embedding": { + "type": "array", + "items": { + "type": "number" + } + }, + "text_embedding": { + "type": "array", + "items": { + "type": "number" + } + }, + "caption": { + "type": "string" + } + } + } + }, + "hand-static": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "human-detection": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "human-reconstruction": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "image-body-reshaping": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-captioning": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "caption": { + "type": "string" + } + } + } + }, + "image-classification": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "image-color-enhancement": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-colorization": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-debanding": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "image-deblurring": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "image-demoireing": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "image-denoising": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-depth-estimation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "image-driving-perception": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "masks": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "image-face-fusion": { + "input": { + "type": "object", + "properties": { + "template": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "user": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "user": { + "type": "object", + "default": null + } + } + }, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-fewshot-detection": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object" + } + }, + "image-inpainting": { + "input": { + "type": "object", + "properties": { + "img": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "mask": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-matching": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "image-multi-view-depth-estimation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "image-object-detection": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "image-paintbyexample": { + "input": { + "type": "object", + "properties": { + "img": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "mask": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "reference": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-portrait-enhancement": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-portrait-stylization": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-quality-assessment-degradation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "image-quality-assessment-mos": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "score": { + "type": "number" + } + } + } + }, + "image-reid-person": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "img_embedding": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "image-segmentation": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "min_size": { + "type": "integer", + "default": 640 + }, + "max_size": { + "type": "integer", + "default": 1333 + }, + "score_thr": { + "type": "number", + "default": 0 + } + } + }, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "masks": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "image-skychange": { + "input": { + "type": "object", + "properties": { + "sky_image": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "scene_image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-style-transfer": { + "input": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "style": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "style": { + "type": "object", + "default": null + } + } + }, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-super-resolution": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-text-retrieval": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "image-to-image-generation": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-to-image-translation": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "image-try-on": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "indoor-layout-estimation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "information-extraction": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "inverse-text-processing": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "keyword-spotting": { + "input": { + "type": "object", + "properties": { + "audio": { + "type": "string", + "description": "Base64 encoded audio file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "kws_list": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "language-guided-video-summarization": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "language-score-prediction": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "license-plate-detection": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "polygons": { + "type": "array", + "items": { + "type": "number" + } + }, + "text": { + "type": "string" + } + } + } + }, + "lineless-table-recognition": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "polygons": { + "type": "array", + "items": { + "type": "number" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "live-category": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "motion-generation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "keypoints": { + "type": "array", + "items": { + "type": "number" + } + }, + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "movie-scene-segmentation": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "shot_num": { + "type": "integer" + }, + "shot_meta_list": { + "type": "array", + "items": { + "type": "integer" + } + }, + "scene_num": { + "type": "integer" + }, + "scene_meta_list": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, + "multi-modal-embedding": { + "input": { + "type": "object", + "properties": { + "img": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "img_embedding": { + "type": "array", + "items": { + "type": "number" + } + }, + "text_embedding": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "multi-modal-similarity": { + "input": { + "type": "object", + "properties": { + "img": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "img_embedding": { + "type": "array", + "items": { + "type": "number" + } + }, + "text_embedding": { + "type": "array", + "items": { + "type": "number" + } + }, + "scores": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "multimodal-dialogue": { + "input": { + "type": "object", + "properties": { + "messages": { + "type": "array" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "named-entity-recognition": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "nerf-recon-4k": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "nerf-recon-acc": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "nerf-recon-vq-compression": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "nli": { + "input": { + "type": "array", + "items": { + "type": "string", + "description": "The input text." + } + }, + "parameters": { + "type": "object", + "properties": { + "topk": { + "type": "integer", + "default": null + } + } + }, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "object-detection-3d": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "ocr-detection": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "polygons": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "ocr-recognition": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "open-vocabulary-detection": { + "input": { + "type": "object", + "properties": { + "img": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "category_names": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "panorama-depth-estimation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "part-of-speech": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "pedestrian-attribute-recognition": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "pointcloud-sceneflow-estimation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "portrait-matting": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "product-retrieval-embedding": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "img_embedding": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "product-segmentation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "masks": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "protein-structure": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "punctuation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "referring-video-object-segmentation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "masks": { + "type": "array", + "items": { + "type": "number" + } + }, + "timestamps": { + "type": "string" + }, + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "relation-extraction": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "spo_list": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "semantic-segmentation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "masks": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "sentence-embedding": { + "input": { + "type": "object", + "properties": { + "source_sentence": { + "type": "array" + }, + "sentences_to_compare": { + "type": "array" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text_embedding": { + "type": "array", + "items": { + "type": "number" + } + }, + "scores": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "sentence-similarity": { + "input": { + "type": "array", + "items": { + "type": "string", + "description": "The input text." + } + }, + "parameters": { + "type": "object", + "properties": { + "topk": { + "type": "integer", + "default": null + } + } + }, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "sentiment-classification": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "topk": { + "type": "integer", + "default": null + } + } + }, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "shop-segmentation": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "masks": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "siamese-uie": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "skin-retouching": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "speaker-diarization": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "speaker-diarization-dialogue-detection": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "topk": { + "type": "integer", + "default": null + } + } + }, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "speaker-diarization-semantic-speaker-turn-detection": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "logits": { + "type": "array", + "items": { + "type": "number" + } + }, + "text": { + "type": "string" + }, + "prediction": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "speaker-verification": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "speech-language-recognition": { + "input": { + "type": "object", + "properties": { + "audio": { + "type": "string", + "description": "Base64 encoded audio file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "speech-separation": { + "input": { + "type": "object", + "properties": { + "audio": { + "type": "string", + "description": "Base64 encoded audio file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_pcm_list": { + "type": "array", + "items": { + "type": "string", + "description": "The base64 encoded PCM." + } + } + } + } + }, + "speech-timestamp": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "sudoku": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "table-question-answering": { + "input": { + "type": "object", + "properties": { + "question": { + "type": "string", + "description": "The input text." + }, + "history_sql": { + "type": "object" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "table-recognition": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "polygons": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "task-oriented-conversation": { + "input": { + "type": "object", + "properties": { + "user_input": { + "type": "string", + "description": "The input text." + }, + "history": { + "type": "object" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "task-template": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "max_length": { + "type": "integer", + "default": 1024 + }, + "top_p": { + "type": "number", + "default": 0.8 + }, + "postprocess_param1": { + "type": "string", + "default": null + } + } + }, + "output": { + "type": "object", + "properties": { + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "output_img": { + "type": "string", + "description": "The base64 encoded image." + }, + "text_embedding": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "text-classification": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + }, + "text2": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "topk": { + "type": "integer", + "default": null + } + } + }, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "text-driven-segmentation": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "masks": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "text-error-correction": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "text-generation": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "max_length": { + "type": "integer", + "default": 50 + }, + "do_sample": { + "type": "boolean", + "default": true + }, + "top_p": { + "type": "number", + "default": 0.85 + }, + "temperature": { + "type": "number", + "default": 1 + }, + "repetition_penalty": { + "type": "number", + "default": 1 + }, + "eos_token_id": { + "type": "integer", + "default": 2 + }, + "bos_token_id": { + "type": "integer", + "default": 1 + }, + "pad_token_id": { + "type": "integer", + "default": 0 + } + } + }, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "text-ranking": { + "input": { + "type": "object", + "properties": { + "source_sentence": { + "type": "array" + }, + "sentences_to_compare": { + "type": "array" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "text-summarization": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "text-to-360panorama-image": { + "input": { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "text-to-image-synthesis": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "init": { + "type": "object", + "default": null + }, + "init_scale": { + "type": "integer", + "default": 2000 + }, + "skip_steps": { + "type": "integer", + "default": 10 + }, + "randomize_class": { + "type": "boolean", + "default": true + }, + "eta": { + "type": "number", + "default": 0.8 + }, + "output_type": { + "type": "string", + "default": "pil" + }, + "return_dict": { + "type": "boolean", + "default": true + }, + "clip_guidance_scale": { + "type": "integer", + "default": 7500 + } + } + }, + "output": { + "type": "object", + "properties": { + "output_imgs": { + "type": "array", + "items": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + } + }, + "text-to-speech": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_wav": { + "type": "string", + "description": "The base64 encoded WAV." + } + } + } + }, + "text-to-video-synthesis": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "text2sql": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + }, + "database": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "text2text-generation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "token-classification": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "translation": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "translation": { + "type": "string" + } + } + } + }, + "translation-evaluation": { + "input": { + "type": "object", + "properties": { + "hyp": { + "type": "array" + }, + "src": { + "type": "array" + }, + "ref": { + "type": "array" + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "score": { + "type": "number" + } + } + } + }, + "universal-matting": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "video-captioning": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "caption": { + "type": "string" + } + } + } + }, + "video-category": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "video-colorization": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "video-deinterlace": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "video-depth-estimation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "video-embedding": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "video_embedding": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "video-frame-interpolation": { + "input": { + "type": "object", + "properties": { + "out_fps": { + "type": "number", + "default": 0 + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "video-human-matting": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "masks": { + "type": "array", + "items": { + "type": "number" + } + }, + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "video-inpainting": { + "input": { + "type": "object", + "properties": { + "video_input_path": { + "type": "string", + "description": "The input text." + }, + "video_output_path": { + "type": "string", + "description": "The input text." + }, + "mask_path": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "video-instance-segmentation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "video-multi-modal-embedding": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "video-multi-object-tracking": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "timestamps": { + "type": "string" + } + } + } + }, + "video-object-detection": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "video-object-segmentation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "masks": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "video-panoptic-segmentation": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "masks": { + "type": "array", + "items": { + "type": "number" + } + }, + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "uuid": { + "type": "string" + } + } + } + }, + "video-question-answering": { + "input": { + "type": "object", + "properties": { + "video": { + "type": "string", + "description": "Base64 encoded video file or url string.." + }, + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "video-single-object-tracking": { + "input": { + "type": "array", + "items": { + "type": "string", + "description": "Base64 encoded video file or url string.." + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "timestamps": { + "type": "string" + } + } + } + }, + "video-stabilization": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "video-summarization": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "video-super-resolution": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_video": { + "type": "string", + "description": "The base64 encoded video." + } + } + } + }, + "video-temporal-grounding": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "tbounds": { + "type": "object" + } + } + } + }, + "video-text-retrieval": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "virtual-try-on": { + "input": { + "type": "array", + "items": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output_img": { + "type": "string", + "description": "The base64 encoded image." + } + } + } + }, + "vision-efficient-tuning": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "visual-entailment": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "text": { + "type": "string", + "description": "The input text." + }, + "text2": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "visual-grounding": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "boxes": { + "type": "array", + "items": { + "type": "number" + } + }, + "scores": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "visual-question-answering": { + "input": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Base64 encoded image file or url string." + }, + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "text": { + "type": "string" + } + } + } + }, + "voice-activity-detection": { + "input": {}, + "parameters": {}, + "output": { + "type": "object" + } + }, + "word-alignment": { + "input": {}, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "word-segmentation": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": {}, + "output": { + "type": "object", + "properties": { + "output": { + "type": "object" + } + } + } + }, + "zero-shot-classification": { + "input": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "The input text." + } + } + }, + "parameters": { + "type": "object", + "properties": { + "candidate_labels": { + "type": "object" + }, + "multi_label": { + "type": "boolean", + "default": false + } + } + }, + "output": { + "type": "object", + "properties": { + "scores": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } +} diff --git a/tests/json_call_test.py b/tests/json_call_test.py new file mode 100644 index 00000000..658c947f --- /dev/null +++ b/tests/json_call_test.py @@ -0,0 +1,76 @@ +import os + +import json + +from modelscope.hub.api import HubApi +from modelscope.hub.file_download import model_file_download +from modelscope.hub.utils.utils import get_cache_dir +from modelscope.pipelines import pipeline +from modelscope.utils.config import Config +from modelscope.utils.constant import ModelFile +from modelscope.utils.input_output import ( + call_pipeline_with_json, get_pipeline_information_by_pipeline, + get_task_input_examples, pipeline_output_to_service_base64_output) + + +class ModelJsonTest: + + def __init__(self): + self.api = HubApi() + + def test_single(self, model_id: str, model_revision=None): + # get model_revision & task info + cache_root = get_cache_dir() + configuration_file = os.path.join(cache_root, model_id, + ModelFile.CONFIGURATION) + if not model_revision: + model_revision = self.api.list_model_revisions( + model_id=model_id)[0] + if not os.path.exists(configuration_file): + + configuration_file = model_file_download( + model_id=model_id, + file_path=ModelFile.CONFIGURATION, + revision=model_revision) + cfg = Config.from_file(configuration_file) + task = cfg.safe_get('task') + + # init pipeline + ppl = pipeline( + task=task, model=model_id, model_revision=model_revision) + pipeline_info = get_pipeline_information_by_pipeline(ppl) + + # call pipeline + data = get_task_input_examples(task) + print(task, data) + infer_result = call_pipeline_with_json(pipeline_info, ppl, data) + result = pipeline_output_to_service_base64_output(task, infer_result) + return result + + +if __name__ == '__main__': + model_list = [ + 'damo/nlp_structbert_nli_chinese-base', + 'damo/nlp_structbert_word-segmentation_chinese-base', + 'damo/nlp_structbert_zero-shot-classification_chinese-base', + 'damo/cv_unet_person-image-cartoon_compound-models', + 'damo/nlp_structbert_sentiment-classification_chinese-tiny', + 'damo/nlp_csanmt_translation_zh2en', + 'damo/nlp_rom_passage-ranking_chinese-base', + 'damo/ofa_image-caption_muge_base_zh', + 'damo/nlp_raner_named-entity-recognition_chinese-base-ecom-50cls', + 'damo/nlp_structbert_sentiment-classification_chinese-ecommerce-base', + 'damo/text-to-video-synthesis', + 'qwen/Qwen-7B', + 'qwen/Qwen-7B-Chat', + 'ZhipuAI/ChatGLM-6B', + ] + tester = ModelJsonTest() + for model in model_list: + try: + res = tester.test_single(model) + print(f'\nmodel_id {model} call_pipeline_with_json run ok.\n') + except BaseException as e: + print( + f'\nmodel_id {model} call_pipeline_with_json run failed: {e}.\n' + ) diff --git a/tests/utils/case_file_analyzer.py b/tests/utils/case_file_analyzer.py index f0445954..63be95bd 100644 --- a/tests/utils/case_file_analyzer.py +++ b/tests/utils/case_file_analyzer.py @@ -62,7 +62,10 @@ class AnalysisTestFile(ast.NodeVisitor): class AnalysisTestClass(ast.NodeVisitor): - def __init__(self, test_class_node, builder_function_name) -> None: + def __init__(self, + test_class_node, + builder_function_name, + file_analyzer=None) -> None: super().__init__() self.test_class_node = test_class_node self.builder_function_name = builder_function_name @@ -72,6 +75,44 @@ class AnalysisTestClass(ast.NodeVisitor): ] # class method trainer builder(call build_trainer) self.custom_class_method_builder_calls = [ ] # the builder call statement + self.variables = {} + + def get_variables(self, key: str): + if key in self.variables: + return self.variables[key] + return key + + def get_ast_value(self, statements): + if not isinstance(statements, list): + statements = [statements] + res = [] + for item in statements: + if isinstance(item, ast.Name): + res.append(self.get_variables(item.id)) + elif isinstance(item, ast.Attribute): + res.append(self.get_variables(item.value.id)) + elif isinstance(item, ast.Str): + res.append(self.get_variables(item.s)) + elif isinstance(item, ast.Dict): + keys = [i.s for i in item.keys] + values = self.get_ast_value(item.values) + res.append(dict(zip(keys, values))) + return res + + def get_final_variables(self, statement: ast.Assign): + if len(statement.targets) == 1 and \ + isinstance(statement.targets[0], ast.Name): + if isinstance(statement.value, ast.Call): + if isinstance(statement.value.func, ast.Attribute) and \ + isinstance(statement.value.func.value, ast.Name) and \ + statement.value.func.value.id == 'Image': + self.variables[str( + statement.targets[0].id)] = self.get_ast_value( + statement.value.args[0]) + else: + self.variables[str( + statement.targets[0].id)] = self.get_ast_value( + statement.value) def visit_FunctionDef(self, node: ast.FunctionDef) -> Any: if node.name.startswith('setUp'): @@ -83,6 +124,7 @@ class AnalysisTestClass(ast.NodeVisitor): self.setup_variables[str( statement.targets[0].attr)] = str( statement.value.attr) + self.get_final_variables(statement) elif node.name.startswith('test_'): self.test_methods.append(node) else: @@ -312,6 +354,48 @@ def analysis_trainer_test_suite(test_file, modified_register_modules): return tested_trainers +def get_test_parameters(test_method, analyzer): + for node in ast.walk(test_method): + func = None + if not isinstance(node, ast.FunctionDef): + continue + for statement in node.body: + if isinstance(statement, ast.Assign): + analyzer.get_final_variables(statement) + if not func and isinstance(statement, ast.Assign): + if isinstance(statement.value, ast.Call) and isinstance( + statement.value.func, ast.Name) and ( # noqa W504 + 'pipeline' in statement.value.func.id + or 'Pipeline' in statement.value.func.id): + func = statement.targets[0].id + if func and isinstance(statement, ast.Assign) and isinstance( + statement.value, ast.Call) and isinstance( + statement.value.func, ast.Name): + if statement.value.func.id == func: + inputs = statement.value.args + return analyzer.get_ast_value(inputs) + + +def analysis_pipeline_test_examples(test_file): + examples = [] + with open(test_file, 'rb') as tsf: + src = tsf.read() + test_root = ast.parse(src, test_file) + test_file_analyzer = AnalysisTestFile( + test_file, SYSTEM_PIPELINE_BUILDER_FUNCTION_NAME) + test_file_analyzer.visit(test_root) + + for test_class in test_file_analyzer.test_classes: + test_class_analyzer = AnalysisTestClass( + test_class, SYSTEM_PIPELINE_BUILDER_FUNCTION_NAME, + test_file_analyzer) + test_class_analyzer.visit(test_class) + for test_method in test_class_analyzer.test_methods: + parameters = get_test_parameters(test_method, test_class_analyzer) + examples.append(parameters) + return examples + + def analysis_pipeline_test_suite(test_file, modified_register_modules): tested_tasks = [] with open(test_file, 'rb') as tsf: @@ -413,7 +497,18 @@ def get_pipelines_trainers_test_info(register_modules): if __name__ == '__main__': - test_file = 'tests/pipelines/test_action_detection.py' - tasks = analysis_pipeline_test_suite(test_file, None) + all_pipeline_cases = [ + os.path.join(dp, f) for dp, dn, filenames in os.walk( + os.path.join(os.getcwd(), 'tests', 'pipelines')) for f in filenames + if os.path.splitext(f)[1] == '.py' + ] + for test_file in all_pipeline_cases: + print('\n', test_file) + tasks = analysis_pipeline_test_suite(test_file, None) + examples = analysis_pipeline_test_examples(test_file) - print(tasks) + from modelsope.metainfo import Tasks + for task, example in zip(tasks, examples): + task_convert = f't = Tasks.{task}' + exec(task_convert) + print(t, example)