108 lines
3.7 KiB
Python
108 lines
3.7 KiB
Python
from dataclasses import dataclass
|
|
from typing import List
|
|
|
|
from dashscope.api_entities.dashscope_response import (DashScopeAPIResponse,
|
|
DictMixin)
|
|
from dashscope.client.base_api import BaseApi
|
|
from dashscope.common.error import InputRequired, ModelRequired
|
|
from dashscope.common.utils import _get_task_group_and_task
|
|
from dashscope.utils.oss_utils import preprocess_message_element
|
|
|
|
|
|
@dataclass(init=False)
|
|
class MultiModalEmbeddingItemBase(DictMixin):
|
|
factor: float
|
|
|
|
def __init__(self, factor: float, **kwargs):
|
|
super().__init__(factor=factor, **kwargs)
|
|
|
|
|
|
@dataclass(init=False)
|
|
class MultiModalEmbeddingItemText(MultiModalEmbeddingItemBase):
|
|
text: str
|
|
|
|
def __init__(self, text: str, factor: float, **kwargs):
|
|
super().__init__(factor, **kwargs)
|
|
self.text = text
|
|
|
|
|
|
@dataclass(init=False)
|
|
class MultiModalEmbeddingItemImage(MultiModalEmbeddingItemBase):
|
|
image: str
|
|
|
|
def __init__(self, image: str, factor: float, **kwargs):
|
|
super().__init__(factor, **kwargs)
|
|
self.image = image
|
|
|
|
|
|
@dataclass(init=False)
|
|
class MultiModalEmbeddingItemAudio(MultiModalEmbeddingItemBase):
|
|
audio: str
|
|
|
|
def __init__(self, audio: str, factor: float, **kwargs):
|
|
super().__init__(factor, **kwargs)
|
|
self.audio = audio
|
|
|
|
|
|
class MultiModalEmbedding(BaseApi):
|
|
task = 'multimodal-embedding'
|
|
|
|
class Models:
|
|
multimodal_embedding_one_peace_v1 = 'multimodal-embedding-one-peace-v1'
|
|
|
|
@classmethod
|
|
def call(cls,
|
|
model: str,
|
|
input: List[MultiModalEmbeddingItemBase],
|
|
api_key: str = None,
|
|
**kwargs) -> DashScopeAPIResponse:
|
|
"""Get embedding multimodal contents..
|
|
|
|
Args:
|
|
model (str): The embedding model name.
|
|
input (List[MultiModalEmbeddingElement]): The embedding elements,
|
|
every element include data, modal, factor field.
|
|
**kwargs:
|
|
auto_truncation(bool, `optional`): Automatically truncate
|
|
audio longer than 15 seconds or text longer than 70 words.
|
|
Default to false(Too long input will result in failure).
|
|
|
|
Returns:
|
|
DashScopeAPIResponse: The embedding result.
|
|
"""
|
|
if input is None or not input:
|
|
raise InputRequired('prompt is required!')
|
|
if model is None or not model:
|
|
raise ModelRequired('Model is required!')
|
|
embedding_input = {}
|
|
has_upload = cls._preprocess_message_inputs(model, input, api_key)
|
|
if has_upload:
|
|
headers = kwargs.pop('headers', {})
|
|
headers['X-DashScope-OssResourceResolve'] = 'enable'
|
|
kwargs['headers'] = headers
|
|
embedding_input['contents'] = input
|
|
kwargs.pop('stream', False) # not support streaming output.
|
|
task_group, function = _get_task_group_and_task(__name__)
|
|
return super().call(model=model,
|
|
input=embedding_input,
|
|
task_group=task_group,
|
|
task=MultiModalEmbedding.task,
|
|
function=function,
|
|
api_key=api_key,
|
|
**kwargs)
|
|
|
|
@classmethod
|
|
def _preprocess_message_inputs(cls, model: str, input: List[dict],
|
|
api_key: str):
|
|
"""preprocess following inputs
|
|
input = [{'factor': 1, 'text': 'hello'},
|
|
{'factor': 2, 'audio': ''},
|
|
{'factor': 3, 'image': ''}]
|
|
"""
|
|
has_upload = False
|
|
for elem in input:
|
|
is_upload = preprocess_message_element(model, elem, api_key)
|
|
if is_upload and not has_upload:
|
|
has_upload = True
|
|
return has_upload
|