162 lines
6.4 KiB
Python
162 lines
6.4 KiB
Python
import mimetypes
|
|
import os
|
|
from datetime import datetime
|
|
from http import HTTPStatus
|
|
from time import mktime
|
|
from typing import List
|
|
from urllib.parse import unquote_plus, urlparse
|
|
from wsgiref.handlers import format_date_time
|
|
|
|
import requests
|
|
|
|
from dashscope.api_entities.dashscope_response import DashScopeAPIResponse
|
|
from dashscope.client.base_api import GetMixin
|
|
from dashscope.common.constants import (FILE_PATH_SCHEMA,
|
|
REQUEST_CONTENT_AUDIO,
|
|
REQUEST_CONTENT_IMAGE,
|
|
REQUEST_CONTENT_TEXT)
|
|
from dashscope.common.error import InvalidInput, UploadFileException
|
|
from dashscope.common.logging import logger
|
|
from dashscope.common.utils import get_user_agent
|
|
|
|
|
|
class OssUtils(GetMixin):
|
|
SUB_PATH = 'uploads'
|
|
|
|
@classmethod
|
|
def _decode_response_error(cls, response: requests.Response):
|
|
if 'application/json' in response.headers.get('content-type', ''):
|
|
message = response.json()
|
|
else:
|
|
message = response.content.decode('utf-8')
|
|
return message
|
|
|
|
@classmethod
|
|
def upload(cls,
|
|
model: str,
|
|
file_path: str,
|
|
api_key: str = None,
|
|
**kwargs) -> DashScopeAPIResponse:
|
|
"""Upload file for model fine-tune or other tasks.
|
|
|
|
Args:
|
|
file_path (str): The local file name to upload.
|
|
purpose (str): The purpose of the file[fine-tune|inference]
|
|
description (str, optional): The file description message.
|
|
api_key (str, optional): The api key. Defaults to None.
|
|
|
|
Returns:
|
|
DashScopeAPIResponse: The upload information
|
|
"""
|
|
upload_info = cls.get_upload_certificate(model=model, api_key=api_key)
|
|
if upload_info.status_code != HTTPStatus.OK:
|
|
raise UploadFileException(
|
|
'Get upload certificate failed, code: %s, message: %s' %
|
|
(upload_info.code, upload_info.message))
|
|
upload_info = upload_info.output
|
|
headers = {}
|
|
headers = {'user-agent': get_user_agent()}
|
|
headers['Accept'] = 'application/json'
|
|
headers['Date'] = format_date_time(mktime(datetime.now().timetuple()))
|
|
form_data = {}
|
|
form_data['OSSAccessKeyId'] = upload_info['oss_access_key_id']
|
|
form_data['Signature'] = upload_info['signature']
|
|
form_data['policy'] = upload_info['policy']
|
|
form_data['key'] = upload_info['upload_dir'] + \
|
|
'/' + os.path.basename(file_path)
|
|
form_data['x-oss-object-acl'] = upload_info['x_oss_object_acl']
|
|
form_data['x-oss-forbid-overwrite'] = upload_info[
|
|
'x_oss_forbid_overwrite']
|
|
form_data['success_action_status'] = '200'
|
|
form_data['x-oss-content-type'] = mimetypes.guess_type(file_path)[0]
|
|
url = upload_info['upload_host']
|
|
files = {'file': open(file_path, 'rb')}
|
|
with requests.Session() as session:
|
|
response = session.post(url,
|
|
files=files,
|
|
data=form_data,
|
|
headers=headers,
|
|
timeout=3600)
|
|
if response.status_code == HTTPStatus.OK:
|
|
return 'oss://' + form_data['key']
|
|
else:
|
|
msg = (
|
|
'Uploading file: %s to oss failed, error: %s' %
|
|
(file_path, cls._decode_response_error(response=response)))
|
|
logger.error(msg)
|
|
raise UploadFileException(msg)
|
|
|
|
@classmethod
|
|
def get_upload_certificate(cls,
|
|
model: str,
|
|
api_key: str = None,
|
|
**kwargs) -> DashScopeAPIResponse:
|
|
"""Get a oss upload certificate.
|
|
|
|
Args:
|
|
api_key (str, optional): The api key. Defaults to None.
|
|
|
|
Returns:
|
|
DashScopeAPIResponse: The job info
|
|
"""
|
|
params = {'action': 'getPolicy'}
|
|
params['model'] = model
|
|
return super().get(None, api_key, params=params, **kwargs)
|
|
|
|
|
|
def upload_file(model: str, upload_path: str, api_key: str):
|
|
if upload_path.startswith(FILE_PATH_SCHEMA):
|
|
parse_result = urlparse(upload_path)
|
|
if parse_result.netloc:
|
|
file_path = parse_result.netloc + unquote_plus(parse_result.path)
|
|
else:
|
|
file_path = unquote_plus(parse_result.path)
|
|
if os.path.exists(file_path):
|
|
file_url = OssUtils.upload(model=model,
|
|
file_path=file_path,
|
|
api_key=api_key)
|
|
if file_url is None:
|
|
raise UploadFileException('Uploading file: %s failed' %
|
|
upload_path)
|
|
return file_url
|
|
else:
|
|
raise InvalidInput('The file: %s is not exists!' % file_path)
|
|
return None
|
|
|
|
|
|
def check_and_upload(model, elem: dict, key: str, api_key):
|
|
is_upload = False
|
|
content = elem[key]
|
|
if content.startswith(FILE_PATH_SCHEMA):
|
|
parse_result = urlparse(content)
|
|
if parse_result.netloc:
|
|
file_path = parse_result.netloc + unquote_plus(parse_result.path)
|
|
else:
|
|
file_path = unquote_plus(parse_result.path)
|
|
if os.path.exists(file_path):
|
|
file_url = OssUtils.upload(model=model,
|
|
file_path=file_path,
|
|
api_key=api_key)
|
|
if file_url is None:
|
|
raise UploadFileException('Uploading file: %s failed' %
|
|
content)
|
|
elem[key] = file_url
|
|
is_upload = True
|
|
else:
|
|
raise InvalidInput('The file: %s is not exists!' % file_path)
|
|
return is_upload
|
|
|
|
|
|
def preprocess_message_element(model: str, elem: List[dict], api_key: str):
|
|
is_upload = False
|
|
if REQUEST_CONTENT_TEXT in elem:
|
|
is_upload = check_and_upload(model, elem, REQUEST_CONTENT_TEXT,
|
|
api_key)
|
|
elif REQUEST_CONTENT_IMAGE in elem:
|
|
is_upload = check_and_upload(model, elem, REQUEST_CONTENT_IMAGE,
|
|
api_key)
|
|
elif REQUEST_CONTENT_AUDIO in elem:
|
|
is_upload = check_and_upload(model, elem, REQUEST_CONTENT_AUDIO,
|
|
api_key)
|
|
return is_upload
|