113 lines
3.8 KiB
Python
113 lines
3.8 KiB
Python
import base64
|
|
import io
|
|
from typing import Generator
|
|
|
|
from dashscope.common.error import UnsupportedDataType
|
|
|
|
|
|
class InputResolver:
|
|
def __init__(self,
|
|
input_instance,
|
|
is_encode_binary: bool = True,
|
|
custom_type_resolver: dict = {}):
|
|
self._instance = input_instance
|
|
self._is_encode_binary = is_encode_binary
|
|
self._custom_type_resolver = custom_type_resolver
|
|
|
|
def __next__(self):
|
|
while True:
|
|
return resolve_input(self._instance, self._is_encode_binary,
|
|
self._custom_type_resolver)
|
|
|
|
def __iter__(self):
|
|
return self
|
|
|
|
|
|
def _is_binary_file(bytes):
|
|
# solution from: https://stackoverflow.com/questions/898669/
|
|
# how-can-i-detect-if-a-file-is-binary-non-text-in-python/7392391#7392391
|
|
text_chars = bytearray({7, 8, 9, 10, 12, 13, 27}
|
|
| set(range(0x20, 0x100)) - {0x7f})
|
|
if len(bytes) > 1024:
|
|
temp_bytes = bytes[:1024]
|
|
else:
|
|
temp_bytes = bytes
|
|
return bool(temp_bytes.translate(None, text_chars))
|
|
|
|
|
|
def resolve_input(input, is_encode_binary, custom_type_resolver: dict = {}):
|
|
"""Resolve input data, if same field is file, generator, we can get data.
|
|
|
|
Args:
|
|
input (object): object
|
|
is_encode_binary (bool): is encode binary, websocket support binary,
|
|
no need encode.
|
|
custom_type_resolver (map): key: the data type, value,
|
|
the data convert, which convert data type key, to jsonable object.
|
|
def ndarray_tolist(ndar):
|
|
return ndar.tolist()
|
|
{numpy.ndarray: ndarray_tolist}
|
|
|
|
Raises:
|
|
UnsupportedDataType: Unsupported data type
|
|
|
|
Returns:
|
|
object: The jsonable object.
|
|
"""
|
|
if input is None:
|
|
return None
|
|
if isinstance(input, dict):
|
|
out_input = {}
|
|
for key, value in input.items():
|
|
out_input[key] = resolve_input(value, is_encode_binary,
|
|
custom_type_resolver)
|
|
return out_input
|
|
elif isinstance(input, (list, tuple, set)):
|
|
out_input = []
|
|
for item in input:
|
|
out_input.append(
|
|
resolve_input(item, is_encode_binary, custom_type_resolver))
|
|
return out_input
|
|
elif isinstance(input, str):
|
|
return input
|
|
elif isinstance(input, (int, float, complex)):
|
|
return input
|
|
elif isinstance(input, bool):
|
|
return input
|
|
elif isinstance(input, (bytearray, bytes, memoryview)):
|
|
if is_encode_binary:
|
|
return base64.b64encode(input).decode('ascii')
|
|
else:
|
|
return input
|
|
elif isinstance(input, io.IOBase):
|
|
# if file is binary, encoded based on is_encode_binary
|
|
# websocket is_encode_binary is false, http True.
|
|
# text file if is_encode_binary will return line of file.
|
|
# otherwise return origin content.
|
|
content = input.read()
|
|
if not content:
|
|
raise StopIteration
|
|
if isinstance(content, bytes):
|
|
is_binary_file = _is_binary_file(content)
|
|
if is_binary_file:
|
|
if is_encode_binary:
|
|
return base64.b64encode(content).decode('ascii')
|
|
else:
|
|
return content
|
|
else: # split line by line.
|
|
return content.decode('utf-8').splitlines()
|
|
else:
|
|
if is_encode_binary:
|
|
return content.splitlines()
|
|
else:
|
|
return content
|
|
|
|
elif isinstance(input, Generator):
|
|
output = next(input)
|
|
return resolve_input(output, is_encode_binary, custom_type_resolver)
|
|
elif type(input) in custom_type_resolver:
|
|
return custom_type_resolver[type(input)](input)
|
|
else:
|
|
raise UnsupportedDataType('Unsupported atom data type: %s' %
|
|
type(input))
|