voice get
This commit is contained in:
Caikecheng
@@ -8,13 +8,12 @@ from apps.extensions import socketio
|
||||
app = create_app()
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 仅本地开发用；生产不要用 werkzeug 自带的开发服务器
|
||||
socketio.run(
|
||||
app,
|
||||
host="0.0.0.0",
|
||||
host="127.0.0.1",
|
||||
port=5551,
|
||||
debug=True, # 开发期打开
|
||||
allow_unsafe_werkzeug=True, # 避免 dev server 的安全限制提示
|
||||
# allow_unsafe_werkzeug=True, # 避免 dev server 的安全限制提示
|
||||
# ssl_context=('server.crt', 'server.key')
|
||||
)
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ def create_app(config_object: type = Config) -> Flask:
|
||||
|
||||
# 2) 注册蓝图（把子文件暴露的蓝图统一挂到 app 上）
|
||||
register_blueprints(app)
|
||||
app.config['TEMPLATES_AUTO_RELOAD'] = True
|
||||
|
||||
# 3) 其他钩子/命令/错误处理
|
||||
@app.route("/healthz")
|
||||
|
||||
@@ -191,13 +191,15 @@ class ASEngineClient:
|
||||
print(f"Failed to send text to route '{route}': {e}")
|
||||
return False
|
||||
|
||||
def send_voice(self, route: str, voice_data: bytes, id:str) -> bool:
|
||||
def send_voice(self, route: str, voice_data: bytes, id:str=None) -> bool:
|
||||
"""
|
||||
发送语音数据
|
||||
:param route: 目标路由
|
||||
:param voice_data: 语音数据字节流
|
||||
:return: 是否发送成功
|
||||
"""
|
||||
if id is None:
|
||||
id = self.id
|
||||
if not self.connected:
|
||||
print("Not connected to server")
|
||||
return False
|
||||
|
||||
@@ -90,12 +90,6 @@ class AgentNamespace(Namespace):
|
||||
entry=self,
|
||||
init_data={'room': user_uuid}
|
||||
)
|
||||
user_uuid2ase_client[user_uuid].register_route_with_entry(
|
||||
route='dialog-hint',
|
||||
callback=self.receive_ase_dialog_hint,
|
||||
entry=self,
|
||||
init_data={'room': user_uuid}
|
||||
)
|
||||
user_uuid2ase_client[user_uuid].register_route_with_entry(
|
||||
route='voice_out',
|
||||
callback=self.receive_ase_voice_out,
|
||||
@@ -154,13 +148,12 @@ class AgentNamespace(Namespace):
|
||||
chatmanager[uuid].chat_historys.append({'role': 'user', 'content': data['data']})
|
||||
user_uuid2ase_client[uuid].send_text(d['route'], data['data'])
|
||||
|
||||
def on_VoiceMessage(self, voice_data):
|
||||
print(f"Message from client: {voice_data}")
|
||||
def on_voice(self, voice_data):
|
||||
ex = current_app.extensions
|
||||
chatmanager = ex["user_uuid2chatmanager"]
|
||||
uuid = session.get('user_uuid')
|
||||
user_uuid2ase_client = ex["user_uuid2ase_client"]
|
||||
res = user_uuid2ase_client[uuid].send_voice('voice_in', voice_data)
|
||||
res = user_uuid2ase_client[uuid].send_voice('voice_in', voice_data['data'])
|
||||
print(f"Send voicce to route 'voice_in' success: {res}")
|
||||
|
||||
def on_connect_to_ase(client_entry: HSAEngineClient, namespace_entry, init_data):
|
||||
@@ -182,7 +175,7 @@ class AgentNamespace(Namespace):
|
||||
#接受语音信息
|
||||
def receive_ase_voice_out(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data):
|
||||
print("receive_ase_voice_out", data)
|
||||
namespace_entry.emit('VoiceMessage', data, room=init_data['room'], namespace='/agent')
|
||||
namespace_entry.emit('voiceMessage', data, room=init_data['room'], namespace='/agent')
|
||||
|
||||
def receive_ase_judge(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data):
|
||||
print("receive_ase_judge", data)
|
||||
|
||||
11
Html/apps/static/js/audio-processor.js
Normal file
11
Html/apps/static/js/audio-processor.js
Normal file
@@ -0,0 +1,11 @@
|
||||
class AudioProcessor extends AudioWorkletProcessor {
|
||||
process(inputs, outputs) {
|
||||
const input = inputs[0];
|
||||
if (input.length > 0) {
|
||||
this.port.postMessage(input[0]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('audio-processor', AudioProcessor);
|
||||
@@ -4,7 +4,7 @@ let system_message_idx = 0
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
data = window.appData;
|
||||
console.log(data);
|
||||
socket = io('wss://hsamooc.com/agent', {
|
||||
socket = io('wss://hsamooc.cn/agent', {
|
||||
query: {
|
||||
user_uuid: data.user_uuid,
|
||||
folder: data.folder
|
||||
@@ -18,6 +18,11 @@ document.addEventListener('DOMContentLoaded', function () {
|
||||
OpenIframe();
|
||||
});
|
||||
// 监听来自服务器的消息
|
||||
socket.on('voiceMessage', function (data) {
|
||||
console.log(data)
|
||||
const input = document.createElement('messageInput');
|
||||
input.data = data['data'];
|
||||
});
|
||||
socket.on('message', function (data) {
|
||||
// 显示服务器的回复消息
|
||||
const serverMessage = document.createElement('div');
|
||||
@@ -212,58 +217,76 @@ function handleMouseClick(event) {
|
||||
stopRecording();
|
||||
}
|
||||
//发送语音
|
||||
const startAudioRecording = async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
sampleRate: 16000,
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true
|
||||
}
|
||||
});
|
||||
const mediaRecorder = new MediaRecorder(stream);
|
||||
const audioContext = new AudioContext({
|
||||
sampleRate: 16000
|
||||
});
|
||||
|
||||
// Load the AudioWorklet
|
||||
await audioContext.audioWorklet.addModule('/static/js/audio-processor.js');
|
||||
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
const workletNode = new AudioWorkletNode(audioContext, 'audio-processor');
|
||||
let audioChunks = [];
|
||||
|
||||
workletNode.port.onmessage = (e) => {
|
||||
audioChunks.push(new Float32Array(e.data));
|
||||
|
||||
if (audioChunks.length >= 16) {
|
||||
// 合并所有音频块
|
||||
const totalLength = audioChunks.reduce((acc, chunk) => acc + chunk.length, 0);
|
||||
const mergedData = new Float32Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of audioChunks) {
|
||||
mergedData.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
const audioMessage = {
|
||||
id: window.data.user_uuid,
|
||||
type: 'audio',
|
||||
data: mergedData,
|
||||
sampleRate: 16000,
|
||||
timestamp: Date.now()
|
||||
};
|
||||
socket.emit('voice', audioMessage);
|
||||
console.log("voice sended")
|
||||
audioChunks = [];
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(workletNode);
|
||||
workletNode.connect(audioContext.destination);
|
||||
|
||||
mediaRecorder.value = {
|
||||
stop: () => {
|
||||
source.disconnect();
|
||||
workletNode.disconnect();
|
||||
stream.getTracks().forEach(track => track.stop());
|
||||
audioContext.close();
|
||||
}
|
||||
}
|
||||
|
||||
isRecording.value = true;
|
||||
} catch (error) {
|
||||
console.error('Error starting audio recording:', error);
|
||||
}
|
||||
};
|
||||
function sendVoiceMessage() {
|
||||
p = pyaudio.PyAudio()
|
||||
startAudioRecording()
|
||||
//变换录音中图标
|
||||
document.getElementById('recordIcon').className = 'bi bi-record-circle';
|
||||
//监听
|
||||
document.addEventListener('click', handleMouseClick);
|
||||
//打开麦克风流
|
||||
voice_stream = p.open(format = FORMAT,
|
||||
channels = CHANNELS,
|
||||
rate = SAMPLE_RATE,
|
||||
input = True,
|
||||
frames_per_buffer = CHUNK_SIZE)
|
||||
audio_chunks = []
|
||||
|
||||
recordingInterval = setInterval(() => {
|
||||
if (!isRecording) {
|
||||
clearInterval(recordingInterval);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// 从麦克风读取音频数据
|
||||
let audio_data = voice_stream.read(CHUNK_SIZE);
|
||||
|
||||
if (!audio_data || audio_data.length === 0) {
|
||||
stopRecording();
|
||||
return;
|
||||
}
|
||||
|
||||
// 保留原始 PCM 二进制数据
|
||||
audio_chunks.push(audio_data);
|
||||
|
||||
// 当积累到一定数量的块时，发送
|
||||
if (audio_chunks.length >= 16) {
|
||||
const merged_data = new Uint8Array(
|
||||
audio_chunks.reduce((acc, chunk) => acc + chunk.length, 0)
|
||||
);
|
||||
let offset = 0;
|
||||
for (const chunk of audio_chunks) {
|
||||
merged_data.set(new Uint8Array(chunk), offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
// 发送到服务器
|
||||
socket.emit('VoiceMessage', merged_data, { binary: true });
|
||||
// 清空音频块，准备接收下一批数据
|
||||
audio_chunks = [];
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error reading audio data:', error);
|
||||
stopRecording();
|
||||
}
|
||||
}, 10); // 每 10 毫秒检查一次录音状态
|
||||
}
|
||||
|
||||
//停止录音
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
<link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.7/css/bootstrap.min.css" rel="stylesheet">
|
||||
<script src="https://unpkg.com/split.js/dist/split.min.js"></script>
|
||||
<script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.0/font/bootstrap-icons.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="maxcontainer" id="maxcontainer" style="height: 100%;">
|
||||
@@ -61,7 +62,12 @@
|
||||
</div>
|
||||
<div class="input-area">
|
||||
<textarea id="messageInput" rows="2" class="form-control" placeholder="输入你的消息 (支持 Markdown 语法)"></textarea>
|
||||
<button class="btn btn-primary" onclick="sendMessage()">发送</button>
|
||||
<button class="btn btn-primary" onclick="sendMessage()">
|
||||
<i class="bi bi-send"></i>
|
||||
</button>
|
||||
<button class="btn btn-secondary" onclick="sendVoiceMessage()">
|
||||
<i class="bi bi-mic" id = "recordIcon"></i>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<script type="text/javascript">
|
||||
|
||||
@@ -112,7 +112,6 @@
|
||||
<div class="icon" id="toggleButton_icon">❮</div>
|
||||
</div>
|
||||
|
||||
<script src="/static/js/chatbox.js"></script>
|
||||
|
||||
<script>
|
||||
// 实现左右边栏可拖动
|
||||
|
||||
Reference in New Issue
Block a user