diff --git a/Html/apps/extension_ase/ase_client/client.py b/Html/apps/extension_ase/ase_client/client.py index 10c5543..aaa2a8b 100644 --- a/Html/apps/extension_ase/ase_client/client.py +++ b/Html/apps/extension_ase/ase_client/client.py @@ -187,7 +187,6 @@ class ASEngineClient: try: # 编码语音数据为base64 # voice_data = 'data:image/jpeg;base64,' +base64.b64encode(voice_data).decode('utf-8') - # 发送数据 self.sio.emit( route, @@ -270,4 +269,3 @@ class HSAEngineClient(ASEngineClient): self.send_text('markdown-in', fmd, self.id) self.send_text('markdown-prompt-in', fmdp, self.id) self.send_text('score-prompt-in', fsp, self.id) - diff --git a/Html/apps/sockets/namespaces.py b/Html/apps/sockets/namespaces.py index 53d6963..281fbf3 100644 --- a/Html/apps/sockets/namespaces.py +++ b/Html/apps/sockets/namespaces.py @@ -63,6 +63,12 @@ class AgentNamespace(Namespace): entry=self, init_data={'room': user_uuid} ) + user_uuid2ase_client[user_uuid].register_route_with_entry( + route='voice_out', + callback=self.receive_ase_voice_out, + entry=self, + init_data={'room': user_uuid} + ) user_uuid2ase_client[user_uuid].register_route_with_entry( route='judge', callback=self.receive_ase_judge, @@ -119,11 +125,25 @@ class AgentNamespace(Namespace): emit(d['route'], d, room=uuid, namespace='/agent') user_uuid2ase_client[uuid].send_text(d['route'], data['data']) + def on_VoiceMessage(self, voice_data): + print(f"Message from client: {voice_data}") + ex = current_app.extensions + chatmanager = ex["user_uuid2chatmanager"] + uuid = session.get('user_uuid') + user_uuid2ase_client = ex["user_uuid2ase_client"] + res = user_uuid2ase_client[uuid].send_voice('voice_in', voice_data) + print(f"Send voicce to route 'voice_in' success: {res}") + def receive_ase_dialog(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data): print("receive_ase_dialog", data) namespace_entry.emit('message', data['data'], room=init_data['room'], namespace='/agent') + #接受语音信息 + def receive_ase_voice_out(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data): + print("receive_ase_voice_out", data) + namespace_entry.emit('VoiceMessage', data, room=init_data['room'], namespace='/agent') + def receive_ase_judge(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data): print("receive_ase_judge", data) namespace_entry.emit('request_function', data, room=init_data['room'], namespace='/agent') # data 是一个列表 /dict也支持 diff --git a/Html/apps/static/js/chatbox.js b/Html/apps/static/js/chatbox.js index 2563921..1fc90ac 100644 --- a/Html/apps/static/js/chatbox.js +++ b/Html/apps/static/js/chatbox.js @@ -1,30 +1,52 @@ var socket; -let system_message_idx=0 -document.addEventListener('DOMContentLoaded', function() { +let system_message_idx = 0 +document.addEventListener('DOMContentLoaded', function () { data = window.appData; console.log(data); - socket = io('wss://hsamooc.com/agent',{ - query:{ - user_uuid:data.user_uuid, - folder:data.folder + socket = io('wss://hsamooc.com/agent', { + query: { + user_uuid: data.user_uuid, + folder: data.folder } }); - socket.on('connect', function() { + socket.on('connect', function () { console.log('Connected to server'); - socket.emit('login',JSON.stringify(data)); + socket.emit('login', JSON.stringify(data)); OpenIframe(); }); // 监听来自服务器的消息 - socket.on('message', function(data) { + socket.on('message', function (data) { // 显示服务器的回复消息 const serverMessage = document.createElement('div'); - serverMessage.innerHTML = `
华实君
${marked.parse(data)}
`; + serverMessage.innerHTML = `
华实君:
${marked.parse(data)}
`; document.getElementById('chatbox').appendChild(serverMessage); // 滚动到最新消息 document.getElementById('chatbox').scrollTop = document.getElementById('chatbox').scrollHeight; }); - socket.on('request_function', function(data){ + + //监听来自服务器的消息 + /*socket.on('VoiceMessage', function(data) { + console.log("Received voice answer message:", data); + // 显示服务器的回复消息 + const VoiceMessage = document.createElement('div'); + VoiceMessage.className = 'chat-message server-message'; + VoiceMessage.innerHTML = ` +
华实君:
+ ${marked.parse(data)} + `; + //插入与滚动 + document.getElementById('chatbox').appendChild(VoiceMessage); + document.getElementById('chatbox').scrollTop = document.getElementById('chatbox').scrollHeight; + });*/ + //监听来自服务器的语音识别消息 + socket.on('VoiceMessage', function (data) { + console.log("Received voice message I sent from server:", data); + //将语音转文字显示在输入框中 + document.getElementById('messageInput').value = data; + }); + + socket.on('request_function', function (data) { try { console.log("request_function", data); if (typeof data === 'string') { @@ -43,18 +65,18 @@ document.addEventListener('DOMContentLoaded', function() { approveButton.innerHTML = '√'; // 使用勾选符号 approveButton.data = element; approveButton.className = 'approve-button'; - approveButton.onclick = function(event) { - console.log(element); - socket.emit('message', JSON.stringify({ type: 'function', data: element })); - event.currentTarget.disabled = true; - event.currentTarget.classList.add('disabled'); + approveButton.onclick = function (event) { + console.log(element); + socket.emit('message', JSON.stringify({ type: 'function', data: element })); + event.currentTarget.disabled = true; + event.currentTarget.classList.add('disabled'); }; requestMessage.appendChild(description); requestMessage.appendChild(approveButton); - + document.getElementById('chatbox').appendChild(requestMessage); document.getElementById('chatbox').scrollTop = document.getElementById('chatbox').scrollHeight; - } + } // for (let i = 0; i < data.length; i++) { // const element = data[i] // const requestMessage = document.createElement('div'); @@ -82,11 +104,11 @@ document.addEventListener('DOMContentLoaded', function() { console.log(error) } }); - socket.on('next_chapter', function(data){ + socket.on('next_chapter', function (data) { next_chapter(data); }); - socket.on('chapter_score',function(data){ - if (typeof data === 'string'){ + socket.on('chapter_score', function (data) { + if (typeof data === 'string') { data = JSON.parse(data); } console.log(data) @@ -96,7 +118,7 @@ document.addEventListener('DOMContentLoaded', function() { document.getElementById('chatbox').appendChild(scoreMessage); document.getElementById('chatbox').scrollTop = document.getElementById('chatbox').scrollHeight; }) - socket.on('system_message',function(data){ + socket.on('system_message', function (data) { console.log(data) let systemMessage = document.createElement('div'); systemMessage.className = 'chat-message server-message'; @@ -104,14 +126,14 @@ document.addEventListener('DOMContentLoaded', function() { closeButton.className = 'close-button'; // 可以根据需要添加样式类 closeButton.innerHTML = '×'; // 关闭按钮的文本内容 closeButton.style.marginRight = '10px'; // 可选：设置关闭按钮的右边距 - + systemMessage.appendChild(closeButton); systemMessage.innerHTML += `系统提示: ${data}`; - systemMessage.id = 'system_message_'+system_message_idx; - system_message_idx+=1 + systemMessage.id = 'system_message_' + system_message_idx; + system_message_idx += 1 document.getElementById('chatbox').appendChild(systemMessage); document.getElementById('chatbox').scrollTop = document.getElementById('chatbox').scrollHeight; - $('.close-button').on('click', function(event) { + $('.close-button').on('click', function (event) { $(this).css({ 'background-color': 'gray', 'color': 'white', // 可选：设置文字颜色为白色，以便更清晰地显示 @@ -119,7 +141,7 @@ document.addEventListener('DOMContentLoaded', function() { }); $(this).parent().remove(); }); - setTimeout(function() { + setTimeout(function () { $(systemMessage.id).remove(); }, 5000); // 5000 毫秒 = 5 秒 }) @@ -130,7 +152,7 @@ document.addEventListener('DOMContentLoaded', function() { const fontSizeSlider = document.getElementById('fontSizeSlider'); const chatbox = document.getElementById('chatbox'); -fontSizeSlider.addEventListener('input', function() { +fontSizeSlider.addEventListener('input', function () { console.log(fontSizeSlider.value); chatbox.style.fontSize = fontSizeSlider.value + 'px'; }); @@ -144,27 +166,27 @@ const chineseLabel = document.getElementById('label_for_zh'); let language = 'zh'; // 默认语言 // 切换语言事件 -englishRadio.addEventListener('change', function() { +englishRadio.addEventListener('change', function () { if (englishRadio.checked) { language = 'en'; englishLabel.classList.add('active'); chineseLabel.classList.remove('active'); } - socket.emit('language',language); + socket.emit('language', language); }); -chineseRadio.addEventListener('change', function() { +chineseRadio.addEventListener('change', function () { if (chineseRadio.checked) { language = 'zh'; chineseLabel.classList.add('active'); englishLabel.classList.remove('active'); } - socket.emit('language',language); + socket.emit('language', language); }); -function sendInitiativeFunctionCall(function_name){ - socket.emit('initiative', {'name': function_name}); +function sendInitiativeFunctionCall(function_name) { + socket.emit('initiative', { 'name': function_name }); } // 发送消息 function sendMessage() { @@ -180,7 +202,7 @@ function sendMessage() { document.getElementById('chatbox').appendChild(userMessage); // 发送消息到服务器 - socket.emit('message', JSON.stringify({data: input, type:'text'})); + socket.emit('message', JSON.stringify({ data: input, type: 'text' })); // 清空输入框 document.getElementById('messageInput').value = ''; @@ -189,6 +211,107 @@ function sendMessage() { document.getElementById('chatbox').scrollTop = document.getElementById('chatbox').scrollHeight; } +let voice_stream = null; +let audio_chunks = []; +let isRecording = false; +let recordingInterval = null; +//鼠标点击终止事件 +function handleMouseClick(event) { + event.stopPropagation(); + stopRecording(); +} +//发送语音 +function sendVoiceMessage() { + p = pyaudio.PyAudio() + //变换录音中图标 + document.getElementById('recordIcon').className = 'bi bi-record-circle'; + //监听 + document.addEventListener('click', handleMouseClick); + //打开麦克风流 + voice_stream = p.open(format = FORMAT, + channels = CHANNELS, + rate = SAMPLE_RATE, + input = True, + frames_per_buffer = CHUNK_SIZE) + audio_chunks = [] + + recordingInterval = setInterval(() => { + if (!isRecording) { + clearInterval(recordingInterval); + return; + } + + try { + // 从麦克风读取音频数据 + let audio_data = voice_stream.read(CHUNK_SIZE); + + if (!audio_data || audio_data.length === 0) { + stopRecording(); + return; + } + + // 保留原始 PCM 二进制数据 + audio_chunks.push(audio_data); + + // 当积累到一定数量的块时，发送 + if (audio_chunks.length >= 16) { + const merged_data = new Uint8Array( + audio_chunks.reduce((acc, chunk) => acc + chunk.length, 0) + ); + let offset = 0; + for (const chunk of audio_chunks) { + merged_data.set(new Uint8Array(chunk), offset); + offset += chunk.length; + } + // 发送到服务器 + socket.emit('VoiceMessage', merged_data, { binary: true }); + // 清空音频块，准备接收下一批数据 + audio_chunks = []; + } + } catch (error) { + console.error('Error reading audio data:', error); + stopRecording(); + } + }, 10); // 每 10 毫秒检查一次录音状态 +} + +//停止录音 +function stopRecording() { + if (!isRecording) return; + isRecording = false; + //取消监听 + document.removeEventListener('click', handleMouseClick); + //取消定时器 + if (recordingInterval) { + clearInterval(recordingInterval); + recordingInterval = null; + } + //变换图标 + document.getElementById('recordIcon').className = 'bi bi-mic'; + //发送剩下的数据 + if (audio_chunks.length > 0) { + const merged_data = new Uint8Array(audio_chunks.reduce((acc, chunk) => acc + chunk.length, 0)); + let offset = 0; + for (const chunk of audio_chunks) { + merged_data.set(new Uint8Array(chunk), offset); + offset += chunk.length; + } + socket.emit('VoiceMessage', JSON.stringify({ data: merged_data, type: 'audio' })); + audio_chunks = []; + } + overlay.style.display = 'none'; + //关闭音频流 + if (voice_stream) { + voice_stream.stop_stream(); + voice_stream.close(); + voice_stream = null; + } + if (p) { + p.terminate(); + p = null; + } +} + let currentChapterIndex = -1; function next_chapter(data) { // 获取所有的 h3 元素 @@ -231,22 +354,22 @@ function next_chapter(data) { function generateProgressBar(N, idx, titles) { const container = document.getElementById('markdown-content-process'); container.innerHTML = ''; // 清空内容 - + // 创建进度条的外框 const progressBox = document.createElement('div'); progressBox.className = 'progress-box'; - + // 获取详细信息容器 const detailBox = document.getElementById('progress-detail'); - + // 根据N值动态调整每个进度节点的宽度 const nodeWidth = 100 / N; // 每个节点的宽度为 100% / N - + // 根据N值生成子框并设置颜色 for (let i = 0; i < N; i++) { const titleBox = document.createElement('div'); titleBox.className = 'progress-title'; - + // 设置进度条的颜色 if (i < idx) { titleBox.classList.add('green'); // 已完成部分 @@ -261,7 +384,7 @@ function generateProgressBar(N, idx, titles) { titleBox.style.width = `${nodeWidth}%`; // 监听鼠标移入事件来显示详情 - titleBox.addEventListener('mouseenter', function() { + titleBox.addEventListener('mouseenter', function () { detailBox.innerHTML = `Step ${i + 1}: ${titles[i] || "No title"}`;// - Additional details here...`; detailBox.style.display = 'block'; // 根据进度条位置显示详情 @@ -271,7 +394,7 @@ function generateProgressBar(N, idx, titles) { }); // 监听鼠标移出事件来隐藏详情 - titleBox.addEventListener('mouseleave', function() { + titleBox.addEventListener('mouseleave', function () { detailBox.style.display = 'none'; }); diff --git a/Html/apps/templates/desktop.html b/Html/apps/templates/desktop.html index 994a9dc..fa495ca 100644 --- a/Html/apps/templates/desktop.html +++ b/Html/apps/templates/desktop.html @@ -1,5 +1,6 @@ + @@ -17,30 +18,32 @@ +