语音交互功能

Html/apps/extension_ase/ase_client/client.py

@@ -187,7 +187,6 @@ class ASEngineClient:
         try:
             # 编码语音数据为base64
             # voice_data = 'data:image/jpeg;base64,' +base64.b64encode(voice_data).decode('utf-8')
-            
             # 发送数据
             self.sio.emit(
                 route,
@@ -270,4 +269,3 @@ class HSAEngineClient(ASEngineClient):
         self.send_text('markdown-in', fmd, self.id)
         self.send_text('markdown-prompt-in', fmdp, self.id)
         self.send_text('score-prompt-in', fsp, self.id)
-

Html/apps/sockets/namespaces.py

@@ -63,6 +63,12 @@ class AgentNamespace(Namespace):
                                             entry=self,               
                                             init_data={'room': user_uuid}          
                                         )
+        user_uuid2ase_client[user_uuid].register_route_with_entry(
+                                            route='voice_out',
+                                            callback=self.receive_ase_voice_out,
+                                            entry=self,               
+                                            init_data={'room': user_uuid}          
+                                        )
         user_uuid2ase_client[user_uuid].register_route_with_entry(
                                             route='judge',
                                             callback=self.receive_ase_judge,
@@ -119,11 +125,25 @@ class AgentNamespace(Namespace):
             emit(d['route'], d, room=uuid, namespace='/agent')
             user_uuid2ase_client[uuid].send_text(d['route'], data['data'])
 
+    def on_VoiceMessage(self, voice_data):
+        print(f"Message from client: {voice_data}")
+        ex = current_app.extensions
+        chatmanager = ex["user_uuid2chatmanager"]
+        uuid = session.get('user_uuid')
+        user_uuid2ase_client = ex["user_uuid2ase_client"]
+        res = user_uuid2ase_client[uuid].send_voice('voice_in', voice_data)
+        print(f"Send voicce to route 'voice_in' success: {res}")
+        
 
     def receive_ase_dialog(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data):
         print("receive_ase_dialog", data)
         namespace_entry.emit('message', data['data'], room=init_data['room'], namespace='/agent')
 
+    #接受语音信息
+    def receive_ase_voice_out(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data):
+        print("receive_ase_voice_out", data)
+        namespace_entry.emit('VoiceMessage', data, room=init_data['room'], namespace='/agent')
+
     def receive_ase_judge(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data):
         print("receive_ase_judge", data)
         namespace_entry.emit('request_function', data, room=init_data['room'], namespace='/agent')  # data 是一个列表 /dict也支持

Html/apps/static/js/chatbox.js

@@ -19,11 +19,33 @@ document.addEventListener('DOMContentLoaded', function() {
     socket.on('message', function (data) {
         // 显示服务器的回复消息
         const serverMessage = document.createElement('div');
-        serverMessage.innerHTML = `<div><b>华实君</b></div><div class="chat-message server-message"> ${marked.parse(data)}</div>`;
+        serverMessage.innerHTML = `<div><b>华实君:</b></div><div class="chat-message server-message"> ${marked.parse(data)}</div>`;
         document.getElementById('chatbox').appendChild(serverMessage);
         // 滚动到最新消息
         document.getElementById('chatbox').scrollTop = document.getElementById('chatbox').scrollHeight;
     });
+
+    //监听来自服务器的消息
+    /*socket.on('VoiceMessage', function(data) {
+        console.log("Received voice answer message:", data);
+        // 显示服务器的回复消息
+        const VoiceMessage = document.createElement('div');
+        VoiceMessage.className = 'chat-message server-message';
+        VoiceMessage.innerHTML = `
+            <div><b>华实君:</b></div>
+            ${marked.parse(data)}
+        `;
+        //插入与滚动
+        document.getElementById('chatbox').appendChild(VoiceMessage);
+        document.getElementById('chatbox').scrollTop = document.getElementById('chatbox').scrollHeight;
+    });*/
+    //监听来自服务器的语音识别消息
+    socket.on('VoiceMessage', function (data) {
+        console.log("Received voice message I sent from server:", data);
+        //将语音转文字显示在输入框中
+        document.getElementById('messageInput').value = data;
+    });
+
     socket.on('request_function', function (data) {
         try {
             console.log("request_function", data);
@@ -189,6 +211,107 @@ function sendMessage() {
     document.getElementById('chatbox').scrollTop = document.getElementById('chatbox').scrollHeight;
 }
 
+let voice_stream = null;
+let audio_chunks = [];
+let isRecording = false;
+let recordingInterval = null;
+//鼠标点击终止事件
+function handleMouseClick(event) {
+    event.stopPropagation();
+    stopRecording();
+}
+//发送语音
+function sendVoiceMessage() {
+    p = pyaudio.PyAudio()
+    //变换录音中图标
+    document.getElementById('recordIcon').className = 'bi bi-record-circle';
+    //监听
+    document.addEventListener('click', handleMouseClick);
+    //打开麦克风流
+    voice_stream = p.open(format = FORMAT,
+        channels = CHANNELS,
+        rate = SAMPLE_RATE,
+        input = True,
+        frames_per_buffer = CHUNK_SIZE)
+    audio_chunks = []
+
+    recordingInterval = setInterval(() => {
+        if (!isRecording) {
+            clearInterval(recordingInterval);
+            return;
+        }
+
+        try {
+            // 从麦克风读取音频数据
+            let audio_data = voice_stream.read(CHUNK_SIZE);
+
+            if (!audio_data || audio_data.length === 0) {
+                stopRecording();
+                return;
+            }
+
+            // 保留原始 PCM 二进制数据
+            audio_chunks.push(audio_data);
+
+            // 当积累到一定数量的块时，发送
+            if (audio_chunks.length >= 16) {
+                const merged_data = new Uint8Array(
+                    audio_chunks.reduce((acc, chunk) => acc + chunk.length, 0)
+                );
+                let offset = 0;
+                for (const chunk of audio_chunks) {
+                    merged_data.set(new Uint8Array(chunk), offset);
+                    offset += chunk.length;
+                }
+                // 发送到服务器
+                socket.emit('VoiceMessage', merged_data, { binary: true });
+                // 清空音频块，准备接收下一批数据
+                audio_chunks = [];
+            }
+        } catch (error) {
+            console.error('Error reading audio data:', error);
+            stopRecording();
+        }
+    }, 10); // 每 10 毫秒检查一次录音状态
+}
+
+//停止录音
+function stopRecording() {
+    if (!isRecording) return;
+    isRecording = false;
+    //取消监听
+    document.removeEventListener('click', handleMouseClick);
+    //取消定时器
+    if (recordingInterval) {
+        clearInterval(recordingInterval);
+        recordingInterval = null;
+    }
+    //变换图标
+    document.getElementById('recordIcon').className = 'bi bi-mic';
+    //发送剩下的数据
+    if (audio_chunks.length > 0) {
+        const merged_data = new Uint8Array(audio_chunks.reduce((acc, chunk) => acc + chunk.length, 0));
+        let offset = 0;
+        for (const chunk of audio_chunks) {
+            merged_data.set(new Uint8Array(chunk), offset);
+            offset += chunk.length;
+        }
+        socket.emit('VoiceMessage', JSON.stringify({ data: merged_data, type: 'audio' }));
+        audio_chunks = [];
+    }
+    overlay.style.display = 'none';
+    //关闭音频流
+    if (voice_stream) {
+        voice_stream.stop_stream();
+        voice_stream.close();
+        voice_stream = null;
+    }
+    if (p) {
+        p.terminate();
+        p = null;
+    }
+}
+
 let currentChapterIndex = -1;
 function next_chapter(data) {
     // 获取所有的 h3 元素

Html/apps/templates/desktop.html

@@ -1,5 +1,6 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
@@ -17,6 +18,7 @@
     <script src="https://unpkg.com/split.js/dist/split.min.js"></script>
     <script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
 </head>
+
 <body>
     <div class="maxcontainer" id="maxcontainer" style="height: 100%;">
         <!-- 左侧内容 -->
@@ -27,7 +29,8 @@
             <div id="markdown-content-process" style="width: 100%;height: 5%;">
 
             </div>
-            <div id="progress-detail" style="position: absolute; top: 0; left: 0; display: none; padding: 10px; background-color: rgba(0, 0, 0, 0.7); color: white; border-radius: 5px;">
+            <div id="progress-detail"
+                style="position: absolute; top: 0; left: 0; display: none; padding: 10px; background-color: rgba(0, 0, 0, 0.7); color: white; border-radius: 5px;">
                 <!-- 这里将显示鼠标悬停的详情 -->
             </div>
         </div>
@@ -60,8 +63,15 @@
                 <!-- 消息会在这里显示 -->
             </div>
             <div class="input-area">
-                <textarea id="messageInput" rows="2" class="form-control" placeholder="输入你的消息 (支持 Markdown 语法)"></textarea>
+                <textarea id="messageInput" rows="2" class="form-control"
-                <button class="btn btn-primary" onclick="sendMessage()">发送</button>
+                    placeholder="输入你的消息 (支持 Markdown 语法)"></textarea>
+                <!--文本与语音的发送-->
+                <button class="btn btn-primary" onclick="sendMessage()">
+                    <i class="bi bi-send"></i>
+                </button>
+                <button class="btn btn-secondary" onclick="sendAudioMessage()">
+                    <i class="bi bi-mic" id = "recordIcon"></i>
+                </button>
             </div>
 
             <script type="text/javascript">
@@ -228,4 +238,5 @@
 
     </script>
 </body>
+
 </html>

study/cake/68bacdfadf5aeae0912f7f18/基础算法/.config

@@ -0,0 +1 @@
+{"user_uuid": "user_4e35d980-622b-49e0-ad29-3a4092274f9d", "course_id": "68bacdfadf5aeae0912f7f18", "chapter_name": "基础算法", "lesson_name": "二分", "path": "E:/agent/huashi/code-agent/study/cake/68bacdfadf5aeae0912f7f18/基础算法"}