voice get

Html/app.py

@@ -8,13 +8,12 @@ from apps.extensions import socketio
 app = create_app()
 
 if __name__ == "__main__":
-    # 仅本地开发用；生产不要用 werkzeug 自带的开发服务器
     socketio.run(
         app,
-        host="0.0.0.0",
+        host="127.0.0.1",
         port=5551,
         debug=True,                # 开发期打开
-        allow_unsafe_werkzeug=True, # 避免 dev server 的安全限制提示
+        # allow_unsafe_werkzeug=True, # 避免 dev server 的安全限制提示
         # ssl_context=('server.crt', 'server.key')
     )
 

Html/apps/__init__.py

@@ -53,6 +53,7 @@ def create_app(config_object: type = Config) -> Flask:
 
     # 2) 注册蓝图（把子文件暴露的蓝图统一挂到 app 上）
     register_blueprints(app)
+    app.config['TEMPLATES_AUTO_RELOAD'] = True 
 
     # 3) 其他钩子/命令/错误处理
     @app.route("/healthz")

Html/apps/extension_ase/ase_client/client.py

@@ -191,13 +191,15 @@ class ASEngineClient:
             print(f"Failed to send text to route '{route}': {e}")
             return False
         
-    def send_voice(self, route: str, voice_data: bytes, id:str) -> bool:
+    def send_voice(self, route: str, voice_data: bytes, id:str=None) -> bool:
         """
         发送语音数据
         :param route: 目标路由
         :param voice_data: 语音数据字节流
         :return: 是否发送成功
         """
+        if id is None:
+            id = self.id
         if not self.connected:
             print("Not connected to server")
             return False

Html/apps/sockets/namespaces.py

@@ -90,12 +90,6 @@ class AgentNamespace(Namespace):
                                             entry=self,               
                                             init_data={'room': user_uuid}          
                                         )
-        user_uuid2ase_client[user_uuid].register_route_with_entry(
-                                            route='dialog-hint',
-                                            callback=self.receive_ase_dialog_hint,
-                                            entry=self,               
-                                            init_data={'room': user_uuid}    
-                                        )
         user_uuid2ase_client[user_uuid].register_route_with_entry(
                                             route='voice_out',
                                             callback=self.receive_ase_voice_out,
@@ -154,13 +148,12 @@ class AgentNamespace(Namespace):
             chatmanager[uuid].chat_historys.append({'role': 'user', 'content': data['data']})
             user_uuid2ase_client[uuid].send_text(d['route'], data['data'])
 
-    def on_VoiceMessage(self, voice_data):
+    def on_voice(self, voice_data):
-        print(f"Message from client: {voice_data}")
         ex = current_app.extensions
         chatmanager = ex["user_uuid2chatmanager"]
         uuid = session.get('user_uuid')
         user_uuid2ase_client = ex["user_uuid2ase_client"]
-        res = user_uuid2ase_client[uuid].send_voice('voice_in', voice_data)
+        res = user_uuid2ase_client[uuid].send_voice('voice_in', voice_data['data'])
         print(f"Send voicce to route 'voice_in' success: {res}")
         
     def on_connect_to_ase(client_entry: HSAEngineClient, namespace_entry, init_data):
@@ -182,7 +175,7 @@ class AgentNamespace(Namespace):
     #接受语音信息
     def receive_ase_voice_out(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data):
         print("receive_ase_voice_out", data)
-        namespace_entry.emit('VoiceMessage', data, room=init_data['room'], namespace='/agent')
+        namespace_entry.emit('voiceMessage', data, room=init_data['room'], namespace='/agent')
 
     def receive_ase_judge(client_entry: HSAEngineClient, namespace_entry: Namespace, data, init_data):
         print("receive_ase_judge", data)

Html/apps/static/js/audio-processor.js

@@ -0,0 +1,11 @@
+class AudioProcessor extends AudioWorkletProcessor {
+  process(inputs, outputs) {
+    const input = inputs[0];
+    if (input.length > 0) {
+      this.port.postMessage(input[0]);
+    }
+    return true;
+  }
+}
+
+registerProcessor('audio-processor', AudioProcessor); 

Html/apps/static/js/chatbox.js

@@ -4,7 +4,7 @@ let system_message_idx = 0
 document.addEventListener('DOMContentLoaded', function () {
     data = window.appData;
     console.log(data);
-    socket = io('wss://hsamooc.com/agent', {
+    socket = io('wss://hsamooc.cn/agent', {
         query: {
             user_uuid: data.user_uuid,
             folder: data.folder
@@ -18,6 +18,11 @@ document.addEventListener('DOMContentLoaded', function () {
         OpenIframe();        
     });
     // 监听来自服务器的消息
+    socket.on('voiceMessage', function (data) {
+        console.log(data)
+        const input =  document.createElement('messageInput');
+        input.data = data['data'];
+    });
     socket.on('message', function (data) {
         // 显示服务器的回复消息
         const serverMessage = document.createElement('div');
@@ -212,58 +217,76 @@ function handleMouseClick(event) {
     stopRecording();
 }
 //发送语音
-function sendVoiceMessage() {
+const startAudioRecording = async () => {
-    p = pyaudio.PyAudio()
-    //变换录音中图标
-    document.getElementById('recordIcon').className = 'bi bi-record-circle';
-    //监听
-    document.addEventListener('click', handleMouseClick);
-    //打开麦克风流
-    voice_stream = p.open(format = FORMAT,
-        channels = CHANNELS,
-        rate = SAMPLE_RATE,
-        input = True,
-        frames_per_buffer = CHUNK_SIZE)
-    audio_chunks = []
-
-    recordingInterval = setInterval(() => {
-        if (!isRecording) {
-            clearInterval(recordingInterval);
-            return;
-        }
-
   try {
-            // 从麦克风读取音频数据
+    const stream = await navigator.mediaDevices.getUserMedia({ 
-            let audio_data = voice_stream.read(CHUNK_SIZE);
+      audio: {
-
+        sampleRate: 16000,
-            if (!audio_data || audio_data.length === 0) {
+        channelCount: 1,
-                stopRecording();
+        echoCancellation: true,
-                return;
+        noiseSuppression: true
       }
+    });
+    const mediaRecorder = new MediaRecorder(stream);
+    const audioContext = new AudioContext({
+      sampleRate: 16000
+    });
     
-            // 保留原始 PCM 二进制数据
+    // Load the AudioWorklet
-            audio_chunks.push(audio_data);
+    await audioContext.audioWorklet.addModule('/static/js/audio-processor.js');
     
-            // 当积累到一定数量的块时，发送
+    const source = audioContext.createMediaStreamSource(stream);
-            if (audio_chunks.length >= 16) {
+    const workletNode = new AudioWorkletNode(audioContext, 'audio-processor');
-                const merged_data = new Uint8Array(
+    let audioChunks = [];
-                    audio_chunks.reduce((acc, chunk) => acc + chunk.length, 0)
+    
-                );
+    workletNode.port.onmessage = (e) => {
+      audioChunks.push(new Float32Array(e.data));
+      
+      if (audioChunks.length >= 16) {
+        // 合并所有音频块
+        const totalLength = audioChunks.reduce((acc, chunk) => acc + chunk.length, 0);
+        const mergedData = new Float32Array(totalLength);
         let offset = 0;
-                for (const chunk of audio_chunks) {
+        for (const chunk of audioChunks) {
-                    merged_data.set(new Uint8Array(chunk), offset);
+          mergedData.set(chunk, offset);
           offset += chunk.length;
         }
-                // 发送到服务器
+        
-                socket.emit('VoiceMessage', merged_data, { binary: true });
+        const audioMessage = {
-                // 清空音频块，准备接收下一批数据
+          id: window.data.user_uuid,
-                audio_chunks = [];
+          type: 'audio',
+          data: mergedData,
+          sampleRate: 16000,
+          timestamp: Date.now()
+        };
+        socket.emit('voice', audioMessage);
+        console.log("voice sended")
+        audioChunks = [];
       }
+    };
+    
+    source.connect(workletNode);
+    workletNode.connect(audioContext.destination);
+    
+    mediaRecorder.value = {
+      stop: () => {
+        source.disconnect();
+        workletNode.disconnect();
+        stream.getTracks().forEach(track => track.stop());
+        audioContext.close();
+      }
+    }
+    
+    isRecording.value = true;
   } catch (error) {
-            console.error('Error reading audio data:', error);
+    console.error('Error starting audio recording:', error);
-            stopRecording();
   }
-    }, 10); // 每 10 毫秒检查一次录音状态
+};
+function sendVoiceMessage() {
+    startAudioRecording()
+    //变换录音中图标
+    document.getElementById('recordIcon').className = 'bi bi-record-circle';
+    document.addEventListener('click', handleMouseClick);
 }
 
 //停止录音

Html/apps/templates/code-like-desktop.html

@@ -16,6 +16,7 @@
     <link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.3.7/css/bootstrap.min.css" rel="stylesheet">
     <script src="https://unpkg.com/split.js/dist/split.min.js"></script>
     <script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.0/font/bootstrap-icons.css">
 </head>
 <body>
     <div class="maxcontainer" id="maxcontainer" style="height: 100%;">
@@ -61,7 +62,12 @@
             </div>
             <div class="input-area">
                 <textarea id="messageInput" rows="2" class="form-control" placeholder="输入你的消息 (支持 Markdown 语法)"></textarea>
-                <button class="btn btn-primary" onclick="sendMessage()">发送</button>
+                <button class="btn btn-primary" onclick="sendMessage()">
+                    <i class="bi bi-send"></i>
+                </button>
+                <button class="btn btn-secondary" onclick="sendVoiceMessage()">
+                    <i class="bi bi-mic" id = "recordIcon"></i>
+                </button>
             </div>
             
             <script type="text/javascript">

Html/apps/templates/desktop.html

@@ -112,7 +112,6 @@
         <div class="icon" id="toggleButton_icon">❮</div>
     </div>
 
-    <script src="/static/js/chatbox.js"></script>
 
     <script>
         // 实现左右边栏可拖动

README.md

@@ -32,7 +32,7 @@ api_key = sk-aMpnWklN2IbsK44d1kNpy6YOP9bk1pdPJjFeEmbb0a5ytEFf
 model=gpt-4.1-nano
 
 [VSCODE_WEB]
-url = https://hsamooc.com
+url = https://hsamooc.cn
 #http://asengine.net:8282
 [VSCODE_WEB_PATH]
 is_wsl = true