|
|
@@ -7,13 +7,14 @@ import wave
|
|
|
import queue
|
|
|
import threading
|
|
|
import traceback
|
|
|
+
|
|
|
import pjsua2 as pj
|
|
|
from enum import Enum
|
|
|
+from src.core.voip.constant import *
|
|
|
|
|
|
calls = {}
|
|
|
-recording_file = '/code/src/core/voip/incoming_call.wav'
|
|
|
-player_file = '/code/src/core/voip/test222.wav'
|
|
|
-player_script_dir = '/code/src/core/voip/scripts/'
|
|
|
+# recording_file = '/code/src/core/voip/incoming_call.wav'
|
|
|
+# player_file = '/code/src/core/voip/test222.wav'
|
|
|
|
|
|
|
|
|
class BotStatus(Enum):
|
|
|
@@ -37,44 +38,80 @@ class UserStatus(Enum):
|
|
|
|
|
|
|
|
|
class MyAudioMediaPort(pj.AudioMediaPort):
|
|
|
- def __init__(self, call, asr=None):
|
|
|
+ def __init__(self, call, aud_med=None, asr=None):
|
|
|
pj.AudioMediaPort.__init__(self)
|
|
|
- # 打开一个 .pcm 文件来保存音频流（可选：保存为 .wav）
|
|
|
- self.wav = wave.open(f"{recording_file}", "wb")
|
|
|
- self.wav.setnchannels(1)
|
|
|
- self.wav.setsampwidth(2) # 假设每个样本是 16 位（2 字节）
|
|
|
- self.wav.setframerate(16000)
|
|
|
+ # # 打开一个 .pcm 文件来保存音频流（可选：保存为 .wav）
|
|
|
+ # self.wav = wave.open(f"{recording_file}", "wb")
|
|
|
+ # self.wav.setnchannels(1)
|
|
|
+ # self.wav.setsampwidth(2) # 假设每个样本是 16 位（2 字节）
|
|
|
+ # self.wav.setframerate(16000)
|
|
|
+
|
|
|
self.call = call
|
|
|
+ self.aud_med = aud_med
|
|
|
self.asr = asr
|
|
|
|
|
|
+ self.user_asr_texts = []
|
|
|
+ self.cur_player_file = None
|
|
|
+
|
|
|
def onFrameRequested(self, frame):
|
|
|
print("Request audio frame:", frame)
|
|
|
|
|
|
def onFrameReceived(self, frame):
|
|
|
- self.wav.writeframes(bytes(frame.buf))
|
|
|
+ # self.wav.writeframes(bytes(frame.buf))
|
|
|
# print("Received audio frame:", frame.buf, frame.size)
|
|
|
if self.asr: # 如果ASR实例存在，则发送音频数据
|
|
|
self.asr.send_audio(frame.buf)
|
|
|
+
|
|
|
+ try:
|
|
|
+ asr_text = self.get_asr_text()
|
|
|
+ play_complete = self.is_play_complete()
|
|
|
+ if asr_text and not play_complete:
|
|
|
+ self.user_asr_texts.append(asr_text)
|
|
|
+ if asr_text and play_complete:
|
|
|
+ self.user_asr_texts.append(asr_text)
|
|
|
+ self.call.chat('###'.join(self.user_asr_texts))
|
|
|
+
|
|
|
+ player_file = self.get_player_file()
|
|
|
+ if player_file and play_complete:
|
|
|
+ self.cur_player_file = player_file
|
|
|
+ self.call.send_bot_speaker(player_file)
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ def is_play_complete(self):
|
|
|
+ if self.cur_player_file:
|
|
|
+ player_id = murmur3_32(self.cur_player_file)
|
|
|
+ return self.call.player_complete_dict.get(player_id)
|
|
|
+
|
|
|
+ def get_asr_text(self):
|
|
|
try:
|
|
|
asr_text = self.call.user_asr_text_queue.get(block=False)
|
|
|
print(asr_text)
|
|
|
- if asr_text:
|
|
|
- self.call.send_bot_speaker(self.call.scripts.get())
|
|
|
+ return asr_text
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ def get_player_file(self):
|
|
|
+ try:
|
|
|
+ player_file = self.call.player_queue.get(block=False)
|
|
|
+ print(player_file)
|
|
|
+ return player_file
|
|
|
except:
|
|
|
pass
|
|
|
|
|
|
|
|
|
class MyAudioMediaPlayer(pj.AudioMediaPlayer):
|
|
|
|
|
|
- def __init__(self, sink, on_complete=None):
|
|
|
+ def __init__(self, player_id, sink, on_complete=None):
|
|
|
pj.AudioMediaPlayer.__init__(self)
|
|
|
+ self.player_id = player_id
|
|
|
self.sink = sink
|
|
|
self.on_complete = on_complete
|
|
|
|
|
|
def onEof2(self):
|
|
|
self.stopTransmit(self.sink)
|
|
|
if self.on_complete:
|
|
|
- self.on_complete()
|
|
|
+ self.on_complete(self.player_id)
|
|
|
|
|
|
|
|
|
# Subclass to extend the Account and get notifications etc.
|
|
|
@@ -109,25 +146,18 @@ class MyCall(pj.Call):
|
|
|
self.user_part = user_part
|
|
|
self.call_id = call_id
|
|
|
self.kwargs = kwargs
|
|
|
- self.audio_port = None
|
|
|
- self.recorder = None
|
|
|
self.aud_med = None
|
|
|
- self.player = None # 用于播放录音的媒体播放器
|
|
|
self.asr = None
|
|
|
+
|
|
|
+ self.scripts = build_demo_script()
|
|
|
self.user_asr_text_queue = queue.Queue(maxsize=100)
|
|
|
- self.scripts = self.build_demo_script()
|
|
|
+ self.player_queue = queue.Queue(maxsize=3)
|
|
|
+ self.player_complete_dict = {}
|
|
|
|
|
|
from src.core.voip.asr import TestSt
|
|
|
self.asr = TestSt(call_id, message_receiver=self.on_receiver_asr_result) # 创建ASR实例
|
|
|
self.asr.start() # 启动ASR线程
|
|
|
|
|
|
- def build_demo_script(self):
|
|
|
- res = queue.Queue(maxsize=10)
|
|
|
- for file in os.listdir(player_script_dir):
|
|
|
- file = os.path.join(player_script_dir, file)
|
|
|
- print('build_demo_script::', file)
|
|
|
- res.put(file)
|
|
|
- return res
|
|
|
|
|
|
def onDtmfDigit(self, prm):
|
|
|
digit = prm.digit
|
|
|
@@ -145,11 +175,10 @@ class MyCall(pj.Call):
|
|
|
# pj.PJSIP_INV_STATE_CONFIRMED
|
|
|
# pj.PJSIP_INV_STATE_DISCONNECTED
|
|
|
|
|
|
- # if call_info.state == pj.PJSIP_INV_STATE_CONFIRMED:
|
|
|
- # # 当呼叫状态为已确认（即接通）
|
|
|
- # audio_media = self.getAudioMedia() # 获取音频媒体对象
|
|
|
- # audio_media.startTransmit(audio_media) # 这里可以对音频流进行处理
|
|
|
- # print ('11111111111')
|
|
|
+ if call_info.state == pj.PJSIP_INV_STATE_CONFIRMED:
|
|
|
+ # 当呼叫状态为已确认（即接通）
|
|
|
+ self.dbot_say_hello()
|
|
|
+
|
|
|
if call_info.state == pj.PJSIP_INV_STATE_DISCONNECTED:
|
|
|
print("通话结束")
|
|
|
# 远程挂机之后要将分机号回收
|
|
|
@@ -166,37 +195,37 @@ class MyCall(pj.Call):
|
|
|
try:
|
|
|
# 建立双向通道
|
|
|
self.receive_user_speaker()
|
|
|
- self.send_bot_speaker(self.scripts.get())
|
|
|
except Exception as e:
|
|
|
traceback.print_exc()
|
|
|
|
|
|
def receive_user_speaker(self):
|
|
|
- self.audio_port = MyAudioMediaPort(self, self.asr)
|
|
|
- self.audio_port.createPort("Incoming Call Port", self.build_audio_format())
|
|
|
- self.aud_med.startTransmit(self.audio_port)
|
|
|
+ audio_port = MyAudioMediaPort(self, self.aud_med, self.asr)
|
|
|
+ audio_port.createPort("Incoming Call Port", build_audio_format())
|
|
|
+ self.aud_med.startTransmit(audio_port)
|
|
|
|
|
|
def send_bot_speaker(self, player_file):
|
|
|
if not player_file or not os.path.exists(player_file):
|
|
|
return
|
|
|
- self.player = MyAudioMediaPlayer(self.aud_med, on_complete=self.on_media_player_complete)
|
|
|
- self.player.createPlayer(player_file)
|
|
|
- self.player.startTransmit(self.aud_med)
|
|
|
+ player_id = murmur3_32(player_file)
|
|
|
+ player = MyAudioMediaPlayer(player_id, self.aud_med, on_complete=self.on_media_player_complete)
|
|
|
+ player.createPlayer(player_file)
|
|
|
+ player.startTransmit(self.aud_med)
|
|
|
|
|
|
def on_receiver_asr_result(self, message, *args):
|
|
|
self.user_asr_text_queue.put(message)
|
|
|
|
|
|
- def on_media_player_complete(self):
|
|
|
+ def on_media_player_complete(self, player_id):
|
|
|
print('player complete')
|
|
|
+ self.player_complete_dict[player_id] = 1
|
|
|
+
|
|
|
+ def bot_say_hello(self):
|
|
|
+ self.chat()
|
|
|
|
|
|
- def build_audio_format(self):
|
|
|
- fmt = pj.MediaFormatAudio()
|
|
|
- fmt.type = pj.PJMEDIA_TYPE_AUDIO
|
|
|
- fmt.id = pj.PJMEDIA_FORMAT_PCM
|
|
|
- fmt.clockRate = 16000 # 采样率
|
|
|
- fmt.channelCount = 1 # 通道数
|
|
|
- fmt.frameTimeUsec = 20000 # 每帧的时间（20 毫秒）
|
|
|
- fmt.bitsPerSample = 16 # 每个采样的位数
|
|
|
- return fmt
|
|
|
+ def chat(self, user_asr_text=None):
|
|
|
+ # TODO 调用文本机器人接口
|
|
|
+ message = {'player_file', self.scripts.get()}
|
|
|
+ player = message.get('player_file')
|
|
|
+ self.player_queue.put(player)
|
|
|
|
|
|
|
|
|
class BotAgent:
|
余尚辉