[人工智能]Freeswitch智能语音开发之NLP

NLP自然语言处理是一种专业分析人类语言的人工智能。工作原理是这样的：接收自然语言，这种语言是通过人类的自然使用演变而来的，我们每天都用它来交流转译自然语言，通常是通过基于概率的算法分析自然语言并输出结果。

Freeswitch如何与NLP对接

1、外呼对接nlp?

1.1? Freeswitch通过originate呼叫被叫号码转入到xml或者lua流程中；

1.2、Freeswitch通过流程调用mod_curl的命令curl来发送http请求到nlp，告知nlp启动nlp流程；

1.3? NLP接收到请求返回json数据告知fs进行下一步动作，比如放音，接收按键，是否进行识别或者转接坐席，是否支持asr过程中用户语音打断等；语音打断方式有两种，一种是等用户说完再打断当前播放的语音，另外一种是检测到用户说话就打断。视场景需要选择合适的打断方式；

1.4? Freeswitch接收到NLP的应答，解析json返回数据，按照指示进行相应的动作；

1.5 Freeswitch接收到用户说话语音，根据返回参数的打断方式，可以选择进行实时打断语音，即用户开始说话，TTS播放暂停；也可以等待用户说完再打断播放语音，并把识别结果返回给nlp，且告知是否打断；

1.6 NLP接收到Freeswitch的反馈，可以告知Freeswtich下一步的动作，如果返回null表示nlp不进行此次处理，这时FS继续播放的当前未播放完的语音，并进行新的识别；如果需要终止播放当前的语音和动作，NLP返回新的动作和相关参数给Freeswitch；如此反复进行1.4到1.6之间的动作；

1.7 如果此时用户挂机， Freeswitch发送挂机事件到NLP告知流程结束；

1.8 当NLP告知 Freeswitch转人工时，FS根据NLP返回的技能组把电话转给技能组中的坐席，并把对话记录通过参数的方式发送给坐席弹屏工单页面；

2、呼入

呼入和呼出只是开始阶段不一样，后面流程是一致的。

2.1、 Freeswitch接收到用户呼入，应答用户来电，流程转到呼出流程的1.2上；

3、例子代码

---
--- Generated by EmmyLua(https://github.com/EmmyLua)
--- Created by Administrator.
--- DateTime: 2021/7/6 15:29
---
--打印日志
session:execute("sleep", "3000");
session:answer();
api = freeswitch.API();
local cjson2 = require "cjson";
--local cjson2 = cjson.new()
local robotId = "4899";
local deviceId = "";
local transferAddress = "192.168.1.54:9066"
local nlpUrl = "http://xx.xx.xx.xx:8080/anonymous/wordManage/answers"
local isInterrupt = "1"
local caller = session:getVariable("caller");
local callee = session:getVariable("callee");
local uuid = session:getVariable("uuid");
session:consoleLog("info", string.format("LUA INFO>>>>>>>>lua rocks uuid %s ,caller %s ,callee %s \n", uuid, caller, callee));
session:setVariable("tts_engine", "fastcall_tts:service_id=1,speed=0;volume=50;pitch=0");
session:setVariable("tts_voice", "*");
session:setVariable("language", "zh");
session:setVariable("playback_delimiter", "!");
session:setVariable("playback_sleep_val", "0");
session:setVariable("cache_speech_handles", "1");
session:setVariable("fs_detect_match_hot_word", 0);

local toNlpRequestResult = {}
local requestParams = {}
local requestBreakParams = {}
local userNoInputCount = 0;
local serviceUnavailable = 0;
local detect_speech_result;
local trunkQues;
local playNvAndDetectSpeech
local dealDetectSpeechResult
local requestHungUp
local actionRecognition
local getRequestParams
local toNlpRequest
local dealNlpResponseData
local dealBreakNlpResponseData
local voiceBreak = 1
local ASR_NLP_RESPONSE
function split(str, split_char)
    local sub_str_tab = {}
    while true do
        local pos = string.find(str, split_char)
        if not pos then
            table.insert(sub_str_tab, str)
            break
        end
        local sub_str = string.sub(str, 1, pos - 1)
        table.insert(sub_str_tab, sub_str)
        str = string.sub(str, pos + 1, string.len(str))
    end
    return sub_str_tab;
end
---trunkQues
---播放tts语音
function trunkQues()
    if (session:ready() == false) then
        return ;
    end
    userNoInputCount = userNoInputCount + 1
    if (userNoInputCount > 2) then
        requestHungUp()
        return
    end
    local date = api:execute("strftime", "%Y%m%d");
    local datetime = api:execute("strftime", "%Y%m%d%H%M%S");
    local filename = "/home/record/" .. date .. "/subsection_" .. datetime .. "_" .. caller .. "_" .. callee .. "_" .. uuid .. ".wav"
    session:setVariable("filename", filename);
    requestBreakParams["question"] = "@result"
    requestBreakParams["uuid"] = "@uuid"
    requestBreakParams["break"] = "@break"
    requestBreakParams["filename"] = "@filename"
    requestBreakParams["time"] = "@time"
    requestBreakParams["deviceId"] = deviceId
    requestBreakParams['robotId'] = robotId
    if (type(toNlpRequestResult['action']) ~= "nil" and toNlpRequestResult['action'] == "action_noInterrupt") then
        voiceBreak = 0
        isInterrupt = 0
    else
        voiceBreak = 1
        isInterrupt = 1
    end
    session:setVariable("ASR_NLP_RESPONSE", "")
    session:consoleLog("info", string.format("%s \n", toNlpRequestResult['answers']))
    session:consoleLog("info", string.format("%s detect:fastcall_asr:service_id=1 {voice-break=%s,speech-timeout=700,break-thresh=1500,start-thresh=480,end-thresh=350,break-span-ms=3000}%s {%s}%s %s \n"
    , toNlpRequestResult['answers'], voiceBreak, filename, cjson2.encode(requestBreakParams), nlpUrl, isInterrupt))

    session:execute("fastcall_play_and_detect_speech", string.format("%s detect:fastcall_asr:service_id=1 {voice-break=%s,speech-timeout=700,break-thresh=1500,start-thresh=480,end-thresh=350,break-span-ms=3000}%s {%s}%s %s"
    , toNlpRequestResult['answers'], voiceBreak, filename, cjson2.encode(requestBreakParams), nlpUrl, isInterrupt));

    ASR_NLP_RESPONSE = session:getVariable("ASR_NLP_RESPONSE")
    detect_speech_result = session:getVariable("detect_speech_result");

    session:consoleLog("info", string.format("LUA INFO>>>>>>>>fastcall_play_and_detect_speech>>>>>>>>>>%s，ASR_NLP_RESPONSE %s \n", detect_speech_result, ASR_NLP_RESPONSE));

    if (type(ASR_NLP_RESPONSE) == "nil") then
        local asr_voice_end = session:getVariable("asr_voice_end")
        if (type(asr_voice_end) ~= "nil" and asr_voice_end == "1"
                and type(detect_speech_result) ~= "nil" and detect_speech_result ~= "voiceover" and detect_speech_result ~= "noresult" and detect_speech_result ~= "nomatch" and detect_speech_result ~= "noinput") then

            dealDetectSpeechResult()
            return
        else
            playNvAndDetectSpeech()
            return
        end
    else
        --local responseData = cjson2.decode(ASR_NLP_RESPONSE);
        dealBreakNlpResponseData(ASR_NLP_RESPONSE)
        return
    end
end

---playNvAndDetectSpeech
---播放空的语音文件并接受用户说话
function playNvAndDetectSpeech()
    local date = api:execute("strftime", "%Y%m%d");
    local datetime = api:execute("strftime", "%Y%m%d%H%M%S");
    local filename = "/home/record/" .. date .. "/subsection_" .. datetime .. "_" .. caller .. "_" .. callee .. "_" .. uuid .. ".wav"
    session:setVariable("filename", filename);
    session:execute("play_and_detect_speech", string.format("%s detect:fastcall_asr:service_id=1 {no-input-timeout=3000,voice-break=1,speech-timeout=700}%s", "nv.wav", filename));
    detect_speech_result = session:getVariable("detect_speech_result");
    session:consoleLog("info", string.format("LUA INFO>>>>>>>>playNvAndDetectSpeech>>>>>>>>>>%s \n", detect_speech_result));
    if (type(detect_speech_result) == "nil" or detect_speech_result == "voiceover" or detect_speech_result == "noresult" or detect_speech_result == "nomatch" or detect_speech_result == "noinput") then
        trunkQues()
        return
    else
        dealDetectSpeechResult()
        return
    end
end

---dealDetectSpeechResult
---处理识别结果
function dealDetectSpeechResult()
    if (session:ready() == false) then
        return ;
    end
    session:consoleLog("info", string.format("LUA INFO>>>>>>>>dealDetectSpeechResult>>>>>>>>>>%s \n", detect_speech_result));
    getRequestParams(detect_speech_result)
    toNlpRequest()
end

---actionRecognition
---动作识别
function actionRecognition()
    if (session:ready() == false) then
        return
    end
    session:consoleLog("info", string.format("LUA INFO>>>>>>>>toNlpRequest====response====%s \n", cjson2.encode(toNlpRequestResult)));

    if (type(toNlpRequestResult['action']) == "nil" or toNlpRequestResult['action'] == "" or toNlpRequestResult['action'] == "0" or toNlpRequestResult['action'] == "null") then
        trunkQues()
        return
    end

    local actionTable = split(toNlpRequestResult['action'], "|");
    local actionValue = ""
    local transferNumber = ""
    for k, v in pairs(actionTable) do
        if (k == 1) then
            actionValue = v;
        end
        if (k == 2) then
            transferNumber = v
        end
    end

    if (actionValue == "action_dial_up") then
        --拨号
    elseif (actionValue == "action_agent") then
        --转坐席
        session:execute("playback", toNlpRequestResult['answers']);
        session:setVariable("hangup_after_bridge", "true")
        session:setVariable("continue_on_fail", "true")
        session:execute("fastcall_turnslic", string.format("%s -1 40 1 0 0 0 0 %s ${uuid}", transferNumber,"waiting.wav"))
        local fs_cancel_reason = session:getVariable("fs_cancel_reason")
        session:consoleLog("info", string.format("fastcall_turnslic   %s ", fs_cancel_reason))
        if (type(fs_cancel_reason) ~= "nil" and (fs_cancel_reason == "TIMEOUT" or fs_cancel_reason == "NO_AGENT" or fs_cancel_reason == "NO_AGENT_TIMEOUT" or fs_cancel_reason == "BREAK_OUT")) then
            getRequestParams("转接失败")
            detect_speech_result = "转接失败"
            toNlpRequest()
            return
        end
    elseif (actionValue == "action_skillgroup") then
        --转技能组
        session:execute("playback", toNlpRequestResult['answers']);
        session:setVariable("hangup_after_bridge", "true")
        session:setVariable("continue_on_fail", "true")
        session:execute("fastcall_turnslic", string.format("-1 %s 40 1 0 0 0 0 %s ${uuid}", transferNumber,"waiting.wav"))
        local fs_cancel_reason = session:getVariable("fs_cancel_reason")
        session:consoleLog("info", string.format("fastcall_turnslic   %s ", fs_cancel_reason))
        if (type(fs_cancel_reason) ~= "nil" and (fs_cancel_reason == "TIMEOUT" or fs_cancel_reason == "NO_AGENT" or fs_cancel_reason == "NO_AGENT_TIMEOUT" or fs_cancel_reason == "BREAK_OUT")) then
            getRequestParams("转接失败")
            detect_speech_result = "转接失败"
            toNlpRequest()
            return
        end
    elseif (actionValue == "action_switching") then
        --转接
        session:execute("playback", toNlpRequestResult['answers']);
        local effectiveCallerIdNumber = session:getVariable("sip_to_user")
        session:consoleLog("info", string.format("LUA INFO>>>>>>>> action_switching========effective_caller_id_number  %s \n", effectiveCallerIdNumber));
        session:setVariable("effective_caller_id_number", effectiveCallerIdNumber);
        session:execute("bridge", "sofia/internal/" .. transferNumber .. "@" .. transferAddress);
    elseif (actionValue == "action_holding") then
        --保持

    elseif (actionValue == "action_on_hook") then
        --挂机
        session:execute("playback", toNlpRequestResult['answers']);
    end
end

---requestHungUp
---请求挂机
function requestHungUp()
    if (session:ready() == false) then
        return ;
    end
    getRequestParams("挂机")
    detect_speech_result = "挂机"
    toNlpRequest()
    session:execute("playback", toNlpRequestResult['answers'])
    session:hangup();
    return
end

---getRequestParams
---@param question 问题
---设置请求参数
function getRequestParams(question)
    if (session:ready() == false) then
        return ;
    end
    requestParams['question'] = question;
    requestParams['robotId'] = robotId;
    --requestParams['deviceId'] = deviceId
end

---toNlpRequest
---请求NLP
function toNlpRequest()
    if (session:ready() == false) then
        return ;
    end
    userNoInputCount = 0
    session:consoleLog("info", string.format("LUA INFO>>>>>>>>start to request  post======== %s \n", cjson2.encode(requestParams)));
    session:execute("curl", string.format("http://xx.xx.xx.xx:8080/anonymous/wordManage/answers  content-type 'application/json' post %s", cjson2.encode(requestParams)));
    local curl_response_data = session:getVariable("curl_response_data");
    local curl_response_code = session:getVariable("curl_response_code");
    session:consoleLog("info", string.format("LUA INFO>>>>>>>>toNlpRequest====data====%s \n", curl_response_data));
    session:consoleLog("info", string.format("LUA INFO>>>>>>>>toNlpRequest====code====%s \n", curl_response_code));
    if (curl_response_code == "200") then
        local responseData = cjson2.decode(curl_response_data);
        if (responseData.message == "success") then
            serviceUnavailable = 0
            dealNlpResponseData(responseData)
            return
        end
    end

    serviceUnavailable = serviceUnavailable + 1
    if (serviceUnavailable > 2) then
        session:execute("playback", "phrase:tts-timeleft:暂时无法提供服务,请稍后重试！");
        session:consoleLog("info", string.format("请求服务两次不可用！\n"));
        session:hangup();
        return
    end
    session:consoleLog("info", string.format("LUA INFO>>>>>>>>toNlpTwoRequest========%s \n", detect_speech_result));
    getRequestParams(detect_speech_result)
    toNlpRequest()
end

---dealNlpResponseData
---@param responseData 响应json
---处理nlp响应结果
function dealNlpResponseData(responseData)
    session:consoleLog("info", string.format("LUA INFO>>>>>>>>dealNlpResponseData========%s \n", type(responseData)));
    toNlpRequestResult["answers"] = "phrase:tts-timeleft:" .. responseData.data.answers;
    if (type(responseData.data.action) == "nil") then
        toNlpRequestResult["action"] = "null"
    else
        toNlpRequestResult["action"] = responseData.data.action;
    end
    deviceId = responseData.data.deviceId

    actionRecognition()
    return
end

---dealBreakNlpResponseData
---@param responseData responseData
---
function dealBreakNlpResponseData(responseData)
    --local nlpData = api:execute("fastcall_jason_attr", responseData.."##data");
    local nlpAnswers = api:execute("fastcall_jason_attr", responseData .. "##data.answers");
    local nlpAction = api:execute("fastcall_jason_attr", responseData .. "##data.action");
    local nlpDeviceId = api:execute("fastcall_jason_attr", responseData .. "##data.deviceId");
    session:consoleLog("info", string.format("LUA INFO>>>>>>>>dealBreakNlpResponseData======== %s %s %s \n", nlpAnswers, nlpAction, nlpDeviceId));
    toNlpRequestResult["answers"] = "phrase:tts-timeleft:" .. nlpAnswers;
    if (type(nlpAction) == "nil") then
        toNlpRequestResult["action"] = "null"
    else
        toNlpRequestResult["action"] = nlpAction;
    end
    deviceId = nlpDeviceId
    actionRecognition()
    return
end

if (session:ready() == true) then
    getRequestParams("您好")
    detect_speech_result = "您好"
    --requestParams['deviceId'] = ""
    toNlpRequest()
    trunkQues();
end

以上内容希望对您有所帮助，更多探讨请加Freeswitch qq群640880657。