VoiceRecoginizer.ets 14.6 KB

import {NativeNui,Constants,MapToJson,INativeNuiCallback, AsrResult, KwsResult} from 'neonui'
import { PermissionUtils,DeviceUtil, Logger, DateTimeUtils } from 'wdKit';
import AudioCapturer from './AudioCapture';
import { common, Permissions } from '@kit.AbilityKit';
import { NetLayerVoiceRecoginizerToken } from 'wdBean';
import { HttpUrlUtils,ResponseDTO } from 'wdNetwork'
import { HttpRequest } from 'wdNetwork/src/main/ets/http/HttpRequest'


class NativeNuiCallbackHandle implements  INativeNuiCallback {
 asrspeechrealtimeResultOld:string=""
 asrmessage:string;
 message:string;
 nuiCallback?:(result: string) => void
 constructor() {
  this.asrmessage=""
  this.message=""
 }
 onNuiEventCallback(event:Constants.NuiEvent, resultCode:number, arg2:number, kwsResult:KwsResult,
  asrResult:AsrResult):void {
  let asrinfo:string = ""
  // console.log("onUsrNuiEventCallback last this.asrmessage is " + this.asrmessage)
  // console.log("onUsrNuiEventCallback this is " + JSON.stringify(this))
  // console.log("onUsrNuiEventCallback event is " + event)

  if (event === Constants.NuiEvent.EVENT_ASR_RESULT){
   this.message = "EVENT_ASR_RESULT"
   this.asrspeechrealtimeResultOld=""
  } else if (event === Constants.NuiEvent.EVENT_ASR_PARTIAL_RESULT || event === Constants.NuiEvent.EVENT_SENTENCE_END){
   if (event === Constants.NuiEvent.EVENT_ASR_PARTIAL_RESULT ) {
    this.message = "EVENT_ASR_PARTIAL_RESULT"
   } else if(event === Constants.NuiEvent.EVENT_SENTENCE_END){
    this.message = "EVENT_SENTENCE_END"
   }
  } else if (event === Constants.NuiEvent.EVENT_ASR_ERROR){
   this.message = "EVENT_ASR_ERROR"
  } else if (event === Constants.NuiEvent.EVENT_MIC_ERROR){
   this.message = "EVENT_MIC_ERROR"
  } else if (event === Constants.NuiEvent.EVENT_DIALOG_EX){
   this.message = "EVENT_DIALOG_EX"
  }
  if (asrResult) {
   asrinfo = asrResult.asrResult
   //console.log(`asrinfo string is ${asrinfo}`)
   if (asrinfo) {
    try {
     let asrresult_json:object|null = JSON.parse(asrResult.asrResult)
     //console.log(JSON.stringify(asrresult_json))
     if (asrresult_json) {
      let payload:object|null = asrresult_json["payload"];
      if (payload) {
       //console.log(JSON.stringify(payload))
       this.asrmessage = this.asrspeechrealtimeResultOld + payload["result"];

       if(event === Constants.NuiEvent.EVENT_SENTENCE_END){
        // console.info("onUsrNuiEventCallback EVENT_SENTENCE_END")
        this.asrspeechrealtimeResultOld = this.asrmessage
       }
       if (this.nuiCallback) {
        this.nuiCallback(this.asrmessage)
       }
      }
     }
    } catch (e){
     // console.error("got asrinfo not json, so donot fresh asrinfo." + JSON.stringify(e))
    }
   }
  }
  // console.info(`womx onUsrNuiEventCallback(${event}, ${resultCode},${arg2}, kwsResult:${kwsResult},asrResult:"${this.asrmessage}") done`)
 }

 onNuiNeedAudioData(buffer:ArrayBuffer):number {
  // console.info(`womx onUsrNuiNeedAudioData(buffer length = ${buffer.byteLength})`)
  // console.log(`womx onUsrNuiNeedAudioData uid[${process.uid}] pid[${process.pid}] tid[${process.tid}]`);
  let num:number = 0;

  if (true){
   if (true){
    //由于录音回调是异步的,所以此处获取共享数据需要加锁/并保持同步 是较难的,需要修改asr回调为用户主动推送方式
    num = AudioCapturer.getVoiceArrayBuffer1(buffer)
   } else {

    let flag_returned:boolean = false
    try {
     AudioCapturer.getVoiceArrayBuffer(buffer).then( (result:number)=>{
      console.log("womx onUsrNuiNeedAudioData getVoiceArrayBuffer done, result="+result)
      num = result
      flag_returned=true
     })
    } catch(e ){
     flag_returned=true

    }
    while(flag_returned==false){
     console.log("womx onUsrNuiNeedAudioData waiting ... num="+num)

    }
   }
  }


  // console.info(`womx onUsrNuiNeedAudioData(buffer length = ${buffer.byteLength}) return ${num}}`)
  return num;//buffer.byteLength;
 }

 onNuiAudioStateChanged(state:Constants.AudioState):void {
  // console.info(`womx onUsrNuiAudioStateChanged(${state})`)
  if (state === Constants.AudioState.STATE_OPEN){
   // console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder start`)
   //AudioCapturer.init(g_asrinstance)
   AudioCapturer.start()
   // console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder start done`)
  } else if (state === Constants.AudioState.STATE_CLOSE){
   // console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder close`)
   AudioCapturer.stop()
   //AudioCapturer.release()
   // console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder close done`)
  } else if (state === Constants.AudioState.STATE_PAUSE){
   // console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder pause`)
   AudioCapturer.stop()
   //AudioCapturer.release()
   // console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder pause done`)
  }
 }
 onNuiAudioRMSChanged(val:number):number{
  // console.info(`womx onNuiAudioRMSChanged(${val})`)
  return 0;
 }
 clearAsrInfo(){
  this.asrmessage = ""
 }
}

const TAG = 'VoiceRecoginizer'

export class VoiceRecoginizer {
 private static MICROPHONEMISSION: Permissions = 'ohos.permission.MICROPHONE'
 private static READMEDIA: Permissions = 'ohos.permission.READ_MEDIA'
 private static INTENT: Permissions = "ohos.permission.INTERNET"
 private static WRITE_MEDIA: Permissions = "ohos.permission.WRITE_MEDIA"

 private static appKey = "EospGmM6mdPljjjm"
 private static url = "wss://nls-gateway.cn-shanghai.aliyuncs.com:443/ws/v1"
 private authToken?: NetLayerVoiceRecoginizerToken

 private vadMode: boolean = false

 private cbhandle: NativeNuiCallbackHandle = new NativeNuiCallbackHandle()

 private g_asrinstance: NativeNui = new NativeNui(Constants.ModeType.MODE_DIALOG, "asr1")

 private static manager: VoiceRecoginizer

 public static sharedManager(): VoiceRecoginizer {
  if (!VoiceRecoginizer.manager) {
   VoiceRecoginizer.manager = new VoiceRecoginizer()
  }
  return VoiceRecoginizer.manager
 }

 voiceRecoginizerResult?:(result: string) => void
 // public voiceRecoginzerResult:(result: string) => void
 genParams():string {
  let params:string = "";
  let nls_config:Map<string, string|number|boolean|object> = new Map();
  nls_config.set("enable_intermediate_result", true);

  //参数可根据实际业务进行配置
  //接口说明可见: https://help.aliyun.com/document_detail/173298.html
  //查看 2.开始识别

  //由于对外的SDK不带有本地VAD模块(仅带有唤醒功能的SDK具有VAD模块),
  //若要使用VAD模式,则需要设置nls_config参数启动在线VAD模式(见genParams())
   if (this.vadMode) {
    nls_config.set("enable_voice_detection", true);
    nls_config.set("max_start_silence", 10000);
    nls_config.set("max_end_silence", 800);
   } else {
    nls_config.set("enable_voice_detection", false);
   }

  nls_config.set("enable_punctuation_prediction", true);
  nls_config.set("enable_inverse_text_normalization", true);
  // nls_config.set("customization_id", "test_id");
  // nls_config.set("vocabulary_id", "test_id");
  // nls_config.set("sample_rate", 16000);
  // nls_config.set("sr_format", "opus");

  let parameters:Map<string, string|number|boolean|object> = new Map();
  parameters.set("nls_config", Object( JSON.parse(MapToJson(nls_config)) ) );

   //一句话识别
   // console.log("start asr for 一句话识别")
   parameters.set("service_type", Constants.kServiceTypeASR); // 必填
  //如果有HttpDns则可进行设置
  //parameters.put("direct_ip", Utils.getDirectIp());

  params = MapToJson(parameters);//parameters.toString();
  console.log("configinfo genParams" + params)
  return params;
 }
 genInitParams(workpath:string, debugpath:string):string {
  let str:string = "";
  //获取token方式:

  let object:Map<string, string|number|boolean|object> = new Map();

  //账号和项目创建
  //  ak_id ak_secret app_key如何获得,请查看https://help.aliyun.com/document_detail/72138.html
  object.set("app_key",VoiceRecoginizer.appKey); // 必填

  //方法1:
  //  首先ak_id ak_secret app_key如何获得,请查看https://help.aliyun.com/document_detail/72138.html
  //  然后请看 https://help.aliyun.com/document_detail/466615.html 使用其中方案一获取临时凭证
  //  此方案简介: 远端服务器生成具有有效时限的临时凭证, 下发给移动端进行使用, 保证账号信息ak_id和ak_secret不被泄露
  //  获得Token方法(运行在APP服务端): https://help.aliyun.com/document_detail/450255.html?spm=a2c4g.72153.0.0.79176297EyBj4k

  if (this.authToken) {
   object.set("token", this.authToken.accessToken)
  }
  //方法2:
  //  STS获取临时凭证方法暂不支持

  //方法3:(强烈不推荐,存在阿里云账号泄露风险)
  //  参考Auth类的实现在端上访问阿里云Token服务获取SDK进行获取。请勿将ak/sk存在本地或端侧环境。
  //  此方法优点: 端侧获得Token, 无需搭建APP服务器。
  //  此方法缺点: 端侧获得ak/sk账号信息, 极易泄露。
  //            JSONObject object = Auth.getAliYunTicket();

  object.set("device_id", DeviceUtil.clientId()/*Utils.getDeviceId()*/); // 必填, 推荐填入具有唯一性的id, 方便定位问题
  object.set("url", VoiceRecoginizer.url); // 默认
  object.set("workspace", workpath); // 必填, 且需要有读写权限
  // object.set("sample_rate", "16000");
  // object.set("format", "pcm");

  //当初始化SDK时的save_log参数取值为true时,该参数生效。表示是否保存音频debug,该数据保存在debug目录中,需要确保debug_path有效可写。
  //            object.put("save_wav", "true");
  //debug目录,当初始化SDK时的save_log参数取值为true时,该目录用于保存中间音频文件。
  object.set("debug_path", debugpath);

  // FullMix = 0   // 选用此模式开启本地功能并需要进行鉴权注册
  // FullCloud = 1
  // FullLocal = 2 // 选用此模式开启本地功能并需要进行鉴权注册
  // AsrMix = 3    // 选用此模式开启本地功能并需要进行鉴权注册
  // AsrCloud = 4
  // AsrLocal = 5  // 选用此模式开启本地功能并需要进行鉴权注册
   //一句话识别
   // console.log("init asr for 一句话识别")
   object.set("service_mode", Constants.ModeAsrCloud); // 必填

  str = MapToJson(object)

  // console.info("configinfo genInitParams:" + str);
  return str;
 }

 async AsrInit(path:string, filePath:string): Promise<number> {
  //console.log("AsrInit this is " + JSON.stringify(this))
  // console.info("AsrInit path: " + path);
  //获取工作路径, 这里获得当前nuisdk.aar中assets路径

  let asset_path:string = path+"/resources_cloud"
  // fs.stat(asset_path).catch((error: BusinessError) => {
  //  if (error.code = 13900002) {
  //   fs.mkdirSync(asset_path)
  //  }
  // })
  // fs.mkdirSync(asset_path)
  // try {
  //  await this.checkPath(asset_path)
  this.cbhandle.nuiCallback = (result: string) => {
   if (this.voiceRecoginizerResult) {
    this.voiceRecoginizerResult(result)
   }
  }

  let parameters: string = ""
  // this.genInitParams(asset_path,filePath)
  //  .then(parameters) {
  //
  // }
   await this.getToken()

   let ret:number = this.g_asrinstance.initialize(this.cbhandle, this.genInitParams(asset_path,filePath), Constants.LogLevel.LOG_LEVEL_NONE, false);
   // console.info("result = " + ret);
   if (ret == Constants.NuiResultCode.SUCCESS) {

   } else {
    // Logger.debug(TAG,"语音识别初始化失败")
    //final String msg_text = Utils.getMsgWithErrorCode(ret, "init");
    //抛出错误异常信息。
    /*
    runOnUiThread(new Runnable() {
        @Override
        public void run() {
            Toast.makeText(SpeechRecognizerActivity.this, msg_text, Toast.LENGTH_LONG).show();
        }
    });
    */
   }
  //  return ret
  // }
  // catch {
  //  return -1;
  // }
  //初始化SDK,注意用户需要在Auth.getAliYunTicket中填入相关ID信息才可以使用。
    return ret
 }

 // checkPath(asset_path:string):Promise<void> {
 //  return new Promise<void> ((success,fail) => {
 //   fs.stat(asset_path).catch((error: BusinessError) => {
 //    if (error.code = 13900002) {
 //     fs.mkdirSync(asset_path)
 //     success()
 //     return
 //    }
 //    success()
 //   })
 //  })
 // }
 startDialog():number {
  // console.log(`womx startDialog uid[${process.uid}] pid[${process.pid}] tid[${process.tid}]`);
  this.cbhandle.clearAsrInfo()
  //由于对外的SDK不带有本地VAD模块(仅带有唤醒功能的SDK具有VAD模块),
  //若要使用VAD模式,则需要设置nls_config参数启动在线VAD模式(见genParams())
  let vad_mode:Constants.VadMode = Constants.VadMode.TYPE_P2T;
  if (this.vadMode) {
   //TYPE_VAD: SDK自动判断句尾结束识别。(此功能仅存在于<029>带唤醒功能的SDK)
   //vad_mode = Constants.VadMode.TYPE_VAD;
   // console.info("使用Voice Active Detection模式");
  } else {
   //TYPE_P2T: 有用户主动stop()以告知识别完成
   //vad_mode = Constants.VadMode.TYPE_P2T;
   // console.info("使用Push To Talk模式");
  }

  //设置相关识别参数,具体参考API文档
  //  initialize()之后startDialog之前调用
  this.g_asrinstance.setParams(this.genParams());
  AudioCapturer.init(this.g_asrinstance)
  let ret:number = this.g_asrinstance.startDialog(vad_mode, "");
  // console.info("start done . ret = ", ret);
  if (ret != 0) {
   // console.info("call startDialog return error. ", ret);
  }
  return ret
 }

 stop() {
  this.cbhandle.clearAsrInfo()
   AudioCapturer.stop()
 }

  static async checkPemmission (context: common.UIAbilityContext)  {

    let grant = await PermissionUtils.checkPermissions(VoiceRecoginizer.WRITE_MEDIA)

    if (grant) {
     ///初始化SDK
     //  Logger.debug("已申请过权限")
     return
    }
    // let context = getContext()
   let requestGrant = await PermissionUtils.reqPermissionsFromUser([VoiceRecoginizer.MICROPHONEMISSION
   ,VoiceRecoginizer.READMEDIA
   ,VoiceRecoginizer.WRITE_MEDIA
   ,VoiceRecoginizer.INTENT],context)
   if (requestGrant) {
    ///初始化SDK
   }
   // return false
  }


  private async getToken():Promise<NetLayerVoiceRecoginizerToken | undefined> {
   return new Promise<NetLayerVoiceRecoginizerToken|undefined>((success,fail) => {
    if (this.authToken && this.authToken.accessToken.length > 0) {
     if (Date.now() < this.authToken.expireTime - 60 * 1000) {
      success(this.authToken)
      return
     }
    }
    
    HttpRequest.get<ResponseDTO<NetLayerVoiceRecoginizerToken>>(
      HttpUrlUtils.getVoiceRecognizerSDKAuthToken()
    ).then((res: ResponseDTO<NetLayerVoiceRecoginizerToken>) => {
     if (res.code != 0) {
      fail(res.message)
      return
     }
     this.authToken = res.data
     success(res.data)
    })
   })

  }


}