developOne / harmonyPool

Go to a project
Toggle navigation Toggle navigation pinning
wuyanan
56697add 1 parent 80611c82

ref |> 评论输入面板集成语音识别

Inline Side-by-side
Showing 20 changed files with 1059 additions and 5 deletions
... ... @@ -888,4 +888,11 @@ export class HttpUrlUtils {
let url = HttpUrlUtils.getHost() + HttpUrlUtils.VISITOR_COMMENT_LIST_DATA_PATH
return url
}
///获取语音识别授权token
static getVoiceRecognizerSDKAuthToken() {
let url = HttpUrlUtils.getHost() + "/api/rmrb-search-api/zh/c/display/search/token"
return url
}
}
... ...
... ... @@ -181,4 +181,6 @@ export { TopicDetailData,GroupItem } from './src/main/ets/bean/content/TopicDeta
export { FeedbackTypeBean } from './src/main/ets/bean/detail/FeedbackTypeBean';
export { FeedBackParams } from './src/main/ets/bean/content/FeedBackParams';
export { NetLayerVoiceRecoginizerToken } from './src/main/ets/bean/voicerecoginizer/NetLayerVoiceRecoginizerToken';
... ...
export interface NetLayerVoiceRecoginizerToken {
accessToken: string
///过期时间，到毫秒
expireTime: number
}
\ No newline at end of file
... ...
import { inputMethodEngine } from '@kit.IMEKit'
import { commentInfo } from 'wdBean/Index'
import { VoiceRecoginizer } from 'wdHwAbility'
import { ToastUtils } from 'wdKit/Index'
import { HttpUtils } from 'wdNetwork/Index'
import { commentItemModel } from '../model/CommentModel'
import { publishCommentModel } from '../model/PublishCommentModel'
import commentViewModel from '../viewmodel/CommentViewModel'
import { VoiceInputView } from './VoiceInputView'
import { common, UIAbility } from '@kit.AbilityKit'
export interface CommentDialogInputContent {
comment?: string
... ... @@ -126,6 +128,7 @@ export struct CommentCustomDialog {
.width(30)
.height(30)
.onClick(() => {
VoiceRecoginizer.checkPemmission(getContext(this) as common.UIAbilityContext)
this.positionInfo = this.textInputController.getCaretOffset()
this.voiceSwitch = !this.voiceSwitch
this.emojiSwitch = false
... ... @@ -193,7 +196,7 @@ export struct CommentCustomDialog {
}
if (this.voiceSwitch) {
VoiceInputView({voiceRecoginizerResult:(result: string) => {
this.publishCommentModel.commentContent += result;
this.publishCommentModel.commentContent = result;
}}).height(150)
}
... ...
import Lottie from '@ohos/lottie'
import { LottieView } from '../../lottie/LottieView'
import lottie from '@ohos/lottie';
import { CommonConstants } from 'wdConstant';
import { common} from '@kit.AbilityKit';
import { VoiceRecoginizer } from 'wdHwAbility/src/main/ets/voiceRecognizer/VoiceRecoginizer';
@Component
export struct VoiceInputView {
... ... @@ -10,8 +14,19 @@ export struct VoiceInputView {
@State private tips:string = "长按开始语音输入"
voiceRe: VoiceRecoginizer = VoiceRecoginizer.sharedManager()
voiceRecoginizerResult?:(result: string) => void
aboutToAppear(): void {
this.initVoiceRecoigzer()
}
aboutToDisappear(): void {
this.voiceRe.stop()
this.voiceRe.voiceRecoginizerResult = undefined
}
build() {
Column() {
Stack() {
... ... @@ -39,17 +54,18 @@ export struct VoiceInputView {
.gesture(
LongPressGesture()
.onAction((event: GestureEvent) => {
// this
this.playAnimationOne = true
this.tips = ""
this.voiceRe.startDialog()
})
.onActionEnd((event: GestureEvent) => {
this.tips = "长按开始语音输入"
if (this.voiceRecoginizerResult) {
this.voiceRecoginizerResult!(this.tips)
}
// lottie.destroy("voice_input_one")
this.playAnimationOne = false
this.voiceRe.stop()
// this.voiceRe.voiceRecoginizerResult = undefined
})
)
}
... ... @@ -59,4 +75,18 @@ export struct VoiceInputView {
.fontColor("#222222")
}
}
initVoiceRecoigzer() {
let context = getContext(this) as common.UIAbilityContext
let resoureDir = context.resourceDir
this.voiceRe.AsrInit(context.resourceDir, context.filesDir)
this.voiceRe.voiceRecoginizerResult = (result: string) => {
if (this.voiceRecoginizerResult) {
this.voiceRecoginizerResult(result)
}
}
}
}
\ No newline at end of file
... ...
... ... @@ -4,3 +4,5 @@ export { HWLocationUtils } from './src/main/ets/location/HWLocationUtils'
// export { WDPushNotificationManager } from "./src/main/ets/notification/WDPushNotificationManager"
export { GetuiPush } from "./src/main/ets/getuiPush/GetuiPush"
export {VoiceRecoginizer} from './src/main/ets/voiceRecognizer/VoiceRecoginizer'
\ No newline at end of file
... ...
... ... @@ -8,6 +8,7 @@
"packageType": "InterfaceHar",
"dependencies": {
"library": "file:./src/main/ets/getuiPush/GT-HM-1.0.4.har",
"neonui":"file:./src/main/ets/voiceRecognizer/neonui.har",
"wdConstant": "file:../../commons/wdConstant",
"wdLogin": "file:../../features/wdLogin",
"wdKit": "file:../../commons/wdKit",
... ...
import { audio } from '@kit.AudioKit'
import { ArkTSUtils } from '@kit.ArkTS'
import {getVoicebufferDataLengthByte} from 'neonui'
import process from '@ohos.process'
import { NativeNui } from 'neonui'
let TAG:string = "AudioCapturer"
//音频采集器类,主要用于采集音频,写入文件
export default class AudioCapturer{
static lock_:ArkTSUtils.locks.AsyncLock = new ArkTSUtils.locks.AsyncLock()
//static voiceBuffer2:Int16Array=new Int16Array(0);
static voiceBuffer1:ArrayBuffer[] = [];
//static voiceBuffer:ArrayBuffer[] =[];
static async clearVoiceArrayBuffer(){
await AudioCapturer.lock_.lockAsync( ()=>{
if (AudioCapturer.voiceBuffer1!==undefined){
AudioCapturer.voiceBuffer1= []
}
console.info(`AudioCapturer clearVoiceArrayBuffer, then voiceBuffer1 size ${AudioCapturer.voiceBuffer1.length}`);
})
}
static async setVoiceArrayBuffer(voice:ArrayBuffer){
console.info(`AudioCapturer setVoiceArrayBuffer, 1 voiceBuffer1 size ${AudioCapturer.voiceBuffer1.length}`);
AudioCapturer.lock_.lockAsync( ()=>{
if (AudioCapturer.voiceBuffer1==undefined){
AudioCapturer.voiceBuffer1= []
}
AudioCapturer.voiceBuffer1.push(voice)
console.info(`AudioCapturer setVoiceArrayBuffer, 3 voiceBuffer1 size ${AudioCapturer.voiceBuffer1.length}`);
})
console.info(`AudioCapturer setVoiceArrayBuffer, 2 voiceBuffer1 size ${AudioCapturer.voiceBuffer1.length}`);
}
static async getVoiceArrayBuffer(buffer:ArrayBuffer):Promise<number>{
let ret:number;
let outbuferleng=buffer.byteLength;
let buffer_out_typedarray = new Int8Array(buffer)
let offset = 0
console.log("AudioCapturer enter getVoiceArrayBuffer");
return await AudioCapturer.lock_.lockAsync( ()=>{
console.info(`AudioCapturer womx getVoiceArrayBuffer, outbuferleng ${buffer.byteLength}.with length ${AudioCapturer.voiceBuffer1.length}`);
// while(AudioRenderer.voiceBuffer1.length <= 0){
// if(AudioRenderer.flagVoiceEnd==true) {
// }
// }
if (AudioCapturer.voiceBuffer1.length > 0) {
let voice_length = getVoicebufferDataLengthByte(AudioCapturer.voiceBuffer1)
if (voice_length>=outbuferleng) {
let bytes_need = outbuferleng
while(bytes_need>0){
let voice1st = AudioCapturer.voiceBuffer1.shift()
if (voice1st==undefined ){
} else {
let out_typedbuffer:Int8Array;
console.info(`AudioCapturer womx voice1st.byteLength=${voice1st.byteLength} vs bytes_need=${bytes_need}`);
if (voice1st.byteLength > bytes_need) {
let out_buffer = voice1st.slice(0,bytes_need)
out_typedbuffer = new Int8Array(out_buffer)
let save_buffer = voice1st.slice(bytes_need)
AudioCapturer.voiceBuffer1.unshift(save_buffer)
} else {
out_typedbuffer = new Int8Array(voice1st)
}
for (let i = 0; i < out_typedbuffer.byteLength; i++) {
buffer_out_typedarray[offset + i] = out_typedbuffer[i]
}
bytes_need -= out_typedbuffer.byteLength
offset += out_typedbuffer.byteLength
console.info(`AudioCapturer womx bytes_need=${bytes_need}`);
}
}
} else {
ret = 0
console.error(`AudioCapturer error getVoiceArrayBuffer, outbuferleng 0.with ringbuffer voicebytes length ${voice_length}`);
}
} else {
ret = 0;
console.error(`AudioCapturer error getVoiceArrayBuffer, outbuferleng 0.with ringbuffer.length ${AudioCapturer.voiceBuffer1.length}<=0`);
}
return ret
})
}
static getVoiceArrayBuffer1(buffer:ArrayBuffer):number{
let ret:number=0;
let outbuferleng=buffer.byteLength;
let buffer_out_typedarray = new Int8Array(buffer)
let offset = 0
console.log("AudioCapturer enter getVoiceArrayBuffer");
console.info(`AudioCapturer womx getVoiceArrayBuffer, outbuferleng ${buffer.byteLength}.with length ${AudioCapturer.voiceBuffer1.length}`);
// while(AudioRenderer.voiceBuffer1.length <= 0){
// if(AudioRenderer.flagVoiceEnd==true) {
// }
// }
if (AudioCapturer.voiceBuffer1.length > 0) {
let voice_length = getVoicebufferDataLengthByte(AudioCapturer.voiceBuffer1)
if (voice_length>=outbuferleng) {
let bytes_need = outbuferleng
while(bytes_need>0){
let voice1st = AudioCapturer.voiceBuffer1.shift()
if (voice1st==undefined ){
break
} else {
let out_typedbuffer:Int8Array;
console.info(`AudioCapturer womx voice1st.byteLength=${voice1st.byteLength} vs bytes_need=${bytes_need}`);
if (voice1st.byteLength > bytes_need) {
let out_buffer = voice1st.slice(0,bytes_need)
out_typedbuffer = new Int8Array(out_buffer)
let save_buffer = voice1st.slice(bytes_need)
AudioCapturer.voiceBuffer1.unshift(save_buffer)
} else {
out_typedbuffer = new Int8Array(voice1st)
}
for (let i = 0; i < out_typedbuffer.byteLength; i++) {
buffer_out_typedarray[offset + i] = out_typedbuffer[i]
}
bytes_need -= out_typedbuffer.byteLength
offset += out_typedbuffer.byteLength
console.info(`AudioCapturer womx bytes_need=${bytes_need}`);
}
}
ret = outbuferleng - bytes_need
} else {
ret = 0
console.error(`AudioCapturer error getVoiceArrayBuffer, outbuferleng 0.with ringbuffer voicebytes length ${voice_length}`);
}
} else {
ret = 0;
console.error(`AudioCapturer error getVoiceArrayBuffer, outbuferleng 0.with ringbuffer.length ${AudioCapturer.voiceBuffer1.length}<=0`);
}
return ret;
}
//定义音频流信息
static audioStreamInfo:audio.AudioStreamInfo = {
//彩样率
samplingRate:audio.AudioSamplingRate.SAMPLE_RATE_16000,
//通道数
channels:audio.AudioChannel.CHANNEL_1,
//采样格式
sampleFormat:audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE,
//编码格式
encodingType:audio.AudioEncodingType.ENCODING_TYPE_RAW
}
//定义音频采集器信息
static audioCapturerInfo:audio.AudioCapturerInfo={
//音源类型
source:audio.SourceType.SOURCE_TYPE_MIC,
//音频采集器标志
capturerFlags:0
}
//定义音频采集器
static audioCapturer:audio.AudioCapturer
static g_asrinstance:NativeNui
//初始化音频采集器
static async init(asrinstance:NativeNui){
AudioCapturer.g_asrinstance = asrinstance
AudioCapturer.audioCapturer = await audio.createAudioCapturer({
//需要音频流信息和音频采集器信息
streamInfo:AudioCapturer.audioStreamInfo,
capturerInfo:AudioCapturer.audioCapturerInfo
})
if (AudioCapturer.audioCapturer !== undefined) {
await AudioCapturer.audioCapturer.on('readData', AudioCapturer.readDataCallback);
}
}
static readDataCallback = (buffer: ArrayBuffer) => {
console.log(`${TAG} read data bytelength is ${buffer.byteLength}. uid[${process.uid}] pid[${process.pid}] tid[${process.tid}]`);
//AudioCapturer.setVoiceArrayBuffer(buffer)
AudioCapturer.g_asrinstance.updateAudio(buffer,false)
}
//开始采集音频
static async start(){
await AudioCapturer.clearVoiceArrayBuffer()
if (AudioCapturer.audioCapturer) {
let stateGroup = [audio.AudioState.STATE_PREPARED,
audio.AudioState.STATE_PAUSED,
audio.AudioState.STATE_STOPPED];
if (stateGroup.indexOf(AudioCapturer.audioCapturer.state.valueOf()) === -1) {
// 当且仅当状态为STATE_PREPARED、STATE_PAUSED和STATE_STOPPED之一时才能启动采集
console.error(`${TAG}: start failed`);
console.error('Capturer is not STATE_PREPARED or STATE_PAUSED or STATE_STOPPED');
return;
}
//开始录音
await AudioCapturer.audioCapturer.start()
console.log(`${TAG} start done`);
} else {
console.log(`${TAG} start with AudioCapturer.audioCapturer is null`);
return
}
}
//停止采集音频
static async stop(){
if (AudioCapturer.audioCapturer) {
// 只有采集器状态为STATE_RUNNING或STATE_PAUSED的时候才可以停止
if (AudioCapturer.audioCapturer.state === audio.AudioState.STATE_RUNNING ||
AudioCapturer.audioCapturer.state === audio.AudioState.STATE_PAUSED
) {
console.error(`Capturer state is ${AudioCapturer.audioCapturer.state}`);
await AudioCapturer.audioCapturer.stop()/*.then( (value)=>{
console.log("result of Capturer stop is " + value);
}); // 停止采集*/
console.error('Capturer stop done');
} else {
console.error('Capturer is not running or paused');
return;
}
} else {
console.log(`${TAG} stop with AudioCapturer.audioCapturer is null`);
return
}
await AudioCapturer.clearVoiceArrayBuffer()
}
// 释放资源
static async release() {
if (AudioCapturer.audioCapturer) {
// 采集器状态不是STATE_RELEASED或STATE_NEW状态，才能release
if (AudioCapturer.audioCapturer.state === audio.AudioState.STATE_RELEASED ||
AudioCapturer.audioCapturer.state === audio.AudioState.STATE_NEW) {
console.info('Capturer already released');
return;
}
await AudioCapturer.audioCapturer.release()
}
}
}
... ...
import {NativeNui,Constants,MapToJson,INativeNuiCallback, AsrResult, KwsResult} from 'neonui'
import { PermissionUtils,DeviceUtil, Logger, DateTimeUtils } from 'wdKit';
import AudioCapturer from './AudioCapture';
import { common, Permissions } from '@kit.AbilityKit';
import { NetLayerVoiceRecoginizerToken } from 'wdBean';
import { HttpUrlUtils,ResponseDTO } from 'wdNetwork'
import { HttpRequest } from 'wdNetwork/src/main/ets/http/HttpRequest'
class NativeNuiCallbackHandle implements INativeNuiCallback {
asrspeechrealtimeResultOld:string=""
asrmessage:string;
message:string;
nuiCallback?:(result: string) => void
constructor() {
this.asrmessage=""
this.message=""
}
onNuiEventCallback(event:Constants.NuiEvent, resultCode:number, arg2:number, kwsResult:KwsResult,
asrResult:AsrResult):void {
let asrinfo:string = ""
// console.log("onUsrNuiEventCallback last this.asrmessage is " + this.asrmessage)
// console.log("onUsrNuiEventCallback this is " + JSON.stringify(this))
// console.log("onUsrNuiEventCallback event is " + event)
if (event === Constants.NuiEvent.EVENT_ASR_RESULT){
this.message = "EVENT_ASR_RESULT"
this.asrspeechrealtimeResultOld=""
} else if (event === Constants.NuiEvent.EVENT_ASR_PARTIAL_RESULT || event === Constants.NuiEvent.EVENT_SENTENCE_END){
if (event === Constants.NuiEvent.EVENT_ASR_PARTIAL_RESULT ) {
this.message = "EVENT_ASR_PARTIAL_RESULT"
} else if(event === Constants.NuiEvent.EVENT_SENTENCE_END){
this.message = "EVENT_SENTENCE_END"
}
} else if (event === Constants.NuiEvent.EVENT_ASR_ERROR){
this.message = "EVENT_ASR_ERROR"
} else if (event === Constants.NuiEvent.EVENT_MIC_ERROR){
this.message = "EVENT_MIC_ERROR"
} else if (event === Constants.NuiEvent.EVENT_DIALOG_EX){
this.message = "EVENT_DIALOG_EX"
}
if (asrResult) {
asrinfo = asrResult.asrResult
//console.log(`asrinfo string is ${asrinfo}`)
if (asrinfo) {
try {
let asrresult_json:object|null = JSON.parse(asrResult.asrResult)
//console.log(JSON.stringify(asrresult_json))
if (asrresult_json) {
let payload:object|null = asrresult_json["payload"];
if (payload) {
//console.log(JSON.stringify(payload))
this.asrmessage = this.asrspeechrealtimeResultOld + payload["result"];
if(event === Constants.NuiEvent.EVENT_SENTENCE_END){
// console.info("onUsrNuiEventCallback EVENT_SENTENCE_END")
this.asrspeechrealtimeResultOld = this.asrmessage
}
if (this.nuiCallback) {
this.nuiCallback(this.asrmessage)
}
}
}
} catch (e){
// console.error("got asrinfo not json, so donot fresh asrinfo." + JSON.stringify(e))
}
}
}
// console.info(`womx onUsrNuiEventCallback(${event}, ${resultCode},${arg2}, kwsResult:${kwsResult},asrResult:"${this.asrmessage}") done`)
}
onNuiNeedAudioData(buffer:ArrayBuffer):number {
// console.info(`womx onUsrNuiNeedAudioData(buffer length = ${buffer.byteLength})`)
// console.log(`womx onUsrNuiNeedAudioData uid[${process.uid}] pid[${process.pid}] tid[${process.tid}]`);
let num:number = 0;
if (true){
if (true){
//由于录音回调是异步的，所以此处获取共享数据需要加锁/并保持同步 是较难的，需要修改asr回调为用户主动推送方式
num = AudioCapturer.getVoiceArrayBuffer1(buffer)
} else {
let flag_returned:boolean = false
try {
AudioCapturer.getVoiceArrayBuffer(buffer).then( (result:number)=>{
console.log("womx onUsrNuiNeedAudioData getVoiceArrayBuffer done, result="+result)
num = result
flag_returned=true
})
} catch(e ){
flag_returned=true
}
while(flag_returned==false){
console.log("womx onUsrNuiNeedAudioData waiting ... num="+num)
}
}
}
// console.info(`womx onUsrNuiNeedAudioData(buffer length = ${buffer.byteLength}) return ${num}}`)
return num;//buffer.byteLength;
}
onNuiAudioStateChanged(state:Constants.AudioState):void {
// console.info(`womx onUsrNuiAudioStateChanged(${state})`)
if (state === Constants.AudioState.STATE_OPEN){
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder start`)
//AudioCapturer.init(g_asrinstance)
AudioCapturer.start()
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder start done`)
} else if (state === Constants.AudioState.STATE_CLOSE){
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder close`)
AudioCapturer.stop()
//AudioCapturer.release()
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder close done`)
} else if (state === Constants.AudioState.STATE_PAUSE){
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder pause`)
AudioCapturer.stop()
//AudioCapturer.release()
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder pause done`)
}
}
onNuiAudioRMSChanged(val:number):number{
// console.info(`womx onNuiAudioRMSChanged(${val})`)
return 0;
}
clearAsrInfo(){
this.asrmessage = ""
}
}
const TAG = 'VoiceRecoginizer'
export class VoiceRecoginizer {
private static MICROPHONEMISSION: Permissions = 'ohos.permission.MICROPHONE'
private static READMEDIA: Permissions = 'ohos.permission.READ_MEDIA'
private static INTENT: Permissions = "ohos.permission.INTERNET"
private static WRITE_MEDIA: Permissions = "ohos.permission.WRITE_MEDIA"
private static appKey = "EospGmM6mdPljjjm"
private static url = "wss://nls-gateway.cn-shanghai.aliyuncs.com:443/ws/v1"
private authToken?: NetLayerVoiceRecoginizerToken
private vadMode: boolean = false
private cbhandle: NativeNuiCallbackHandle = new NativeNuiCallbackHandle()
private g_asrinstance: NativeNui = new NativeNui(Constants.ModeType.MODE_DIALOG, "asr1")
private static manager: VoiceRecoginizer
public static sharedManager(): VoiceRecoginizer {
if (!VoiceRecoginizer.manager) {
VoiceRecoginizer.manager = new VoiceRecoginizer()
}
return VoiceRecoginizer.manager
}
voiceRecoginizerResult?:(result: string) => void
// public voiceRecoginzerResult:(result: string) => void
genParams():string {
let params:string = "";
let nls_config:Map<string, string|number|boolean|object> = new Map();
nls_config.set("enable_intermediate_result", true);
//参数可根据实际业务进行配置
//接口说明可见: https://help.aliyun.com/document_detail/173298.html
//查看 2.开始识别
//由于对外的SDK不带有本地VAD模块(仅带有唤醒功能的SDK具有VAD模块)，
//若要使用VAD模式，则需要设置nls_config参数启动在线VAD模式(见genParams())
if (this.vadMode) {
nls_config.set("enable_voice_detection", true);
nls_config.set("max_start_silence", 10000);
nls_config.set("max_end_silence", 800);
} else {
nls_config.set("enable_voice_detection", false);
}
nls_config.set("enable_punctuation_prediction", true);
nls_config.set("enable_inverse_text_normalization", true);
// nls_config.set("customization_id", "test_id");
// nls_config.set("vocabulary_id", "test_id");
// nls_config.set("sample_rate", 16000);
// nls_config.set("sr_format", "opus");
let parameters:Map<string, string|number|boolean|object> = new Map();
parameters.set("nls_config", Object( JSON.parse(MapToJson(nls_config)) ) );
//一句话识别
// console.log("start asr for 一句话识别")
parameters.set("service_type", Constants.kServiceTypeASR); // 必填
//如果有HttpDns则可进行设置
//parameters.put("direct_ip", Utils.getDirectIp());
params = MapToJson(parameters);//parameters.toString();
console.log("configinfo genParams" + params)
return params;
}
genInitParams(workpath:string, debugpath:string):string {
let str:string = "";
//获取token方式：
let object:Map<string, string|number|boolean|object> = new Map();
//账号和项目创建
// ak_id ak_secret app_key如何获得,请查看https://help.aliyun.com/document_detail/72138.html
object.set("app_key",VoiceRecoginizer.appKey); // 必填
//方法1：
// 首先ak_id ak_secret app_key如何获得,请查看https://help.aliyun.com/document_detail/72138.html
// 然后请看 https://help.aliyun.com/document_detail/466615.html 使用其中方案一获取临时凭证
// 此方案简介: 远端服务器生成具有有效时限的临时凭证, 下发给移动端进行使用, 保证账号信息ak_id和ak_secret不被泄露
// 获得Token方法(运行在APP服务端): https://help.aliyun.com/document_detail/450255.html?spm=a2c4g.72153.0.0.79176297EyBj4k
if (this.authToken) {
object.set("token", this.authToken.accessToken)
}
//方法2：
// STS获取临时凭证方法暂不支持
//方法3：（强烈不推荐，存在阿里云账号泄露风险）
// 参考Auth类的实现在端上访问阿里云Token服务获取SDK进行获取。请勿将ak/sk存在本地或端侧环境。
// 此方法优点: 端侧获得Token, 无需搭建APP服务器。
// 此方法缺点: 端侧获得ak/sk账号信息, 极易泄露。
// JSONObject object = Auth.getAliYunTicket();
object.set("device_id", DeviceUtil.clientId()/*Utils.getDeviceId()*/); // 必填, 推荐填入具有唯一性的id, 方便定位问题
object.set("url", VoiceRecoginizer.url); // 默认
object.set("workspace", workpath); // 必填, 且需要有读写权限
// object.set("sample_rate", "16000");
// object.set("format", "pcm");
//当初始化SDK时的save_log参数取值为true时，该参数生效。表示是否保存音频debug，该数据保存在debug目录中，需要确保debug_path有效可写。
// object.put("save_wav", "true");
//debug目录，当初始化SDK时的save_log参数取值为true时，该目录用于保存中间音频文件。
object.set("debug_path", debugpath);
// FullMix = 0 // 选用此模式开启本地功能并需要进行鉴权注册
// FullCloud = 1
// FullLocal = 2 // 选用此模式开启本地功能并需要进行鉴权注册
// AsrMix = 3 // 选用此模式开启本地功能并需要进行鉴权注册
// AsrCloud = 4
// AsrLocal = 5 // 选用此模式开启本地功能并需要进行鉴权注册
//一句话识别
// console.log("init asr for 一句话识别")
object.set("service_mode", Constants.ModeAsrCloud); // 必填
str = MapToJson(object)
// console.info("configinfo genInitParams:" + str);
return str;
}
async AsrInit(path:string, filePath:string): Promise<number> {
//console.log("AsrInit this is " + JSON.stringify(this))
// console.info("AsrInit path: " + path);
//获取工作路径, 这里获得当前nuisdk.aar中assets路径
let asset_path:string = path+"/resources_cloud"
// fs.stat(asset_path).catch((error: BusinessError) => {
// if (error.code = 13900002) {
// fs.mkdirSync(asset_path)
// }
// })
// fs.mkdirSync(asset_path)
// try {
// await this.checkPath(asset_path)
this.cbhandle.nuiCallback = (result: string) => {
if (this.voiceRecoginizerResult) {
this.voiceRecoginizerResult(result)
}
}
let parameters: string = ""
// this.genInitParams(asset_path,filePath)
// .then(parameters) {
//
// }
await this.getToken()
let ret:number = this.g_asrinstance.initialize(this.cbhandle, this.genInitParams(asset_path,filePath), Constants.LogLevel.LOG_LEVEL_NONE, false);
// console.info("result = " + ret);
if (ret == Constants.NuiResultCode.SUCCESS) {
} else {
// Logger.debug(TAG,"语音识别初始化失败")
//final String msg_text = Utils.getMsgWithErrorCode(ret, "init");
//抛出错误异常信息。
/*
runOnUiThread(new Runnable() {
@Override
public void run() {
Toast.makeText(SpeechRecognizerActivity.this, msg_text, Toast.LENGTH_LONG).show();
}
});
*/
}
// return ret
// }
// catch {
// return -1;
// }
//初始化SDK，注意用户需要在Auth.getAliYunTicket中填入相关ID信息才可以使用。
return ret
}
// checkPath(asset_path:string):Promise<void> {
// return new Promise<void> ((success,fail) => {
// fs.stat(asset_path).catch((error: BusinessError) => {
// if (error.code = 13900002) {
// fs.mkdirSync(asset_path)
// success()
// return
// }
// success()
// })
// })
// }
startDialog():number {
// console.log(`womx startDialog uid[${process.uid}] pid[${process.pid}] tid[${process.tid}]`);
this.cbhandle.clearAsrInfo()
//由于对外的SDK不带有本地VAD模块(仅带有唤醒功能的SDK具有VAD模块)，
//若要使用VAD模式，则需要设置nls_config参数启动在线VAD模式(见genParams())
let vad_mode:Constants.VadMode = Constants.VadMode.TYPE_P2T;
if (this.vadMode) {
//TYPE_VAD: SDK自动判断句尾结束识别。(此功能仅存在于<029>带唤醒功能的SDK)
//vad_mode = Constants.VadMode.TYPE_VAD;
// console.info("使用Voice Active Detection模式");
} else {
//TYPE_P2T: 有用户主动stop()以告知识别完成
//vad_mode = Constants.VadMode.TYPE_P2T;
// console.info("使用Push To Talk模式");
}
//设置相关识别参数，具体参考API文档
// initialize()之后startDialog之前调用
this.g_asrinstance.setParams(this.genParams());
AudioCapturer.init(this.g_asrinstance)
let ret:number = this.g_asrinstance.startDialog(vad_mode, "");
// console.info("start done . ret = ", ret);
if (ret != 0) {
// console.info("call startDialog return error. ", ret);
}
return ret
}
stop() {
this.cbhandle.clearAsrInfo()
AudioCapturer.stop()
}
static async checkPemmission (context: common.UIAbilityContext) {
let grant = await PermissionUtils.checkPermissions(VoiceRecoginizer.WRITE_MEDIA)
if (grant) {
///初始化SDK
// Logger.debug("已申请过权限")
return
}
// let context = getContext()
let requestGrant = await PermissionUtils.reqPermissionsFromUser([VoiceRecoginizer.MICROPHONEMISSION
,VoiceRecoginizer.READMEDIA
,VoiceRecoginizer.WRITE_MEDIA
,VoiceRecoginizer.INTENT],context)
if (requestGrant) {
///初始化SDK
}
// return false
}
private async getToken():Promise<NetLayerVoiceRecoginizerToken | undefined> {
return new Promise<NetLayerVoiceRecoginizerToken|undefined>((success,fail) => {
if (this.authToken && this.authToken.accessToken.length > 0) {
if (Date.now() < this.authToken.expireTime - 60 * 1000) {
success(this.authToken)
return
}
}
HttpRequest.get<ResponseDTO<NetLayerVoiceRecoginizerToken>>(
HttpUrlUtils.getVoiceRecognizerSDKAuthToken()
).then((res: ResponseDTO<NetLayerVoiceRecoginizerToken>) => {
if (res.code != 0) {
fail(res.message)
return
}
this.authToken = res.data
success(res.data)
})
})
}
}
\ No newline at end of file
... ...
{
"cei":{
"cei_param_version":"2.2.0",
"cei_param_device_type":"car",
"cei_param_reco_mode":0,
"cei_param_log_level":2,
"cei_param_debug_path":"asr_debug",
"cei_param_is_debug_enable":false,
"cei_param_resource_path":"",
"cei_param_resource_is_multi_language":false,
"cei_param_audio_format_str":"16k16bitmono",
"cei_param_mcs_mode":0,
"cei_param_work_mode":0,
"cei_param_max_cache_frames":1000,
"cei_param_is_aec_bf_active":false,
"cei_param_is_agc_active":false,
"cei_param_is_vad_active":true,
"cei_param_is_kws_active":true,
"cei_param_is_sr_active":true
},
"asp":{
"asp_param_is_process_parallel":false,
"asp_param_is_input_debug_enable":false,
"asp_param_is_output_debug_enable":false,
"asp_param_debug_path":"asr_debug",
"asp_param_is_callback_enable":false,
"asp_param_callback_period_frames":5
},
"vad":{
"vad_param_is_input_debug_enable":false,
"vad_param_is_output_debug_enable":false,
"vad_param_debug_path":"asr_debug",
"vad_param_asleep_speech_noise_thres":-0.8,
"vad_param_awake_speech_noise_thres":-0.5,
"vad_param_asleep_max_speech_segment_time":300000,
"vad_param_awake_max_speech_segment_time":60000,
"vad_param_asleep_block_size":3,
"vad_param_awake_block_size":3,
"vad_param_front_timeout_interval":8000,
"vad_param_tail_timeout_interval":800,
"vad_param_is_detect_start":true,
"vad_param_is_detect_end":true
},
"kws":{
"kws_param_is_input_debug_enable":false,
"kws_param_is_output_debug_enable":false,
"kws_param_debug_path":"asr_debug",
"kws_param_is_process_parallel":false,
"kws_param_front_extend_frames":10,
"kws_param_tail_extend_frames":5,
"kws_param_encoder_type_str":"opu",
"kws_param_encoder_bitrate":16000,
"kws_param_encoder_complexity":2,
"kws_param_callback_period_ms":100,
"kws_param_max_frames_per_callback":25,
"kws_param_max_bytes_per_callback":16000
},
"sr":{
"sr_param_is_input_debug_enable":false,
"sr_param_is_output_debug_enable":false,
"sr_param_debug_path":"asr_debug",
"sr_param_is_itn_enable":true,
"sr_param_is_do_conf_filter":false,
"sr_param_is_process_parallel":true,
"sr_param_is_need_result":false,
"sr_param_is_need_voice":true,
"sr_param_ngram_conf_thres":65.0,
"sr_param_jsgf_conf_thres":65.0,
"sr_param_encoder_type_str":"opu",
"sr_param_encoder_bitrate":16000,
"sr_param_encoder_complexity":2,
"sr_param_callback_period_ms":100,
"sr_param_max_frames_per_callback":25,
"sr_param_max_bytes_per_callback":16000
},
"ou":{
"oss_upload_param_is_enable":false,
"oss_upload_param_asp_in":false,
"oss_upload_param_asp_out":false,
"oss_upload_param_vad_in":false,
"oss_upload_param_vad_out":false,
"oss_upload_param_kws_in":false,
"oss_upload_param_kws_susp_in":false,
"oss_upload_param_kws_out":false,
"oss_upload_param_kws_susp_out":false,
"oss_upload_param_sr_in":false
}
}
... ...
{
"device_type":"phone",
"assets_version":"1.1.20190902",
"nui_config":{
"service_mode":"kModeFullCloud",
"log_level":"kLogLevelVerbose",
"enable_recorder_by_user":true,
"enable_callback_recording":false,
"enable_dialog":false
},
"nls_config":{
"debug_level":4,
"sr_format":"opus",
"sample_rate":16000,
"dns_timeout":5000,
"vocab_default_weight":2
},
"audio_config":{
"debug":"none",
"16k_audio":{
"name":"16kmono16bit",
"id":0,
"mic":{
"name":"16kmono16bit",
"debug_heap_pollution":false,
"read_cnt":0,
"sample_rate":16000,
"bits_per_sample":16,
"channels":1,
"recording_interval":10,
"cei_frame_time_len":20,
"channel_mask":"kAMChannalMaskNone",
"format_type":"kAMDataFormatPcmInterleaved",
"endianness":"kAMByteOrderLittleEndian"
}
},
"8k_audio":{
"name":"8kmono16bit",
"id":0,
"mic":{
"name":"8kmono16bit",
"debug_heap_pollution":false,
"read_cnt":0,
"sample_rate":8000,
"bits_per_sample":16,
"channels":1,
"recording_interval":10,
"cei_frame_time_len":20,
"channel_mask":"kAMChannalMaskNone",
"format_type":"kAMDataFormatPcmInterleaved",
"endianness":"kAMByteOrderLittleEndian"
}
}
}
}
... ...
{
"device_type":"car",
"mode_type":2,
"smart_engine":3,
"encode_type":"pcm",
"debug_info":{
"debug_level":0,
"debug_path":"/sdcard/tmp"
},
"common_info":{
"pitch_level":0,
"speed_level":1.0,
"volume":1.0,
"play_audio":0,
"callback_raw_data":1,
"sample_rate":16000,
"font_name":"xiaoyun",
"wait_time":2000
},
"cache":{
"on":true,
"max_cache_size":2000000,
"save_path":"/sdcard/mit/"
},
"font":{
"on":false,
"mode":"cmd",
"save_path":"/sdcard/mit"
}
}
... ...
{
"cei":{
"cei_param_version":"2.2.0",
"cei_param_device_type":"car",
"cei_param_reco_mode":0,
"cei_param_log_level":2,
"cei_param_debug_path":"asr_debug",
"cei_param_is_debug_enable":false,
"cei_param_resource_path":"",
"cei_param_resource_is_multi_language":false,
"cei_param_audio_format_str":"16k16bitmono",
"cei_param_mcs_mode":0,
"cei_param_work_mode":0,
"cei_param_max_cache_frames":1000,
"cei_param_is_aec_bf_active":false,
"cei_param_is_agc_active":false,
"cei_param_is_vad_active":true,
"cei_param_is_kws_active":true,
"cei_param_is_sr_active":true
},
"asp":{
"asp_param_is_process_parallel":false,
"asp_param_is_input_debug_enable":false,
"asp_param_is_output_debug_enable":false,
"asp_param_debug_path":"asr_debug",
"asp_param_is_callback_enable":false,
"asp_param_callback_period_frames":5
},
"vad":{
"vad_param_is_input_debug_enable":false,
"vad_param_is_output_debug_enable":false,
"vad_param_debug_path":"asr_debug",
"vad_param_asleep_speech_noise_thres":-0.8,
"vad_param_awake_speech_noise_thres":-0.5,
"vad_param_asleep_max_speech_segment_time":300000,
"vad_param_awake_max_speech_segment_time":60000,
"vad_param_asleep_block_size":3,
"vad_param_awake_block_size":3,
"vad_param_front_timeout_interval":8000,
"vad_param_tail_timeout_interval":800,
"vad_param_is_detect_start":true,
"vad_param_is_detect_end":true
},
"kws":{
"kws_param_is_input_debug_enable":false,
"kws_param_is_output_debug_enable":false,
"kws_param_debug_path":"asr_debug",
"kws_param_is_process_parallel":false,
"kws_param_front_extend_frames":10,
"kws_param_tail_extend_frames":5,
"kws_param_encoder_type_str":"opu",
"kws_param_encoder_bitrate":16000,
"kws_param_encoder_complexity":2,
"kws_param_callback_period_ms":100,
"kws_param_max_frames_per_callback":25,
"kws_param_max_bytes_per_callback":16000
},
"sr":{
"sr_param_is_input_debug_enable":false,
"sr_param_is_output_debug_enable":false,
"sr_param_debug_path":"asr_debug",
"sr_param_is_itn_enable":true,
"sr_param_is_do_conf_filter":false,
"sr_param_is_process_parallel":true,
"sr_param_is_need_result":false,
"sr_param_is_need_voice":true,
"sr_param_ngram_conf_thres":65.0,
"sr_param_jsgf_conf_thres":65.0,
"sr_param_encoder_type_str":"opu",
"sr_param_encoder_bitrate":16000,
"sr_param_encoder_complexity":2,
"sr_param_callback_period_ms":100,
"sr_param_max_frames_per_callback":25,
"sr_param_max_bytes_per_callback":16000
},
"ou":{
"oss_upload_param_is_enable":false,
"oss_upload_param_asp_in":false,
"oss_upload_param_asp_out":false,
"oss_upload_param_vad_in":false,
"oss_upload_param_vad_out":false,
"oss_upload_param_kws_in":false,
"oss_upload_param_kws_susp_in":false,
"oss_upload_param_kws_out":false,
"oss_upload_param_kws_susp_out":false,
"oss_upload_param_sr_in":false
}
}
... ...
{
"device_type":"phone",
"assets_version":"1.1.20190902",
"nui_config":{
"service_mode":"kModeFullCloud",
"log_level":"kLogLevelVerbose",
"enable_recorder_by_user":true,
"enable_callback_recording":false,
"enable_dialog":false
},
"nls_config":{
"debug_level":4,
"sr_format":"opus",
"sample_rate":16000,
"dns_timeout":5000,
"vocab_default_weight":2
},
"audio_config":{
"debug":"none",
"16k_audio":{
"name":"16kmono16bit",
"id":0,
"mic":{
"name":"16kmono16bit",
"debug_heap_pollution":false,
"read_cnt":0,
"sample_rate":16000,
"bits_per_sample":16,
"channels":1,
"recording_interval":10,
"cei_frame_time_len":20,
"channel_mask":"kAMChannalMaskNone",
"format_type":"kAMDataFormatPcmInterleaved",
"endianness":"kAMByteOrderLittleEndian"
}
},
"8k_audio":{
"name":"8kmono16bit",
"id":0,
"mic":{
"name":"8kmono16bit",
"debug_heap_pollution":false,
"read_cnt":0,
"sample_rate":8000,
"bits_per_sample":16,
"channels":1,
"recording_interval":10,
"cei_frame_time_len":20,
"channel_mask":"kAMChannalMaskNone",
"format_type":"kAMDataFormatPcmInterleaved",
"endianness":"kAMByteOrderLittleEndian"
}
}
}
}
... ...
{
"device_type":"car",
"mode_type":2,
"smart_engine":3,
"encode_type":"pcm",
"debug_info":{
"debug_level":0,
"debug_path":"/sdcard/tmp"
},
"common_info":{
"pitch_level":0,
"speed_level":1.0,
"volume":1.0,
"play_audio":0,
"callback_raw_data":1,
"sample_rate":16000,
"font_name":"xiaoyun",
"wait_time":2000
},
"cache":{
"on":true,
"max_cache_size":2000000,
"save_path":"/sdcard/mit/"
},
"font":{
"on":false,
"mode":"cmd",
"save_path":"/sdcard/mit"
}
}
... ...