VoiceRecoginizer.ets
14.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
import lazy {NativeNui,Constants,MapToJson,INativeNuiCallback, AsrResult, KwsResult} from 'neonui'
import { PermissionUtils,DeviceUtil, Logger, DateTimeUtils } from 'wdKit';
import AudioCapturer from './AudioCapture';
import { common, Permissions } from '@kit.AbilityKit';
import { NetLayerVoiceRecoginizerToken } from 'wdBean';
import { HttpUrlUtils,ResponseDTO } from 'wdNetwork'
import { HttpRequest } from 'wdNetwork/src/main/ets/http/HttpRequest'
class NativeNuiCallbackHandle implements INativeNuiCallback {
asrspeechrealtimeResultOld:string=""
asrmessage:string;
message:string;
nuiCallback?:(result: string) => void
constructor() {
this.asrmessage=""
this.message=""
}
onNuiEventCallback(event:Constants.NuiEvent, resultCode:number, arg2:number, kwsResult:KwsResult,
asrResult:AsrResult):void {
let asrinfo:string = ""
// console.log("onUsrNuiEventCallback last this.asrmessage is " + this.asrmessage)
// console.log("onUsrNuiEventCallback this is " + JSON.stringify(this))
// console.log("onUsrNuiEventCallback event is " + event)
if (event === Constants.NuiEvent.EVENT_ASR_RESULT){
this.message = "EVENT_ASR_RESULT"
this.asrspeechrealtimeResultOld=""
} else if (event === Constants.NuiEvent.EVENT_ASR_PARTIAL_RESULT || event === Constants.NuiEvent.EVENT_SENTENCE_END){
if (event === Constants.NuiEvent.EVENT_ASR_PARTIAL_RESULT ) {
this.message = "EVENT_ASR_PARTIAL_RESULT"
} else if(event === Constants.NuiEvent.EVENT_SENTENCE_END){
this.message = "EVENT_SENTENCE_END"
}
} else if (event === Constants.NuiEvent.EVENT_ASR_ERROR){
this.message = "EVENT_ASR_ERROR"
} else if (event === Constants.NuiEvent.EVENT_MIC_ERROR){
this.message = "EVENT_MIC_ERROR"
} else if (event === Constants.NuiEvent.EVENT_DIALOG_EX){
this.message = "EVENT_DIALOG_EX"
}
if (asrResult) {
asrinfo = asrResult.asrResult
//console.log(`asrinfo string is ${asrinfo}`)
if (asrinfo) {
try {
let asrresult_json:object|null = JSON.parse(asrResult.asrResult)
//console.log(JSON.stringify(asrresult_json))
if (asrresult_json) {
let payload:object|null = asrresult_json["payload"];
if (payload) {
//console.log(JSON.stringify(payload))
this.asrmessage = this.asrspeechrealtimeResultOld + payload["result"];
if(event === Constants.NuiEvent.EVENT_SENTENCE_END){
// console.info("onUsrNuiEventCallback EVENT_SENTENCE_END")
this.asrspeechrealtimeResultOld = this.asrmessage
}
if (this.nuiCallback) {
this.nuiCallback(this.asrmessage)
}
}
}
} catch (e){
// console.error("got asrinfo not json, so donot fresh asrinfo." + JSON.stringify(e))
}
}
}
// console.info(`womx onUsrNuiEventCallback(${event}, ${resultCode},${arg2}, kwsResult:${kwsResult},asrResult:"${this.asrmessage}") done`)
}
onNuiNeedAudioData(buffer:ArrayBuffer):number {
// console.info(`womx onUsrNuiNeedAudioData(buffer length = ${buffer.byteLength})`)
// console.log(`womx onUsrNuiNeedAudioData uid[${process.uid}] pid[${process.pid}] tid[${process.tid}]`);
let num:number = 0;
if (true){
if (true){
//由于录音回调是异步的,所以此处获取共享数据需要加锁/并保持同步 是较难的,需要修改asr回调为用户主动推送方式
num = AudioCapturer.getVoiceArrayBuffer1(buffer)
} else {
let flag_returned:boolean = false
try {
AudioCapturer.getVoiceArrayBuffer(buffer).then( (result:number)=>{
console.log("womx onUsrNuiNeedAudioData getVoiceArrayBuffer done, result="+result)
num = result
flag_returned=true
})
} catch(e ){
flag_returned=true
}
while(flag_returned==false){
console.log("womx onUsrNuiNeedAudioData waiting ... num="+num)
}
}
}
// console.info(`womx onUsrNuiNeedAudioData(buffer length = ${buffer.byteLength}) return ${num}}`)
return num;//buffer.byteLength;
}
onNuiAudioStateChanged(state:Constants.AudioState):void {
// console.info(`womx onUsrNuiAudioStateChanged(${state})`)
if (state === Constants.AudioState.STATE_OPEN){
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder start`)
//AudioCapturer.init(g_asrinstance)
AudioCapturer.start()
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder start done`)
} else if (state === Constants.AudioState.STATE_CLOSE){
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder close`)
AudioCapturer.stop()
//AudioCapturer.release()
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder close done`)
} else if (state === Constants.AudioState.STATE_PAUSE){
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder pause`)
AudioCapturer.stop()
//AudioCapturer.release()
// console.info(`womx onUsrNuiAudioStateChanged(${state}) audio recorder pause done`)
}
}
onNuiAudioRMSChanged(val:number):number{
// console.info(`womx onNuiAudioRMSChanged(${val})`)
return 0;
}
clearAsrInfo(){
this.asrmessage = ""
}
}
const TAG = 'VoiceRecoginizer'
export class VoiceRecoginizer {
private static MICROPHONEMISSION: Permissions = 'ohos.permission.MICROPHONE'
private static READMEDIA: Permissions = 'ohos.permission.READ_IMAGEVIDEO'
private static INTENT: Permissions = "ohos.permission.INTERNET"
private static WRITE_MEDIA: Permissions = "ohos.permission.WRITE_IMAGEVIDEO"
private static appKey = "EospGmM6mdPljjjm"
private static url = "wss://nls-gateway.cn-shanghai.aliyuncs.com:443/ws/v1"
private authToken?: NetLayerVoiceRecoginizerToken
private vadMode: boolean = false
private cbhandle: NativeNuiCallbackHandle = new NativeNuiCallbackHandle()
private g_asrinstance: NativeNui = new NativeNui(Constants.ModeType.MODE_DIALOG, "asr1")
private static manager: VoiceRecoginizer
public static sharedManager(): VoiceRecoginizer {
if (!VoiceRecoginizer.manager) {
VoiceRecoginizer.manager = new VoiceRecoginizer()
}
return VoiceRecoginizer.manager
}
voiceRecoginizerResult?:(result: string) => void
// public voiceRecoginzerResult:(result: string) => void
genParams():string {
let params:string = "";
let nls_config:Map<string, string|number|boolean|object> = new Map();
nls_config.set("enable_intermediate_result", true);
//参数可根据实际业务进行配置
//接口说明可见: https://help.aliyun.com/document_detail/173298.html
//查看 2.开始识别
//由于对外的SDK不带有本地VAD模块(仅带有唤醒功能的SDK具有VAD模块),
//若要使用VAD模式,则需要设置nls_config参数启动在线VAD模式(见genParams())
if (this.vadMode) {
nls_config.set("enable_voice_detection", true);
nls_config.set("max_start_silence", 10000);
nls_config.set("max_end_silence", 800);
} else {
nls_config.set("enable_voice_detection", false);
}
nls_config.set("enable_punctuation_prediction", true);
nls_config.set("enable_inverse_text_normalization", true);
// nls_config.set("customization_id", "test_id");
// nls_config.set("vocabulary_id", "test_id");
// nls_config.set("sample_rate", 16000);
// nls_config.set("sr_format", "opus");
let parameters:Map<string, string|number|boolean|object> = new Map();
parameters.set("nls_config", Object( JSON.parse(MapToJson(nls_config)) ) );
//一句话识别
// console.log("start asr for 一句话识别")
parameters.set("service_type", Constants.kServiceTypeASR); // 必填
//如果有HttpDns则可进行设置
//parameters.put("direct_ip", Utils.getDirectIp());
params = MapToJson(parameters);//parameters.toString();
console.log("configinfo genParams" + params)
return params;
}
genInitParams(workpath:string, debugpath:string):string {
let str:string = "";
//获取token方式:
let object:Map<string, string|number|boolean|object> = new Map();
//账号和项目创建
// ak_id ak_secret app_key如何获得,请查看https://help.aliyun.com/document_detail/72138.html
object.set("app_key",VoiceRecoginizer.appKey); // 必填
//方法1:
// 首先ak_id ak_secret app_key如何获得,请查看https://help.aliyun.com/document_detail/72138.html
// 然后请看 https://help.aliyun.com/document_detail/466615.html 使用其中方案一获取临时凭证
// 此方案简介: 远端服务器生成具有有效时限的临时凭证, 下发给移动端进行使用, 保证账号信息ak_id和ak_secret不被泄露
// 获得Token方法(运行在APP服务端): https://help.aliyun.com/document_detail/450255.html?spm=a2c4g.72153.0.0.79176297EyBj4k
if (this.authToken) {
object.set("token", this.authToken.accessToken)
}
//方法2:
// STS获取临时凭证方法暂不支持
//方法3:(强烈不推荐,存在阿里云账号泄露风险)
// 参考Auth类的实现在端上访问阿里云Token服务获取SDK进行获取。请勿将ak/sk存在本地或端侧环境。
// 此方法优点: 端侧获得Token, 无需搭建APP服务器。
// 此方法缺点: 端侧获得ak/sk账号信息, 极易泄露。
// JSONObject object = Auth.getAliYunTicket();
object.set("device_id", DeviceUtil.clientId()/*Utils.getDeviceId()*/); // 必填, 推荐填入具有唯一性的id, 方便定位问题
object.set("url", VoiceRecoginizer.url); // 默认
object.set("workspace", workpath); // 必填, 且需要有读写权限
// object.set("sample_rate", "16000");
// object.set("format", "pcm");
//当初始化SDK时的save_log参数取值为true时,该参数生效。表示是否保存音频debug,该数据保存在debug目录中,需要确保debug_path有效可写。
// object.put("save_wav", "true");
//debug目录,当初始化SDK时的save_log参数取值为true时,该目录用于保存中间音频文件。
object.set("debug_path", debugpath);
// FullMix = 0 // 选用此模式开启本地功能并需要进行鉴权注册
// FullCloud = 1
// FullLocal = 2 // 选用此模式开启本地功能并需要进行鉴权注册
// AsrMix = 3 // 选用此模式开启本地功能并需要进行鉴权注册
// AsrCloud = 4
// AsrLocal = 5 // 选用此模式开启本地功能并需要进行鉴权注册
//一句话识别
// console.log("init asr for 一句话识别")
object.set("service_mode", Constants.ModeAsrCloud); // 必填
str = MapToJson(object)
// console.info("configinfo genInitParams:" + str);
return str;
}
async AsrInit(path:string, filePath:string): Promise<number> {
Logger.debugOptimize(TAG, () => {
return "AsrInit this is " + JSON.stringify(this)
})
// console.info("AsrInit path: " + path);
//获取工作路径, 这里获得当前nuisdk.aar中assets路径
let asset_path:string = path+"/resources_cloud"
// fs.stat(asset_path).catch((error: BusinessError) => {
// if (error.code = 13900002) {
// fs.mkdirSync(asset_path)
// }
// })
// fs.mkdirSync(asset_path)
// try {
// await this.checkPath(asset_path)
this.cbhandle.nuiCallback = (result: string) => {
if (this.voiceRecoginizerResult) {
this.voiceRecoginizerResult(result)
}
}
let parameters: string = ""
// this.genInitParams(asset_path,filePath)
// .then(parameters) {
//
// }
await this.getToken()
let ret:number = this.g_asrinstance.initialize(this.cbhandle, this.genInitParams(asset_path,filePath), Constants.LogLevel.LOG_LEVEL_NONE, false);
// console.info("result = " + ret);
if (ret == Constants.NuiResultCode.SUCCESS) {
} else {
// Logger.debug(TAG,"语音识别初始化失败")
//final String msg_text = Utils.getMsgWithErrorCode(ret, "init");
//抛出错误异常信息。
/*
runOnUiThread(new Runnable() {
@Override
public void run() {
Toast.makeText(SpeechRecognizerActivity.this, msg_text, Toast.LENGTH_LONG).show();
}
});
*/
}
// return ret
// }
// catch {
// return -1;
// }
//初始化SDK,注意用户需要在Auth.getAliYunTicket中填入相关ID信息才可以使用。
return ret
}
// checkPath(asset_path:string):Promise<void> {
// return new Promise<void> ((success,fail) => {
// fs.stat(asset_path).catch((error: BusinessError) => {
// if (error.code = 13900002) {
// fs.mkdirSync(asset_path)
// success()
// return
// }
// success()
// })
// })
// }
startDialog():number {
// console.log(`womx startDialog uid[${process.uid}] pid[${process.pid}] tid[${process.tid}]`);
this.cbhandle.clearAsrInfo()
//由于对外的SDK不带有本地VAD模块(仅带有唤醒功能的SDK具有VAD模块),
//若要使用VAD模式,则需要设置nls_config参数启动在线VAD模式(见genParams())
let vad_mode:Constants.VadMode = Constants.VadMode.TYPE_P2T;
if (this.vadMode) {
//TYPE_VAD: SDK自动判断句尾结束识别。(此功能仅存在于<029>带唤醒功能的SDK)
//vad_mode = Constants.VadMode.TYPE_VAD;
// console.info("使用Voice Active Detection模式");
} else {
//TYPE_P2T: 有用户主动stop()以告知识别完成
//vad_mode = Constants.VadMode.TYPE_P2T;
// console.info("使用Push To Talk模式");
}
//设置相关识别参数,具体参考API文档
// initialize()之后startDialog之前调用
this.g_asrinstance.setParams(this.genParams());
AudioCapturer.init(this.g_asrinstance)
let ret:number = this.g_asrinstance.startDialog(vad_mode, "");
// console.info("start done . ret = ", ret);
if (ret != 0) {
// console.info("call startDialog return error. ", ret);
}
return ret
}
stop() {
this.cbhandle.clearAsrInfo()
AudioCapturer.stop()
}
static async checkPemmission (context: common.UIAbilityContext) {
let grant = await PermissionUtils.checkPermissions(VoiceRecoginizer.WRITE_MEDIA)
if (grant) {
///初始化SDK
// Logger.debug("已申请过权限")
return
}
// let context = getContext()
let requestGrant = await PermissionUtils.reqPermissionsFromUser([VoiceRecoginizer.MICROPHONEMISSION
,VoiceRecoginizer.READMEDIA
,VoiceRecoginizer.WRITE_MEDIA
,VoiceRecoginizer.INTENT],context)
if (requestGrant) {
///初始化SDK
}
// return false
}
private async getToken():Promise<NetLayerVoiceRecoginizerToken | undefined> {
return new Promise<NetLayerVoiceRecoginizerToken|undefined>((success,fail) => {
if (this.authToken && this.authToken.accessToken.length > 0) {
if (Date.now() < this.authToken.expireTime - 60 * 1000) {
success(this.authToken)
return
}
}
HttpRequest.get<ResponseDTO<NetLayerVoiceRecoginizerToken>>(
HttpUrlUtils.getVoiceRecognizerSDKAuthToken()
).then((res: ResponseDTO<NetLayerVoiceRecoginizerToken>) => {
if (res.code != 0) {
fail(res.message)
return
}
this.authToken = res.data
success(res.data)
})
})
}
}