Skip to content

R
robot-trans-platform
Commit c740db49 authored by 赵啸非's avatar 赵啸非
Browse files
Options

添加音频识别

parent f8ab5faa
Hide whitespace changes
Inline Side-by-side
Showing with 327 additions and 39 deletions
robot-trans-manager/src/main/java/com/mortals/xhx/busiz/web/ApiSendMsgController.java
View file @ c740db49
......@@ -7,7 +7,11 @@ import com.mortals.framework.service.ILogService;
import com.mortals.framework.service.impl.FileLogServiceImpl;
import com.mortals.xhx.base.system.upload.service.UploadService;
import com.mortals.xhx.common.utils.IatModelMulMain;
import com.mortals.xhx.common.utils.IatModelMulUtil;
import lombok.extern.slf4j.Slf4j;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.WebSocket;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
......@@ -49,39 +53,11 @@ public class ApiSendMsgController {
@GetMapping(value = "/events/create", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public Flux<String> streamEventsCreate() {
/* return Flux.push(sink -> {
new Thread(() -> {
try {
for (int i = 1; i <= 1; i++) {
sink.next("Message " + i);
Thread.sleep(1000);
}
sink.complete();
} catch (InterruptedException e) {
sink.error(e);
}
}).start();
});*/
/* return Flux.create(sink -> {
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
executor.scheduleAtFixedRate(() -> {
sink.next("Message " + System.currentTimeMillis());
}, 0, 1, TimeUnit.SECONDS);
});*/
return Flux.create(sink -> {
for (int i = 1; i <= 10; i++) {
new Thread(() -> { // 需要异步执行，否则会阻塞
sink.next("Message ");
// 每秒发送一条数据
}).start();
try {
......@@ -92,9 +68,6 @@ public class ApiSendMsgController {
}
sink.complete(); // 结束流
});
//return null;
}
@GetMapping("/emitter")
......@@ -113,17 +86,27 @@ public class ApiSendMsgController {
return emitter;
}
@RequestMapping(value = "upload")
public String doFileUpload(MultipartFile file, @RequestParam(value = "prePath", defaultValue = "") String prePath) {
@RequestMapping(value = "upload", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
@UnAuth
public SseEmitter doFileUpload(MultipartFile file, @RequestParam(value = "prePath", defaultValue = "") String prePath) {
Map<String, Object> model = new HashMap<>();
String jsonStr = "";
SseEmitter emitter = new SseEmitter(30000L);
try {
String filePath = uploadService.saveFileUpload(file, prePath, null);
//将音频文件传给ai大模型
filePath = uploadService.getFilePath(filePath);
IatModelMulUtil iatModelMulUtil = new IatModelMulUtil(filePath, appid, emitter);
String authUrl = iatModelMulUtil.getAuthUrl(hostUrl, apiKey, apiSecret);
//log.info("authUrl==>" + authUrl);
OkHttpClient client = new OkHttpClient.Builder().build();
// IatModelMulMain.getAuthUrl()
String url = authUrl.toString().replace("http://", "ws://").replace("https://", "wss://");
Request request = new Request.Builder().url(url).build();
WebSocket webSocket = client.newWebSocket(request, iatModelMulUtil);
// model.put("url", filePath);
model.put("fileName", file.getOriginalFilename());
......@@ -140,7 +123,7 @@ public class ApiSendMsgController {
model.put(KEY_RESULT_MSG, "文件上传失败");*/
jsonStr = JSONObject.toJSONString(model);
}
return jsonStr;
return emitter;
}
}
robot-trans-manager/src/main/java/com/mortals/xhx/common/utils/IatModelMulMain.java
View file @ c740db49
......@@ -114,7 +114,7 @@ public class IatModelMulMain extends WebSocketListener {
"}";
webSocket.send(json);
// System.err.println(json);
System.out.println("中间帧音频发送中..."+seq);
// System.out.println("中间帧音频发送中..."+seq);
break;
case StatusLastFrame: // 最后一帧音频status = 2 ，标志音频发送结束
json = "{\n" +
......@@ -155,7 +155,7 @@ public class IatModelMulMain extends WebSocketListener {
@Override
public void onMessage(WebSocket webSocket, String text) {
super.onMessage(webSocket, text);
System.out.println(text);
//System.out.println(text);
JsonParse jsonParse = gson.fromJson(text, JsonParse.class);
if (jsonParse != null) {
if (jsonParse.header.code != 0) {
......
robot-trans-manager/src/main/java/com/mortals/xhx/common/utils/IatModelMulUtil.java 0 → 100644
View file @ c740db49
package com.mortals.xhx.common.utils;
import com.google.gson.Gson;
import lombok.extern.slf4j.Slf4j;
import okhttp3.HttpUrl;
import okhttp3.Response;
import okhttp3.WebSocket;
import okhttp3.WebSocketListener;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* 大模型语音识别---方言识别
*/
@Slf4j
public class IatModelMulUtil extends WebSocketListener {
/* private static final String hostUrl = "https://iat.cn-huabei-1.xf-yun.com/v1"; // 注意多语种识别，也支持中文音频
private static final String appid = "3cc52607"; //在控制台-我的应用获取
private static final String apiSecret = "ZTdmMjFjMGYxYmJhN2VmYjFlMTg3N2Rk"; // 在控制台-我的应用获取
private static final String apiKey = "d0f73d44e996c2da9924c4476c578a30"; // 在控制台-我的应用获取
private static final String file = "E://test1.mp3"; // 识别音频位置*/
public static final int StatusFirstFrame = 0;
public static final int StatusContinueFrame = 1;
public static final int StatusLastFrame = 2;
public static final Gson gson = new Gson();
// 开始时间
private static Date dateBegin = new Date();
// 结束时间
private static Date dateEnd = new Date();
private static final SimpleDateFormat sdf = new SimpleDateFormat("yyy-MM-dd HH:mm:ss.SSS");
private String file; // 识别音频位置*/
private String appid;
private SseEmitter emitter;
public IatModelMulUtil(String file, String appid, SseEmitter emitter) {
this.file = file;
this.appid = appid;
this.emitter = emitter;
}
@Override
public void onOpen(WebSocket webSocket, Response response) {
super.onOpen(webSocket, response);
new Thread(() -> {
//连接成功，开始发送数据
int frameSize = 1280; //每一帧音频的大小,建议每 40ms 发送 122B
int intervel = 40;
int status = 0; // 音频的状态
int seq = 0; //数据序号
try (FileInputStream fs = new FileInputStream(file)) {
byte[] buffer = new byte[frameSize];
// 发送音频
end:
while (true) {
seq++; // 每次循环更新下seq
int len = fs.read(buffer);
if (len == -1) {
status = StatusLastFrame; //文件读完，改变status 为 2
}
switch (status) {
case StatusFirstFrame: // 第一帧音频status = 0
String json = "{\n" +
" \"header\": {\n" +
" \"app_id\": \"" + appid + "\",\n" +
" \"status\": " + StatusFirstFrame + "\n" +
" },\n" +
" \"parameter\": {\n" +
" \"iat\": {\n" +
" \"domain\": \"slm\",\n" +
" \"language\": \"zh_cn\",\n" +
" \"accent\": \"mulacc\",\n" +
" \"eos\": 6000,\n" +
" \"vinfo\": 1,\n" +
" \"result\": {\n" +
" \"encoding\": \"utf8\",\n" +
" \"compress\": \"raw\",\n" +
" \"format\": \"json\"\n" +
" }\n" +
" }\n" +
" },\n" +
" \"payload\": {\n" +
" \"audio\": {\n" +
" \"encoding\": \"raw\",\n" +
" \"sample_rate\": 16000,\n" +
" \"channels\": 1,\n" +
" \"bit_depth\": 16,\n" +
" \"seq\": " + seq + ",\n" +
" \"status\": 0,\n" +
" \"audio\": \"" + Base64.getEncoder().encodeToString(Arrays.copyOf(buffer, len)) + "\"\n" +
" }\n" +
" }\n" +
"}";
webSocket.send(json);
status = StatusContinueFrame; // 发送完第一帧改变status 为 1
break;
case StatusContinueFrame: //中间帧status = 1
json = "{\n" +
" \"header\": {\n" +
" \"app_id\": \"" + appid + "\",\n" +
" \"status\": 1\n" +
" },\n" +
" \"payload\": {\n" +
" \"audio\": {\n" +
" \"encoding\": \"raw\",\n" +
" \"sample_rate\": 16000,\n" +
" \"channels\": 1,\n" +
" \"bit_depth\": 16,\n" +
" \"seq\": " + seq + ",\n" +
" \"status\": 1,\n" +
" \"audio\": \"" + Base64.getEncoder().encodeToString(Arrays.copyOf(buffer, len)) + "\"\n" +
" }\n" +
" }\n" +
"}";
webSocket.send(json);
break;
case StatusLastFrame: // 最后一帧音频status = 2 ，标志音频发送结束
json = "{\n" +
" \"header\": {\n" +
" \"app_id\": \"" + appid + "\",\n" +
" \"status\": 2\n" +
" },\n" +
" \"payload\": {\n" +
" \"audio\": {\n" +
" \"encoding\": \"raw\",\n" +
" \"sample_rate\": 16000,\n" +
" \"channels\": 1,\n" +
" \"bit_depth\": 16,\n" +
" \"seq\": " + seq + ",\n" +
" \"status\": 2,\n" +
" \"audio\": \"\"\n" +
" }\n" +
" }\n" +
"}";
webSocket.send(json);
break end;
}
Thread.sleep(intervel); //模拟音频采样延时
}
} catch (Exception e) {
log.info("error==>" + e.getMessage());
}
}).start();
}
@Override
public void onMessage(WebSocket webSocket, String text) {
super.onMessage(webSocket, text);
JsonParse jsonParse = gson.fromJson(text, JsonParse.class);
if (jsonParse != null) {
if (jsonParse.header.code != 0) {
log.error("code=>{} error=>{} sid={}", jsonParse.header.code, jsonParse.header.message, jsonParse.header.sid);
return;
}
if (jsonParse.payload != null) {
if (jsonParse.payload.result.text != null) { // 中间结果
byte[] decodedBytes = Base64.getDecoder().decode(jsonParse.payload.result.text);
String decodeRes = new String(decodedBytes, StandardCharsets.UTF_8);
JsonParseText jsonParseText = gson.fromJson(decodeRes, JsonParseText.class);
List<Ws> wsList = jsonParseText.ws;
// System.out.print("中间识别结果==》");
for (Ws ws : wsList) {
List<Cw> cwList = ws.cw;
for (Cw cw : cwList) {
log.info("识别结果==》{}", cw.w);
try {
emitter.send(cw.w);
} catch (IOException e) {
log.error("发送异常==》{}", e.getMessage());
}
//emitter.n
// emitter.next(rsp);
// System.out.print(cw.w);
}
}
}
if (jsonParse.payload.result.status == 2) { // 最终结果 说明数据全部返回完毕，可以关闭连接，释放资源
System.out.println("session end ");
dateEnd = new Date();
log.info(sdf.format(dateBegin) + "开始");
log.info(sdf.format(dateEnd) + "结束");
log.info("耗时:" + (dateEnd.getTime() - dateBegin.getTime()) + "ms");
// System.out.println("最终识别结果 ==》" + decodeRes); // 按照规则替换与追加出最终识别结果
log.info("本次识别sid ==》" + jsonParse.header.sid);
emitter.complete();
webSocket.close(1000, "");
}
}
}
}
@Override
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
super.onFailure(webSocket, t, response);
try {
if (null != response) {
int code = response.code();
log.info("onFailure code:" + code);
log.info("onFailure body:" + response.body().string());
/* if (101 != code) {
System.out.println("connection failed");
System.exit(0);
}*/
}
} catch (IOException e) {
// TODO Auto-generated catch block
log.error("onFailure error:" + e.getMessage());
}
}
public static String getAuthUrl(String hostUrl, String apiKey, String apiSecret) throws Exception {
URL url = new URL(hostUrl);
SimpleDateFormat format = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US);
format.setTimeZone(TimeZone.getTimeZone("GMT"));
String date = format.format(new Date());
StringBuilder builder = new StringBuilder("host: ").append(url.getHost()).append("\n").//
append("date: ").append(date).append("\n").//
append("GET ").append(url.getPath()).append(" HTTP/1.1");
// System.out.println(builder);
Charset charset = Charset.forName("UTF-8");
Mac mac = Mac.getInstance("hmacsha256");
SecretKeySpec spec = new SecretKeySpec(apiSecret.getBytes(charset), "hmacsha256");
mac.init(spec);
byte[] hexDigits = mac.doFinal(builder.toString().getBytes(charset));
String sha = Base64.getEncoder().encodeToString(hexDigits);
//System.out.println(sha);
String authorization = String.format("api_key=\"%s\", algorithm=\"%s\", headers=\"%s\", signature=\"%s\"", apiKey, "hmac-sha256", "host date request-line", sha);
//System.out.println(authorization);
HttpUrl httpUrl = HttpUrl.parse("https://" + url.getHost() + url.getPath()).newBuilder().//
addQueryParameter("authorization", Base64.getEncoder().encodeToString(authorization.getBytes(charset))).//
addQueryParameter("date", date).//
addQueryParameter("host", url.getHost()).//
build();
return httpUrl.toString();
}
// 返回结果拆分与展示，仅供参考
// 返回结果拆分与展示，仅供参考
class JsonParse {
Header header;
Payload payload;
}
class Header {
int code;
String message;
String sid;
int status;
}
class Payload {
Result result;
}
class Result {
String text;
int status;
}
class JsonParseText {
List<Ws> ws;
String pgs;
List<Integer> rg;
}
class Ws {
List<Cw> cw;
}
class Cw {
String w;
}
}
robot-trans-manager/src/test/java/httpclient/system.http
View file @ c740db49
......@@ -33,6 +33,17 @@ Content-Disposition: form-data; name="file"; filename="file.zip"
< ./file.zip
--WebAppBoundary--
###上传音频文件
POST {{baseUrl}}/audio/upload
Content-Type: multipart/form-data; boundary=WebAppBoundary
--WebAppBoundary
Content-Disposition: form-data; name="file"; filename="test.pcm"
< ./test.pcm
--WebAppBoundary--
###测试链接数据库
POST {{baseUrl}}/m/db/connect
Content-Type: application/json
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment