PublicMirrors/voice-changer
1
0
Fork 0
mirror of https://github.com/w-okada/voice-changer.git synced 2025-02-02 16:23:58 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

add internal callback

Browse Source
This commit is contained in:
w-okada 2023-09-28 04:19:41 +09:00
parent 265705f087
commit cfe2e6afd1
11 changed files with 1337 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/demo/dist/assets/gui_settings/GUI.json vendored
View File

@ -22,7 +22,7 @@
"name": "configArea", "name": "configArea",
"options": { "options": {
"detectors": ["dio", "harvest", "crepe", "crepe_full", "crepe_tiny", "rmvpe", "rmvpe_onnx"], "detectors": ["dio", "harvest", "crepe", "crepe_full", "crepe_tiny", "rmvpe", "rmvpe_onnx"],
"inputChunkNums": [8, 16, 24, 32, 40, 48, 64, 80, 96, 112, 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 2048] "inputChunkNums": [1, 2, 8, 16, 24, 32, 40, 48, 64, 80, 96, 112, 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 2048]
} }
} }
] ]

11
client/demo/dist/index.html vendored
View File

@ -1 +1,10 @@
<!doctype html><html style="width:100%;height:100%;overflow:hidden"><head><meta charset="utf-8"/><title>Voice Changer Client Demo</title><script defer="defer" src="index.js"></script></head><body style="width:100%;height:100%;margin:0"><div id="app" style="width:100%;height:100%"></div></body></html> <!DOCTYPE html>
<html style="width: 100%; height: 100%; overflow: hidden">
<head>
<meta charset="utf-8" />
<title>Voice Changer Client Demo</title>
<script defer src="index.js"></script></head>
<body style="width: 100%; height: 100%; margin: 0px">
<div id="app" style="width: 100%; height: 100%"></div>
</body>
</html>

1255
client/demo/dist/index.js vendored
View File

File diff suppressed because one or more lines are too long

31
client/demo/dist/index.js.LICENSE.txt vendored
View File

@ -1,31 +0,0 @@
/*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
/**
* @license React
* react-dom.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/**
* @license React
* react.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
/**
* @license React
* scheduler.production.min.js
*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/

2
client/demo/public/assets/gui_settings/GUI.json
View File

@ -22,7 +22,7 @@
"name": "configArea", "name": "configArea",
"options": { "options": {
"detectors": ["dio", "harvest", "crepe", "crepe_full", "crepe_tiny", "rmvpe", "rmvpe_onnx"], "detectors": ["dio", "harvest", "crepe", "crepe_full", "crepe_tiny", "rmvpe", "rmvpe_onnx"],
"inputChunkNums": [8, 16, 24, 32, 40, 48, 64, 80, 96, 112, 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 2048] "inputChunkNums": [1, 2, 8, 16, 24, 32, 40, 48, 64, 80, 96, 112, 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 2048]
} }
} }
] ]

44
client/demo/src/001_provider/001_AppStateProvider.tsx
View File

@ -10,9 +10,9 @@ type Props = {
}; };
type AppStateValue = ClientState & { type AppStateValue = ClientState & {
audioContext: AudioContext audioContext: AudioContext;
initializedRef: React.MutableRefObject<boolean> initializedRef: React.MutableRefObject<boolean>;
} };
const AppStateContext = React.createContext<AppStateValue | null>(null); const AppStateContext = React.createContext<AppStateValue | null>(null);
export const useAppState = (): AppStateValue => { export const useAppState = (): AppStateValue => {
@ -24,36 +24,44 @@ export const useAppState = (): AppStateValue => {
}; };
export const AppStateProvider = ({ children }: Props) => { export const AppStateProvider = ({ children }: Props) => {
const appRoot = useAppRoot() const appRoot = useAppRoot();
const clientState = useVCClient({ audioContext: appRoot.audioContextState.audioContext }) const clientState = useVCClient({ audioContext: appRoot.audioContextState.audioContext });
const messageBuilderState = useMessageBuilder() const messageBuilderState = useMessageBuilder();
useEffect(() => { useEffect(() => {
messageBuilderState.setMessage(__filename, "ioError", { messageBuilderState.setMessage(__filename, "ioError", {
"ja": "エラーが頻発しています。対象としているフレームワークのモデルがロードされているか確認してください。", ja: "エラーが頻発しています。対象としているフレームワークのモデルがロードされているか確認してください。",
"en": "Frequent errors occur. Please check if the model of the framework being targeted is loaded." en: "Frequent errors occur. Please check if the model of the framework being targeted is loaded.",
}) });
}, []) }, []);
const initializedRef = useRef<boolean>(false) const initializedRef = useRef<boolean>(false);
useEffect(() => { useEffect(() => {
if (clientState.clientState.initialized) { if (clientState.clientState.initialized) {
initializedRef.current = true initializedRef.current = true;
clientState.clientState.getInfo() clientState.clientState.getInfo();
// clientState.clientState.setVoiceChangerClientSetting({ // clientState.clientState.setVoiceChangerClientSetting({
// ...clientState.clientState.setting.voiceChangerClientSetting // ...clientState.clientState.setting.voiceChangerClientSetting
// }) // })
} }
}, [clientState.clientState.initialized]) }, [clientState.clientState.initialized]);
useEffect(() => { useEffect(() => {
if (clientState.clientState.ioErrorCount > 100) { if (clientState.clientState.ioErrorCount > 100) {
alert(messageBuilderState.getMessage(__filename, "ioError")) alert(messageBuilderState.getMessage(__filename, "ioError"));
clientState.clientState.resetIoErrorCount() clientState.clientState.resetIoErrorCount();
} }
}, [clientState.clientState.ioErrorCount]);
}, [clientState.clientState.ioErrorCount]) useEffect(() => {
if (clientState.clientState.initialized) {
clientState.clientState.setInternalAudioProcessCallback({
processAudio: (data: Uint8Array) => {
return data;
},
});
}
}, [clientState.clientState.initialized]);
const providerValue: AppStateValue = { const providerValue: AppStateValue = {
audioContext: appRoot.audioContextState.audioContext!, audioContext: appRoot.audioContextState.audioContext!,

5
client/lib/src/VoiceChangerClient.ts
View File

@ -1,4 +1,4 @@
import { VoiceChangerWorkletNode, VoiceChangerWorkletListener } from "./client/VoiceChangerWorkletNode"; import { VoiceChangerWorkletNode, VoiceChangerWorkletListener, InternalCallback } from "./client/VoiceChangerWorkletNode";
// @ts-ignore // @ts-ignore
import workerjs from "raw-loader!../worklet/dist/index.js"; import workerjs from "raw-loader!../worklet/dist/index.js";
import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js"; import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js";
@ -342,6 +342,9 @@ export class VoiceChangerClient {
this.vcInNode.updateSetting(setting); this.vcInNode.updateSetting(setting);
this.vcOutNode.updateSetting(setting); this.vcOutNode.updateSetting(setting);
}; };
setInternalAudioProcessCallback = (internalCallback: InternalCallback) => {
this.vcInNode.setInternalAudioProcessCallback(internalCallback);
};
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
// 情報取得 // 情報取得

30
client/lib/src/client/VoiceChangerWorkletNode.ts
View File

@ -11,6 +11,10 @@ export type VoiceChangerWorkletListener = {
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void; notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void;
}; };
export type InternalCallback = {
processAudio: (data: Uint8Array) => Uint8Array;
};
export class VoiceChangerWorkletNode extends AudioWorkletNode { export class VoiceChangerWorkletNode extends AudioWorkletNode {
private listener: VoiceChangerWorkletListener; private listener: VoiceChangerWorkletListener;
@ -28,6 +32,9 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
private startPromiseResolve: ((value: void | PromiseLike<void>) => void) | null = null; private startPromiseResolve: ((value: void | PromiseLike<void>) => void) | null = null;
private stopPromiseResolve: ((value: void | PromiseLike<void>) => void) | null = null; private stopPromiseResolve: ((value: void | PromiseLike<void>) => void) | null = null;
// InternalCallback
private internalCallback: InternalCallback | null = null;
constructor(context: AudioContext, listener: VoiceChangerWorkletListener) { constructor(context: AudioContext, listener: VoiceChangerWorkletListener) {
super(context, "voice-changer-worklet-processor"); super(context, "voice-changer-worklet-processor");
this.port.onmessage = this.handleMessage.bind(this); this.port.onmessage = this.handleMessage.bind(this);
@ -53,6 +60,10 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
} }
}; };
setInternalAudioProcessCallback = (internalCallback: InternalCallback) => {
this.internalCallback = internalCallback;
};
getSettings = (): WorkletNodeSetting => { getSettings = (): WorkletNodeSetting => {
return this.setting; return this.setting;
}; };
@ -192,7 +203,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
this.listener.notifyVolume(event.data.volume as number); this.listener.notifyVolume(event.data.volume as number);
} else if (event.data.responseType === "inputData") { } else if (event.data.responseType === "inputData") {
const inputData = event.data.inputData as Float32Array; const inputData = event.data.inputData as Float32Array;
// console.log("receive input data", inputData) // console.log("receive input data", inputData);
// ダウンサンプリング // ダウンサンプリング
let downsampledBuffer: Float32Array | null = null; let downsampledBuffer: Float32Array | null = null;
@ -261,10 +272,9 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
} }
// console.log("emit!") // console.log("emit!")
this.socket.emit("request_message", [timestamp, newBuffer.buffer]); this.socket.emit("request_message", [timestamp, newBuffer.buffer]);
} else { } else if (this.setting.protocol === "rest") {
const restClient = new ServerRestClient(this.setting.serverUrl); const restClient = new ServerRestClient(this.setting.serverUrl);
const res = await restClient.postVoice(timestamp, newBuffer.buffer); const res = await restClient.postVoice(timestamp, newBuffer.buffer);
if (res.byteLength < 128 * 2) { if (res.byteLength < 128 * 2) {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`); this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`);
} else { } else {
@ -275,9 +285,23 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
} }
this.listener.notifyResponseTime(Date.now() - timestamp); this.listener.notifyResponseTime(Date.now() - timestamp);
} }
} else if (this.setting.protocol == "internal") {
if (!this.internalCallback) {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_INTERNAL_AUDIO_PROCESS_CALLBACK_IS_NOT_INITIALIZED, `[AudioWorkletNode] internal audio process callback is not initialized`);
return;
}
const res = this.internalCallback.processAudio(newBuffer);
if (this.outputNode != null) {
this.outputNode.postReceivedVoice(res.buffer);
} else {
this.postReceivedVoice(res.buffer);
}
} else {
throw "unknown protocol";
} }
}; };
// Worklet操作
configure = (setting: WorkletSetting) => { configure = (setting: WorkletSetting) => {
const req: VoiceChangerWorkletProcessorRequest = { const req: VoiceChangerWorkletProcessorRequest = {
requestType: "config", requestType: "config",

2
client/lib/src/const.ts
View File

@ -436,6 +436,7 @@ export type WorkletSetting = {
export const Protocol = { export const Protocol = {
sio: "sio", sio: "sio",
rest: "rest", rest: "rest",
internal: "internal",
} as const; } as const;
export type Protocol = (typeof Protocol)[keyof typeof Protocol]; export type Protocol = (typeof Protocol)[keyof typeof Protocol];
@ -524,6 +525,7 @@ export const VOICE_CHANGER_CLIENT_EXCEPTION = {
ERR_SIO_INVALID_RESPONSE: "ERR_SIO_INVALID_RESPONSE", ERR_SIO_INVALID_RESPONSE: "ERR_SIO_INVALID_RESPONSE",
ERR_REST_INVALID_RESPONSE: "ERR_REST_INVALID_RESPONSE", ERR_REST_INVALID_RESPONSE: "ERR_REST_INVALID_RESPONSE",
ERR_MIC_STREAM_NOT_INITIALIZED: "ERR_MIC_STREAM_NOT_INITIALIZED", ERR_MIC_STREAM_NOT_INITIALIZED: "ERR_MIC_STREAM_NOT_INITIALIZED",
ERR_INTERNAL_AUDIO_PROCESS_CALLBACK_IS_NOT_INITIALIZED: "ERR_INTERNAL_AUDIO_PROCESS_CALLBACK_IS_NOT_INITIALIZED",
} as const; } as const;
export type VOICE_CHANGER_CLIENT_EXCEPTION = (typeof VOICE_CHANGER_CLIENT_EXCEPTION)[keyof typeof VOICE_CHANGER_CLIENT_EXCEPTION]; export type VOICE_CHANGER_CLIENT_EXCEPTION = (typeof VOICE_CHANGER_CLIENT_EXCEPTION)[keyof typeof VOICE_CHANGER_CLIENT_EXCEPTION];

3
client/lib/src/hooks/useClient.ts
View File

@ -6,6 +6,7 @@ import { ServerSettingState, useServerSetting } from "./useServerSetting";
import { useWorkletNodeSetting } from "./useWorkletNodeSetting"; import { useWorkletNodeSetting } from "./useWorkletNodeSetting";
import { useWorkletSetting } from "./useWorkletSetting"; import { useWorkletSetting } from "./useWorkletSetting";
import { ClientSetting, DefaultClientSettng, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "../const"; import { ClientSetting, DefaultClientSettng, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "../const";
import { InternalCallback } from "../client/VoiceChangerWorkletNode";
export type UseClientProps = { export type UseClientProps = {
audioContext: AudioContext | null; audioContext: AudioContext | null;
@ -25,6 +26,7 @@ export type ClientState = {
startOutputRecording: () => void; startOutputRecording: () => void;
stopOutputRecording: () => Promise<Float32Array>; stopOutputRecording: () => Promise<Float32Array>;
trancateBuffer: () => Promise<void>; trancateBuffer: () => Promise<void>;
setInternalAudioProcessCallback: (internalCallback: InternalCallback) => Promise<void>;
setWorkletSetting: (_workletSetting: WorkletSetting) => void; setWorkletSetting: (_workletSetting: WorkletSetting) => void;
// workletSetting: WorkletSetting // workletSetting: WorkletSetting
@ -268,6 +270,7 @@ export const useClient = (props: UseClientProps): ClientState => {
startOutputRecording: workletNodeSetting.startOutputRecording, startOutputRecording: workletNodeSetting.startOutputRecording,
stopOutputRecording: workletNodeSetting.stopOutputRecording, stopOutputRecording: workletNodeSetting.stopOutputRecording,
trancateBuffer: workletNodeSetting.trancateBuffer, trancateBuffer: workletNodeSetting.trancateBuffer,
setInternalAudioProcessCallback: workletNodeSetting.setInternalAudioProcessCallback,
setWorkletSetting, setWorkletSetting,
// workletSetting: workletSettingIF.setting, // workletSetting: workletSettingIF.setting,

10
client/lib/src/hooks/useWorkletNodeSetting.ts
View File

@ -2,6 +2,7 @@ import { useState, useMemo, useEffect } from "react";
import { WorkletNodeSetting } from "../const"; import { WorkletNodeSetting } from "../const";
import { VoiceChangerClient } from "../VoiceChangerClient"; import { VoiceChangerClient } from "../VoiceChangerClient";
import { InternalCallback } from "../client/VoiceChangerWorkletNode";
export type UseWorkletNodeSettingProps = { export type UseWorkletNodeSettingProps = {
voiceChangerClient: VoiceChangerClient | null; voiceChangerClient: VoiceChangerClient | null;
@ -12,6 +13,7 @@ export type WorkletNodeSettingState = {
startOutputRecording: () => void; startOutputRecording: () => void;
stopOutputRecording: () => Promise<Float32Array>; stopOutputRecording: () => Promise<Float32Array>;
trancateBuffer: () => Promise<void>; trancateBuffer: () => Promise<void>;
setInternalAudioProcessCallback: (internalCallback: InternalCallback) => Promise<void>;
}; };
export const useWorkletNodeSetting = (props: UseWorkletNodeSettingProps): WorkletNodeSettingState => { export const useWorkletNodeSetting = (props: UseWorkletNodeSettingProps): WorkletNodeSettingState => {
@ -55,9 +57,17 @@ export const useWorkletNodeSetting = (props: UseWorkletNodeSettingProps): Workle
}; };
}, [props.voiceChangerClient]); }, [props.voiceChangerClient]);
const setInternalAudioProcessCallback = useMemo(() => {
return async (internalCallback: InternalCallback) => {
if (!props.voiceChangerClient) return;
props.voiceChangerClient.setInternalAudioProcessCallback(internalCallback);
};
}, [props.voiceChangerClient]);
return { return {
startOutputRecording, startOutputRecording,
stopOutputRecording, stopOutputRecording,
trancateBuffer, trancateBuffer,
setInternalAudioProcessCallback,
}; };
}; };