PublicMirrors/voice-changer
1
0
Fork 0
mirror of https://github.com/w-okada/voice-changer.git synced 2025-01-23 13:35:12 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

apply code formatter

Browse Source
This commit is contained in:
w-okada 2023-09-25 13:25:07 +09:00
parent acc44fb83f
commit 3ef54e979b
26 changed files with 1396 additions and 1484 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
client/lib/.vscode/settings.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"workbench.colorCustomizations": {
"tab.activeBackground": "#65952acc"
},
"editor.defaultFormatter": "esbenp.prettier-vscode",
"prettier.printWidth": 1024,
"prettier.tabWidth": 4
}

112
client/lib/package.json
View File

@ -1,58 +1,58 @@
{ {
"name": "@dannadori/voice-changer-client-js", "name": "@dannadori/voice-changer-client-js",
"version": "1.0.172", "version": "1.0.172",
"description": "", "description": "",
"main": "dist/index.js", "main": "dist/index.js",
"directories": { "directories": {
"lib": "lib" "lib": "lib"
}, },
"scripts": { "scripts": {
"clean:worklet": "rimraf worklet/dist/", "clean:worklet": "rimraf worklet/dist/",
"webpack:worklet:dev": "webpack --config webpack.worklet.dev.js", "webpack:worklet:dev": "webpack --config webpack.worklet.dev.js",
"webpack:worklet:prod": "webpack --config webpack.worklet.prod.js", "webpack:worklet:prod": "webpack --config webpack.worklet.prod.js",
"build:worklet:dev": "npm-run-all clean:worklet webpack:worklet:dev", "build:worklet:dev": "npm-run-all clean:worklet webpack:worklet:dev",
"build:worklet:prod": "npm-run-all clean:worklet webpack:worklet:prod", "build:worklet:prod": "npm-run-all clean:worklet webpack:worklet:prod",
"clean": "rimraf dist/", "clean": "rimraf dist/",
"webpack:dev": "webpack --config webpack.dev.js", "webpack:dev": "webpack --config webpack.dev.js",
"webpack:prod": "webpack --config webpack.prod.js", "webpack:prod": "webpack --config webpack.prod.js",
"build:dev": "npm-run-all build:worklet:dev clean webpack:dev", "build:dev": "npm-run-all build:worklet:dev clean webpack:dev",
"build:prod": "npm-run-all build:worklet:prod clean webpack:prod", "build:prod": "npm-run-all build:worklet:prod clean webpack:prod",
"release": "npm version patch && npm publish --access=public", "release": "npm version patch && npm publish --access=public",
"test": "echo \"Error: no test specified\" && exit 1" "test": "echo \"Error: no test specified\" && exit 1"
}, },
"keywords": [ "keywords": [
"voice conversion" "voice conversion"
], ],
"author": "wataru.okada@flect.co.jp", "author": "wataru.okada@flect.co.jp",
"license": "ISC", "license": "ISC",
"devDependencies": { "devDependencies": {
"@types/audioworklet": "^0.0.50", "@types/audioworklet": "^0.0.50",
"@types/node": "^20.6.3", "@types/node": "^20.6.3",
"@types/react": "18.2.22", "@types/react": "18.2.22",
"@types/react-dom": "18.2.7", "@types/react-dom": "18.2.7",
"eslint": "^8.49.0", "eslint": "^8.49.0",
"eslint-config-prettier": "^9.0.0", "eslint-config-prettier": "^9.0.0",
"eslint-plugin-prettier": "^5.0.0", "eslint-plugin-prettier": "^5.0.0",
"eslint-plugin-react": "^7.33.2", "eslint-plugin-react": "^7.33.2",
"eslint-webpack-plugin": "^4.0.1", "eslint-webpack-plugin": "^4.0.1",
"npm-run-all": "^4.1.5", "npm-run-all": "^4.1.5",
"prettier": "^3.0.3", "prettier": "^3.0.3",
"raw-loader": "^4.0.2", "raw-loader": "^4.0.2",
"rimraf": "^5.0.1", "rimraf": "^5.0.1",
"ts-loader": "^9.4.4", "ts-loader": "^9.4.4",
"typescript": "^5.2.2", "typescript": "^5.2.2",
"webpack": "^5.88.2", "webpack": "^5.88.2",
"webpack-cli": "^5.1.4", "webpack-cli": "^5.1.4",
"webpack-dev-server": "^4.15.1" "webpack-dev-server": "^4.15.1"
}, },
"dependencies": { "dependencies": {
"@types/readable-stream": "^4.0.2", "@types/readable-stream": "^4.0.2",
"amazon-chime-sdk-js": "^3.17.0", "amazon-chime-sdk-js": "^3.17.0",
"buffer": "^6.0.3", "buffer": "^6.0.3",
"localforage": "^1.10.0", "localforage": "^1.10.0",
"protobufjs": "^7.2.5", "protobufjs": "^7.2.5",
"react": "^18.2.0", "react": "^18.2.0",
"react-dom": "^18.2.0", "react-dom": "^18.2.0",
"socket.io-client": "^4.7.2" "socket.io-client": "^4.7.2"
} }
} }

4
client/lib/src/@types/voice-changer-worklet-processor.d.ts vendored
View File

@ -5,14 +5,14 @@ export declare const RequestType: {
readonly stop: "stop"; readonly stop: "stop";
readonly trancateBuffer: "trancateBuffer"; readonly trancateBuffer: "trancateBuffer";
}; };
export type RequestType = typeof RequestType[keyof typeof RequestType]; export type RequestType = (typeof RequestType)[keyof typeof RequestType];
export declare const ResponseType: { export declare const ResponseType: {
readonly volume: "volume"; readonly volume: "volume";
readonly inputData: "inputData"; readonly inputData: "inputData";
readonly start_ok: "start_ok"; readonly start_ok: "start_ok";
readonly stop_ok: "stop_ok"; readonly stop_ok: "stop_ok";
}; };
export type ResponseType = typeof ResponseType[keyof typeof ResponseType]; export type ResponseType = (typeof ResponseType)[keyof typeof ResponseType];
export type VoiceChangerWorkletProcessorRequest = { export type VoiceChangerWorkletProcessorRequest = {
requestType: RequestType; requestType: RequestType;
voice: Float32Array; voice: Float32Array;

351
client/lib/src/VoiceChangerClient.ts
View File

@ -1,92 +1,89 @@
import { VoiceChangerWorkletNode, VoiceChangerWorkletListener } from "./VoiceChangerWorkletNode"; import { VoiceChangerWorkletNode, VoiceChangerWorkletListener } from "./client/VoiceChangerWorkletNode";
// @ts-ignore // @ts-ignore
import workerjs from "raw-loader!../worklet/dist/index.js"; import workerjs from "raw-loader!../worklet/dist/index.js";
import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js"; import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js";
import { createDummyMediaStream, validateUrl } from "./util"; import { createDummyMediaStream, validateUrl } from "./util";
import { DefaultClientSettng, MergeModelRequest, ServerSettingKey, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "./const"; import { DefaultClientSettng, MergeModelRequest, ServerSettingKey, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "./const";
import { ServerConfigurator } from "./ServerConfigurator"; import { ServerConfigurator } from "./client/ServerConfigurator";
// オーディオデータの流れ // オーディオデータの流れ
// input node(mic or MediaStream) -> [vf node] -> [vc node] -> // input node(mic or MediaStream) -> [vf node] -> [vc node] ->
// sio/rest server -> [vc node] -> output node // sio/rest server -> [vc node] -> output node
import { BlockingQueue } from "./utils/BlockingQueue"; import { BlockingQueue } from "./utils/BlockingQueue";
export class VoiceChangerClient { export class VoiceChangerClient {
private configurator: ServerConfigurator private configurator: ServerConfigurator;
private ctx: AudioContext private ctx: AudioContext;
private vfEnable = false private vfEnable = false;
private vf: VoiceFocusDeviceTransformer | null = null private vf: VoiceFocusDeviceTransformer | null = null;
private currentDevice: VoiceFocusTransformDevice | null = null private currentDevice: VoiceFocusTransformDevice | null = null;
private currentMediaStream: MediaStream | null = null private currentMediaStream: MediaStream | null = null;
private currentMediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null private currentMediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null;
private inputGainNode: GainNode | null = null private inputGainNode: GainNode | null = null;
private outputGainNode: GainNode | null = null private outputGainNode: GainNode | null = null;
private monitorGainNode: GainNode | null = null private monitorGainNode: GainNode | null = null;
private vcInNode!: VoiceChangerWorkletNode private vcInNode!: VoiceChangerWorkletNode;
private vcOutNode!: VoiceChangerWorkletNode private vcOutNode!: VoiceChangerWorkletNode;
private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode;
private currentMediaStreamAudioDestinationMonitorNode!: MediaStreamAudioDestinationNode private currentMediaStreamAudioDestinationMonitorNode!: MediaStreamAudioDestinationNode;
private promiseForInitialize: Promise<void>;
private _isVoiceChanging = false;
private promiseForInitialize: Promise<void> private setting: VoiceChangerClientSetting = DefaultClientSettng.voiceChangerClientSetting;
private _isVoiceChanging = false
private setting: VoiceChangerClientSetting = DefaultClientSettng.voiceChangerClientSetting private sslCertified: string[] = [];
private sslCertified: string[] = []
private sem = new BlockingQueue<number>(); private sem = new BlockingQueue<number>();
constructor(ctx: AudioContext, vfEnable: boolean, voiceChangerWorkletListener: VoiceChangerWorkletListener) { constructor(ctx: AudioContext, vfEnable: boolean, voiceChangerWorkletListener: VoiceChangerWorkletListener) {
this.sem.enqueue(0); this.sem.enqueue(0);
this.configurator = new ServerConfigurator() this.configurator = new ServerConfigurator();
this.ctx = ctx this.ctx = ctx;
this.vfEnable = vfEnable this.vfEnable = vfEnable;
this.promiseForInitialize = new Promise<void>(async (resolve) => { this.promiseForInitialize = new Promise<void>(async (resolve) => {
const scriptUrl = URL.createObjectURL(new Blob([workerjs], { type: "text/javascript" })); const scriptUrl = URL.createObjectURL(new Blob([workerjs], { type: "text/javascript" }));
// await this.ctx.audioWorklet.addModule(scriptUrl) // await this.ctx.audioWorklet.addModule(scriptUrl)
// this.vcInNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node // this.vcInNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node
try { try {
this.vcInNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node this.vcInNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node
} catch (err) { } catch (err) {
await this.ctx.audioWorklet.addModule(scriptUrl) await this.ctx.audioWorklet.addModule(scriptUrl);
this.vcInNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node this.vcInNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node
} }
// const ctx44k = new AudioContext({ sampleRate: 44100 }) // これでもプチプチが残る // const ctx44k = new AudioContext({ sampleRate: 44100 }) // これでもプチプチが残る
const ctx44k = new AudioContext({ sampleRate: 48000 }) // 結局これが一番まし。 const ctx44k = new AudioContext({ sampleRate: 48000 }); // 結局これが一番まし。
console.log("audio out:", ctx44k) console.log("audio out:", ctx44k);
try { try {
this.vcOutNode = new VoiceChangerWorkletNode(ctx44k, voiceChangerWorkletListener); // vc node this.vcOutNode = new VoiceChangerWorkletNode(ctx44k, voiceChangerWorkletListener); // vc node
} catch (err) { } catch (err) {
await ctx44k.audioWorklet.addModule(scriptUrl) await ctx44k.audioWorklet.addModule(scriptUrl);
this.vcOutNode = new VoiceChangerWorkletNode(ctx44k, voiceChangerWorkletListener); // vc node this.vcOutNode = new VoiceChangerWorkletNode(ctx44k, voiceChangerWorkletListener); // vc node
} }
this.currentMediaStreamAudioDestinationNode = ctx44k.createMediaStreamDestination() // output node this.currentMediaStreamAudioDestinationNode = ctx44k.createMediaStreamDestination(); // output node
this.outputGainNode = ctx44k.createGain() this.outputGainNode = ctx44k.createGain();
this.outputGainNode.gain.value = this.setting.outputGain this.outputGainNode.gain.value = this.setting.outputGain;
this.vcOutNode.connect(this.outputGainNode) // vc node -> output node this.vcOutNode.connect(this.outputGainNode); // vc node -> output node
this.outputGainNode.connect(this.currentMediaStreamAudioDestinationNode) this.outputGainNode.connect(this.currentMediaStreamAudioDestinationNode);
this.currentMediaStreamAudioDestinationMonitorNode = ctx44k.createMediaStreamDestination() // output node this.currentMediaStreamAudioDestinationMonitorNode = ctx44k.createMediaStreamDestination(); // output node
this.monitorGainNode = ctx44k.createGain() this.monitorGainNode = ctx44k.createGain();
this.monitorGainNode.gain.value = this.setting.monitorGain this.monitorGainNode.gain.value = this.setting.monitorGain;
this.vcOutNode.connect(this.monitorGainNode) // vc node -> monitor node this.vcOutNode.connect(this.monitorGainNode); // vc node -> monitor node
this.monitorGainNode.connect(this.currentMediaStreamAudioDestinationMonitorNode) this.monitorGainNode.connect(this.currentMediaStreamAudioDestinationMonitorNode);
if (this.vfEnable) { if (this.vfEnable) {
this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' }) this.vf = await VoiceFocusDeviceTransformer.create({ variant: "c20" });
const dummyMediaStream = createDummyMediaStream(this.ctx) const dummyMediaStream = createDummyMediaStream(this.ctx);
this.currentDevice = (await this.vf.createTransformDevice(dummyMediaStream)) || null; this.currentDevice = (await this.vf.createTransformDevice(dummyMediaStream)) || null;
} }
resolve() resolve();
}) });
} }
private lock = async () => { private lock = async () => {
@ -99,44 +96,46 @@ export class VoiceChangerClient {
isInitialized = async () => { isInitialized = async () => {
if (this.promiseForInitialize) { if (this.promiseForInitialize) {
await this.promiseForInitialize await this.promiseForInitialize;
} }
return true return true;
} };
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
// オペレーション // オペレーション
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
/// Operations /// /// Operations ///
setup = async () => { setup = async () => {
const lockNum = await this.lock() const lockNum = await this.lock();
console.log(`Input Setup=> echo: ${this.setting.echoCancel}, noise1: ${this.setting.noiseSuppression}, noise2: ${this.setting.noiseSuppression2}`) console.log(`Input Setup=> echo: ${this.setting.echoCancel}, noise1: ${this.setting.noiseSuppression}, noise2: ${this.setting.noiseSuppression2}`);
// condition check // condition check
if (!this.vcInNode) { if (!this.vcInNode) {
console.warn("vc node is not initialized.") console.warn("vc node is not initialized.");
throw "vc node is not initialized." throw "vc node is not initialized.";
} }
// Main Process // Main Process
//// shutdown & re-generate mediastream //// shutdown & re-generate mediastream
if (this.currentMediaStream) { if (this.currentMediaStream) {
this.currentMediaStream.getTracks().forEach(x => { x.stop() }) this.currentMediaStream.getTracks().forEach((x) => {
this.currentMediaStream = null x.stop();
});
this.currentMediaStream = null;
} }
//// Input デバイスがnullの時はmicStreamを止めてリターン //// Input デバイスがnullの時はmicStreamを止めてリターン
if (!this.setting.audioInput) { if (!this.setting.audioInput) {
console.log(`Input Setup=> client mic is disabled. ${this.setting.audioInput}`) console.log(`Input Setup=> client mic is disabled. ${this.setting.audioInput}`);
this.vcInNode.stop() this.vcInNode.stop();
await this.unlock(lockNum) await this.unlock(lockNum);
return return;
} }
if (typeof this.setting.audioInput == "string") { if (typeof this.setting.audioInput == "string") {
try { try {
if (this.setting.audioInput == "none") { if (this.setting.audioInput == "none") {
this.currentMediaStream = createDummyMediaStream(this.ctx) this.currentMediaStream = createDummyMediaStream(this.ctx);
} else { } else {
this.currentMediaStream = await navigator.mediaDevices.getUserMedia({ this.currentMediaStream = await navigator.mediaDevices.getUserMedia({
audio: { audio: {
@ -146,15 +145,15 @@ export class VoiceChangerClient {
sampleSize: 16, sampleSize: 16,
autoGainControl: false, autoGainControl: false,
echoCancellation: this.setting.echoCancel, echoCancellation: this.setting.echoCancel,
noiseSuppression: this.setting.noiseSuppression noiseSuppression: this.setting.noiseSuppression,
} },
}) });
} }
} catch (e) { } catch (e) {
console.warn(e) console.warn(e);
this.vcInNode.stop() this.vcInNode.stop();
await this.unlock(lockNum) await this.unlock(lockNum);
throw e throw e;
} }
// this.currentMediaStream.getAudioTracks().forEach((x) => { // this.currentMediaStream.getAudioTracks().forEach((x) => {
// console.log("MIC Setting(cap)", x.getCapabilities()) // console.log("MIC Setting(cap)", x.getCapabilities())
@ -162,19 +161,19 @@ export class VoiceChangerClient {
// console.log("MIC Setting(setting)", x.getSettings()) // console.log("MIC Setting(setting)", x.getSettings())
// }) // })
} else { } else {
this.currentMediaStream = this.setting.audioInput this.currentMediaStream = this.setting.audioInput;
} }
// connect nodes. // connect nodes.
this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream) this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream);
this.inputGainNode = this.ctx.createGain() this.inputGainNode = this.ctx.createGain();
this.inputGainNode.gain.value = this.setting.inputGain this.inputGainNode.gain.value = this.setting.inputGain;
this.currentMediaStreamAudioSourceNode.connect(this.inputGainNode) this.currentMediaStreamAudioSourceNode.connect(this.inputGainNode);
if (this.currentDevice && this.setting.noiseSuppression2) { if (this.currentDevice && this.setting.noiseSuppression2) {
this.currentDevice.chooseNewInnerDevice(this.currentMediaStream) this.currentDevice.chooseNewInnerDevice(this.currentMediaStream);
const voiceFocusNode = await this.currentDevice.createAudioNode(this.ctx); // vf node const voiceFocusNode = await this.currentDevice.createAudioNode(this.ctx); // vf node
this.inputGainNode.connect(voiceFocusNode.start) // input node -> vf node this.inputGainNode.connect(voiceFocusNode.start); // input node -> vf node
voiceFocusNode.end.connect(this.vcInNode) voiceFocusNode.end.connect(this.vcInNode);
} else { } else {
// console.log("input___ media stream", this.currentMediaStream) // console.log("input___ media stream", this.currentMediaStream)
// this.currentMediaStream.getTracks().forEach(x => { // this.currentMediaStream.getTracks().forEach(x => {
@ -184,193 +183,181 @@ export class VoiceChangerClient {
// }) // })
// console.log("input___ media node", this.currentMediaStreamAudioSourceNode) // console.log("input___ media node", this.currentMediaStreamAudioSourceNode)
// console.log("input___ gain node", this.inputGainNode.channelCount, this.inputGainNode) // console.log("input___ gain node", this.inputGainNode.channelCount, this.inputGainNode)
this.inputGainNode.connect(this.vcInNode) this.inputGainNode.connect(this.vcInNode);
} }
this.vcInNode.setOutputNode(this.vcOutNode) this.vcInNode.setOutputNode(this.vcOutNode);
console.log("Input Setup=> success") console.log("Input Setup=> success");
await this.unlock(lockNum) await this.unlock(lockNum);
} };
get stream(): MediaStream { get stream(): MediaStream {
return this.currentMediaStreamAudioDestinationNode.stream return this.currentMediaStreamAudioDestinationNode.stream;
} }
get monitorStream(): MediaStream { get monitorStream(): MediaStream {
return this.currentMediaStreamAudioDestinationMonitorNode.stream return this.currentMediaStreamAudioDestinationMonitorNode.stream;
} }
start = async () => { start = async () => {
await this.vcInNode.start() await this.vcInNode.start();
this._isVoiceChanging = true this._isVoiceChanging = true;
} };
stop = async () => { stop = async () => {
await this.vcInNode.stop() await this.vcInNode.stop();
this._isVoiceChanging = false this._isVoiceChanging = false;
} };
get isVoiceChanging(): boolean { get isVoiceChanging(): boolean {
return this._isVoiceChanging return this._isVoiceChanging;
} }
//////////////////////// ////////////////////////
/// 設定 /// 設定
////////////////////////////// //////////////////////////////
setServerUrl = (serverUrl: string, openTab: boolean = false) => { setServerUrl = (serverUrl: string, openTab: boolean = false) => {
const url = validateUrl(serverUrl) const url = validateUrl(serverUrl);
const pageUrl = `${location.protocol}//${location.host}` const pageUrl = `${location.protocol}//${location.host}`;
if (url != pageUrl && url.length != 0 && location.protocol == "https:" && this.sslCertified.includes(url) == false) { if (url != pageUrl && url.length != 0 && location.protocol == "https:" && this.sslCertified.includes(url) == false) {
if (openTab) { if (openTab) {
const value = window.confirm("MMVC Server is different from this page's origin. Open tab to open ssl connection. OK? (You can close the opened tab after ssl connection succeed.)"); const value = window.confirm("MMVC Server is different from this page's origin. Open tab to open ssl connection. OK? (You can close the opened tab after ssl connection succeed.)");
if (value) { if (value) {
window.open(url, '_blank') window.open(url, "_blank");
this.sslCertified.push(url) this.sslCertified.push(url);
} else { } else {
alert("Your voice conversion may fail...") alert("Your voice conversion may fail...");
} }
} }
} }
this.vcInNode.updateSetting({ ...this.vcInNode.getSettings(), serverUrl: url }) this.vcInNode.updateSetting({ ...this.vcInNode.getSettings(), serverUrl: url });
this.configurator.setServerUrl(url) this.configurator.setServerUrl(url);
} };
updateClientSetting = async (setting: VoiceChangerClientSetting) => { updateClientSetting = async (setting: VoiceChangerClientSetting) => {
let reconstructInputRequired = false let reconstructInputRequired = false;
if ( if (this.setting.audioInput != setting.audioInput || this.setting.echoCancel != setting.echoCancel || this.setting.noiseSuppression != setting.noiseSuppression || this.setting.noiseSuppression2 != setting.noiseSuppression2 || this.setting.sampleRate != setting.sampleRate) {
this.setting.audioInput != setting.audioInput || reconstructInputRequired = true;
this.setting.echoCancel != setting.echoCancel ||
this.setting.noiseSuppression != setting.noiseSuppression ||
this.setting.noiseSuppression2 != setting.noiseSuppression2 ||
this.setting.sampleRate != setting.sampleRate
) {
reconstructInputRequired = true
} }
if (this.setting.inputGain != setting.inputGain) { if (this.setting.inputGain != setting.inputGain) {
this.setInputGain(setting.inputGain) this.setInputGain(setting.inputGain);
} }
if (this.setting.outputGain != setting.outputGain) { if (this.setting.outputGain != setting.outputGain) {
this.setOutputGain(setting.outputGain) this.setOutputGain(setting.outputGain);
} }
if (this.setting.monitorGain != setting.monitorGain) { if (this.setting.monitorGain != setting.monitorGain) {
this.setMonitorGain(setting.monitorGain) this.setMonitorGain(setting.monitorGain);
} }
this.setting = setting this.setting = setting;
if (reconstructInputRequired) { if (reconstructInputRequired) {
await this.setup() await this.setup();
} }
} };
setInputGain = (val: number) => { setInputGain = (val: number) => {
this.setting.inputGain = val this.setting.inputGain = val;
if (!this.inputGainNode) { if (!this.inputGainNode) {
return return;
} }
if(!val){ if (!val) {
return return;
} }
this.inputGainNode.gain.value = val this.inputGainNode.gain.value = val;
} };
setOutputGain = (val: number) => { setOutputGain = (val: number) => {
if (!this.outputGainNode) { if (!this.outputGainNode) {
return return;
} }
if(!val){ if (!val) {
return return;
} }
this.outputGainNode.gain.value = val this.outputGainNode.gain.value = val;
} };
setMonitorGain = (val: number) => { setMonitorGain = (val: number) => {
if (!this.monitorGainNode) { if (!this.monitorGainNode) {
return return;
} }
if(!val){ if (!val) {
return return;
} }
this.monitorGainNode.gain.value = val this.monitorGainNode.gain.value = val;
} };
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
// コンポーネント設定、操作 // コンポーネント設定、操作
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
//## Server ##// //## Server ##//
getModelType = () => { getModelType = () => {
return this.configurator.getModelType() return this.configurator.getModelType();
} };
getOnnx = async () => { getOnnx = async () => {
return this.configurator.export2onnx() return this.configurator.export2onnx();
} };
mergeModel = async (req: MergeModelRequest) => { mergeModel = async (req: MergeModelRequest) => {
return this.configurator.mergeModel(req) return this.configurator.mergeModel(req);
} };
updateModelDefault = async () => { updateModelDefault = async () => {
return this.configurator.updateModelDefault() return this.configurator.updateModelDefault();
} };
updateModelInfo = async (slot: number, key: string, val: string) => { updateModelInfo = async (slot: number, key: string, val: string) => {
return this.configurator.updateModelInfo(slot, key, val) return this.configurator.updateModelInfo(slot, key, val);
} };
updateServerSettings = (key: ServerSettingKey, val: string) => { updateServerSettings = (key: ServerSettingKey, val: string) => {
return this.configurator.updateSettings(key, val) return this.configurator.updateSettings(key, val);
} };
uploadFile = (buf: ArrayBuffer, filename: string, onprogress: (progress: number, end: boolean) => void) => { uploadFile = (buf: ArrayBuffer, filename: string, onprogress: (progress: number, end: boolean) => void) => {
return this.configurator.uploadFile(buf, filename, onprogress) return this.configurator.uploadFile(buf, filename, onprogress);
} };
uploadFile2 = (dir: string, file: File, onprogress: (progress: number, end: boolean) => void) => { uploadFile2 = (dir: string, file: File, onprogress: (progress: number, end: boolean) => void) => {
return this.configurator.uploadFile2(dir, file, onprogress) return this.configurator.uploadFile2(dir, file, onprogress);
} };
concatUploadedFile = (filename: string, chunkNum: number) => { concatUploadedFile = (filename: string, chunkNum: number) => {
return this.configurator.concatUploadedFile(filename, chunkNum) return this.configurator.concatUploadedFile(filename, chunkNum);
} };
loadModel = ( loadModel = (slot: number, isHalf: boolean, params: string) => {
slot: number, return this.configurator.loadModel(slot, isHalf, params);
isHalf: boolean, };
params: string,
) => {
return this.configurator.loadModel(slot, isHalf, params)
}
uploadAssets = (params: string) => { uploadAssets = (params: string) => {
return this.configurator.uploadAssets(params) return this.configurator.uploadAssets(params);
} };
//## Worklet ##// //## Worklet ##//
configureWorklet = (setting: WorkletSetting) => { configureWorklet = (setting: WorkletSetting) => {
this.vcInNode.configure(setting) this.vcInNode.configure(setting);
this.vcOutNode.configure(setting) this.vcOutNode.configure(setting);
} };
startOutputRecording = () => { startOutputRecording = () => {
this.vcOutNode.startOutputRecording() this.vcOutNode.startOutputRecording();
} };
stopOutputRecording = () => { stopOutputRecording = () => {
return this.vcOutNode.stopOutputRecording() return this.vcOutNode.stopOutputRecording();
} };
trancateBuffer = () => { trancateBuffer = () => {
this.vcOutNode.trancateBuffer() this.vcOutNode.trancateBuffer();
} };
//## Worklet Node ##// //## Worklet Node ##//
updateWorkletNodeSetting = (setting: WorkletNodeSetting) => { updateWorkletNodeSetting = (setting: WorkletNodeSetting) => {
this.vcInNode.updateSetting(setting) this.vcInNode.updateSetting(setting);
this.vcOutNode.updateSetting(setting) this.vcOutNode.updateSetting(setting);
} };
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
// 情報取得 // 情報取得
///////////////////////////////////////////////////// /////////////////////////////////////////////////////
// Information // Information
getClientSettings = () => { getClientSettings = () => {
return this.vcInNode.getSettings() return this.vcInNode.getSettings();
} };
getServerSettings = () => { getServerSettings = () => {
return this.configurator.getSettings() return this.configurator.getSettings();
} };
getPerformance = () => { getPerformance = () => {
return this.configurator.getPerformance() return this.configurator.getPerformance();
} };
getSocketId = () => { getSocketId = () => {
return this.vcInNode.getSocketId() return this.vcInNode.getSocketId();
} };
}
}

259
client/lib/src/ServerConfigurator.ts → client/lib/src/client/ServerConfigurator.ts
View File

@ -1,108 +1,107 @@
import { MergeModelRequest, OnnxExporterInfo, ServerInfo, ServerSettingKey } from "./const"; import { MergeModelRequest, OnnxExporterInfo, ServerInfo, ServerSettingKey } from "../const";
type FileChunk = { type FileChunk = {
hash: number, hash: number;
chunk: ArrayBuffer chunk: ArrayBuffer;
} };
export class ServerConfigurator { export class ServerConfigurator {
private serverUrl = "" private serverUrl = "";
setServerUrl = (serverUrl: string) => { setServerUrl = (serverUrl: string) => {
this.serverUrl = serverUrl this.serverUrl = serverUrl;
console.log(`[ServerConfigurator] Server URL: ${this.serverUrl}`) console.log(`[ServerConfigurator] Server URL: ${this.serverUrl}`);
} };
getSettings = async () => { getSettings = async () => {
const url = this.serverUrl + "/info" const url = this.serverUrl + "/info";
const info = await new Promise<ServerInfo>((resolve) => { const info = await new Promise<ServerInfo>((resolve) => {
const request = new Request(url, { const request = new Request(url, {
method: 'GET', method: "GET",
}); });
fetch(request).then(async (response) => { fetch(request).then(async (response) => {
const json = await response.json() as ServerInfo const json = (await response.json()) as ServerInfo;
resolve(json) resolve(json);
}) });
}) });
return info return info;
} };
getPerformance = async () => { getPerformance = async () => {
const url = this.serverUrl + "/performance" const url = this.serverUrl + "/performance";
const info = await new Promise<number[]>((resolve) => { const info = await new Promise<number[]>((resolve) => {
const request = new Request(url, { const request = new Request(url, {
method: 'GET', method: "GET",
}); });
fetch(request).then(async (response) => { fetch(request).then(async (response) => {
const json = await response.json() as number[] const json = (await response.json()) as number[];
resolve(json) resolve(json);
}) });
}) });
return info return info;
} };
updateSettings = async (key: ServerSettingKey, val: string) => { updateSettings = async (key: ServerSettingKey, val: string) => {
const url = this.serverUrl + "/update_settings" const url = this.serverUrl + "/update_settings";
const info = await new Promise<ServerInfo>(async (resolve) => { const info = await new Promise<ServerInfo>(async (resolve) => {
const formData = new FormData(); const formData = new FormData();
formData.append("key", key); formData.append("key", key);
formData.append("val", val); formData.append("val", val);
const request = new Request(url, { const request = new Request(url, {
method: 'POST', method: "POST",
body: formData, body: formData,
}); });
const res = await (await fetch(request)).json() as ServerInfo const res = (await (await fetch(request)).json()) as ServerInfo;
resolve(res) resolve(res);
}) });
return info return info;
} };
uploadFile2 = async (dir: string, file: File, onprogress: (progress: number, end: boolean) => void) => { uploadFile2 = async (dir: string, file: File, onprogress: (progress: number, end: boolean) => void) => {
const url = this.serverUrl + "/upload_file" const url = this.serverUrl + "/upload_file";
onprogress(0, false) onprogress(0, false);
const size = 1024 * 1024; const size = 1024 * 1024;
let index = 0; // index値 let index = 0; // index値
const fileLength = file.size const fileLength = file.size;
const filename = dir + file.name const filename = dir + file.name;
const fileChunkNum = Math.ceil(fileLength / size) const fileChunkNum = Math.ceil(fileLength / size);
while (true) { while (true) {
const promises: Promise<void>[] = [] const promises: Promise<void>[] = [];
for (let i = 0; i < 10; i++) { for (let i = 0; i < 10; i++) {
if (index * size >= fileLength) { if (index * size >= fileLength) {
break break;
} }
const chunk = file.slice(index * size, (index + 1) * size) const chunk = file.slice(index * size, (index + 1) * size);
const p = new Promise<void>((resolve) => { const p = new Promise<void>((resolve) => {
const formData = new FormData(); const formData = new FormData();
formData.append("file", new Blob([chunk])); formData.append("file", new Blob([chunk]));
formData.append("filename", `${filename}_${index}`); formData.append("filename", `${filename}_${index}`);
const request = new Request(url, { const request = new Request(url, {
method: 'POST', method: "POST",
body: formData, body: formData,
}); });
fetch(request).then(async (_response) => { fetch(request).then(async (_response) => {
// console.log(await response.text()) // console.log(await response.text())
resolve() resolve();
}) });
}) });
index += 1 index += 1;
promises.push(p) promises.push(p);
} }
await Promise.all(promises) await Promise.all(promises);
if (index * size >= fileLength) { if (index * size >= fileLength) {
break break;
} }
onprogress(Math.floor(((index) / (fileChunkNum + 1)) * 100), false) onprogress(Math.floor((index / (fileChunkNum + 1)) * 100), false);
} }
return fileChunkNum return fileChunkNum;
} };
uploadFile = async (buf: ArrayBuffer, filename: string, onprogress: (progress: number, end: boolean) => void) => { uploadFile = async (buf: ArrayBuffer, filename: string, onprogress: (progress: number, end: boolean) => void) => {
const url = this.serverUrl + "/upload_file" const url = this.serverUrl + "/upload_file";
onprogress(0, false) onprogress(0, false);
const size = 1024 * 1024; const size = 1024 * 1024;
const fileChunks: FileChunk[] = []; const fileChunks: FileChunk[] = [];
let index = 0; // index値 let index = 0; // index値
@ -113,65 +112,64 @@ export class ServerConfigurator {
}); });
} }
const chunkNum = fileChunks.length const chunkNum = fileChunks.length;
// console.log("FILE_CHUNKS:", chunkNum, fileChunks) // console.log("FILE_CHUNKS:", chunkNum, fileChunks)
while (true) { while (true) {
const promises: Promise<void>[] = [] const promises: Promise<void>[] = [];
for (let i = 0; i < 10; i++) { for (let i = 0; i < 10; i++) {
const chunk = fileChunks.shift() const chunk = fileChunks.shift();
if (!chunk) { if (!chunk) {
break break;
} }
const p = new Promise<void>((resolve) => { const p = new Promise<void>((resolve) => {
const formData = new FormData(); const formData = new FormData();
formData.append("file", new Blob([chunk.chunk])); formData.append("file", new Blob([chunk.chunk]));
formData.append("filename", `${filename}_${chunk.hash}`); formData.append("filename", `${filename}_${chunk.hash}`);
const request = new Request(url, { const request = new Request(url, {
method: 'POST', method: "POST",
body: formData, body: formData,
}); });
fetch(request).then(async (_response) => { fetch(request).then(async (_response) => {
// console.log(await response.text()) // console.log(await response.text())
resolve() resolve();
}) });
}) });
promises.push(p) promises.push(p);
} }
await Promise.all(promises) await Promise.all(promises);
if (fileChunks.length == 0) { if (fileChunks.length == 0) {
break break;
} }
onprogress(Math.floor(((chunkNum - fileChunks.length) / (chunkNum + 1)) * 100), false) onprogress(Math.floor(((chunkNum - fileChunks.length) / (chunkNum + 1)) * 100), false);
} }
return chunkNum return chunkNum;
} };
concatUploadedFile = async (filename: string, chunkNum: number) => { concatUploadedFile = async (filename: string, chunkNum: number) => {
const url = this.serverUrl + "/concat_uploaded_file" const url = this.serverUrl + "/concat_uploaded_file";
await new Promise<void>((resolve) => { await new Promise<void>((resolve) => {
const formData = new FormData(); const formData = new FormData();
formData.append("filename", filename); formData.append("filename", filename);
formData.append("filenameChunkNum", "" + chunkNum); formData.append("filenameChunkNum", "" + chunkNum);
const request = new Request(url, { const request = new Request(url, {
method: 'POST', method: "POST",
body: formData, body: formData,
}); });
fetch(request).then(async (response) => { fetch(request).then(async (response) => {
console.log(await response.text()) console.log(await response.text());
resolve() resolve();
}) });
}) });
} };
loadModel = async (slot: number, isHalf: boolean, params: string = "{}") => { loadModel = async (slot: number, isHalf: boolean, params: string = "{}") => {
if (isHalf == undefined || isHalf == null) { if (isHalf == undefined || isHalf == null) {
console.warn("isHalf is invalid value", isHalf) console.warn("isHalf is invalid value", isHalf);
isHalf = false isHalf = false;
} }
const url = this.serverUrl + "/load_model" const url = this.serverUrl + "/load_model";
const info = new Promise<ServerInfo>(async (resolve) => { const info = new Promise<ServerInfo>(async (resolve) => {
const formData = new FormData(); const formData = new FormData();
formData.append("slot", "" + slot); formData.append("slot", "" + slot);
@ -179,102 +177,101 @@ export class ServerConfigurator {
formData.append("params", params); formData.append("params", params);
const request = new Request(url, { const request = new Request(url, {
method: 'POST', method: "POST",
body: formData, body: formData,
}); });
const res = await (await fetch(request)).json() as ServerInfo const res = (await (await fetch(request)).json()) as ServerInfo;
resolve(res) resolve(res);
}) });
return await info return await info;
} };
uploadAssets = async (params: string) => { uploadAssets = async (params: string) => {
const url = this.serverUrl + "/upload_model_assets" const url = this.serverUrl + "/upload_model_assets";
const info = new Promise<ServerInfo>(async (resolve) => { const info = new Promise<ServerInfo>(async (resolve) => {
const formData = new FormData(); const formData = new FormData();
formData.append("params", params); formData.append("params", params);
const request = new Request(url, { const request = new Request(url, {
method: 'POST', method: "POST",
body: formData, body: formData,
}); });
const res = await (await fetch(request)).json() as ServerInfo const res = (await (await fetch(request)).json()) as ServerInfo;
resolve(res) resolve(res);
}) });
return await info return await info;
} };
getModelType = async () => { getModelType = async () => {
const url = this.serverUrl + "/model_type" const url = this.serverUrl + "/model_type";
const info = new Promise<ServerInfo>(async (resolve) => { const info = new Promise<ServerInfo>(async (resolve) => {
const request = new Request(url, { const request = new Request(url, {
method: 'GET', method: "GET",
}); });
const res = await (await fetch(request)).json() as ServerInfo const res = (await (await fetch(request)).json()) as ServerInfo;
resolve(res) resolve(res);
}) });
return await info return await info;
} };
export2onnx = async () => { export2onnx = async () => {
const url = this.serverUrl + "/onnx" const url = this.serverUrl + "/onnx";
const info = new Promise<OnnxExporterInfo>(async (resolve) => { const info = new Promise<OnnxExporterInfo>(async (resolve) => {
const request = new Request(url, { const request = new Request(url, {
method: 'GET', method: "GET",
}); });
const res = await (await fetch(request)).json() as OnnxExporterInfo const res = (await (await fetch(request)).json()) as OnnxExporterInfo;
resolve(res) resolve(res);
}) });
return await info return await info;
} };
mergeModel = async (req: MergeModelRequest) => { mergeModel = async (req: MergeModelRequest) => {
const url = this.serverUrl + "/merge_model" const url = this.serverUrl + "/merge_model";
const info = new Promise<ServerInfo>(async (resolve) => { const info = new Promise<ServerInfo>(async (resolve) => {
const formData = new FormData(); const formData = new FormData();
formData.append("request", JSON.stringify(req)); formData.append("request", JSON.stringify(req));
const request = new Request(url, { const request = new Request(url, {
method: 'POST', method: "POST",
body: formData, body: formData,
}); });
const res = await (await fetch(request)).json() as ServerInfo const res = (await (await fetch(request)).json()) as ServerInfo;
console.log("RESPONSE", res) console.log("RESPONSE", res);
resolve(res) resolve(res);
}) });
return await info return await info;
} };
updateModelDefault = async () => { updateModelDefault = async () => {
const url = this.serverUrl + "/update_model_default" const url = this.serverUrl + "/update_model_default";
const info = new Promise<ServerInfo>(async (resolve) => { const info = new Promise<ServerInfo>(async (resolve) => {
const request = new Request(url, { const request = new Request(url, {
method: 'POST', method: "POST",
}); });
const res = await (await fetch(request)).json() as ServerInfo const res = (await (await fetch(request)).json()) as ServerInfo;
console.log("RESPONSE", res) console.log("RESPONSE", res);
resolve(res) resolve(res);
}) });
return await info return await info;
} };
updateModelInfo = async (slot: number, key: string, val: string) => { updateModelInfo = async (slot: number, key: string, val: string) => {
const url = this.serverUrl + "/update_model_info" const url = this.serverUrl + "/update_model_info";
const newData = { slot, key, val } const newData = { slot, key, val };
const info = new Promise<ServerInfo>(async (resolve) => { const info = new Promise<ServerInfo>(async (resolve) => {
const formData = new FormData(); const formData = new FormData();
formData.append("newData", JSON.stringify(newData)); formData.append("newData", JSON.stringify(newData));
const request = new Request(url, { const request = new Request(url, {
method: 'POST', method: "POST",
body: formData, body: formData,
}); });
const res = await (await fetch(request)).json() as ServerInfo const res = (await (await fetch(request)).json()) as ServerInfo;
console.log("RESPONSE", res) console.log("RESPONSE", res);
resolve(res) resolve(res);
}) });
return await info return await info;
} };
} }

317
client/lib/src/VoiceChangerWorkletNode.ts → client/lib/src/client/VoiceChangerWorkletNode.ts
View File

@ -1,140 +1,134 @@
import { VoiceChangerWorkletProcessorRequest } from "./@types/voice-changer-worklet-processor"; import { VoiceChangerWorkletProcessorRequest } from "../@types/voice-changer-worklet-processor";
import { DefaultClientSettng, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletNodeSetting, WorkletSetting } from "./const"; import { DefaultClientSettng, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletNodeSetting, WorkletSetting } from "../const";
import { io, Socket } from "socket.io-client"; import { io, Socket } from "socket.io-client";
import { DefaultEventsMap } from "@socket.io/component-emitter"; import { DefaultEventsMap } from "@socket.io/component-emitter";
export type VoiceChangerWorkletListener = { export type VoiceChangerWorkletListener = {
notifyVolume: (vol: number) => void notifyVolume: (vol: number) => void;
notifySendBufferingTime: (time: number) => void notifySendBufferingTime: (time: number) => void;
notifyResponseTime: (time: number, perf?: number[]) => void notifyResponseTime: (time: number, perf?: number[]) => void;
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void;
} };
export class VoiceChangerWorkletNode extends AudioWorkletNode { export class VoiceChangerWorkletNode extends AudioWorkletNode {
private listener: VoiceChangerWorkletListener private listener: VoiceChangerWorkletListener;
private setting: WorkletNodeSetting = DefaultClientSettng.workletNodeSetting private setting: WorkletNodeSetting = DefaultClientSettng.workletNodeSetting;
private requestChunks: ArrayBuffer[] = [] private requestChunks: ArrayBuffer[] = [];
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null;
// performance monitor // performance monitor
private bufferStart = 0; private bufferStart = 0;
private isOutputRecording = false; private isOutputRecording = false;
private recordingOutputChunk: Float32Array[] = [] private recordingOutputChunk: Float32Array[] = [];
private outputNode: VoiceChangerWorkletNode | null = null private outputNode: VoiceChangerWorkletNode | null = null;
// Promises // Promises
private startPromiseResolve: ((value: void | PromiseLike<void>) => void) | null = null private startPromiseResolve: ((value: void | PromiseLike<void>) => void) | null = null;
private stopPromiseResolve: ((value: void | PromiseLike<void>) => void) | null = null private stopPromiseResolve: ((value: void | PromiseLike<void>) => void) | null = null;
constructor(context: AudioContext, listener: VoiceChangerWorkletListener) { constructor(context: AudioContext, listener: VoiceChangerWorkletListener) {
super(context, "voice-changer-worklet-processor"); super(context, "voice-changer-worklet-processor");
this.port.onmessage = this.handleMessage.bind(this); this.port.onmessage = this.handleMessage.bind(this);
this.listener = listener this.listener = listener;
this.createSocketIO() this.createSocketIO();
console.log(`[worklet_node][voice-changer-worklet-processor] created.`); console.log(`[worklet_node][voice-changer-worklet-processor] created.`);
} }
setOutputNode = (outputNode: VoiceChangerWorkletNode | null) => { setOutputNode = (outputNode: VoiceChangerWorkletNode | null) => {
this.outputNode = outputNode this.outputNode = outputNode;
} };
// 設定 // 設定
updateSetting = (setting: WorkletNodeSetting) => { updateSetting = (setting: WorkletNodeSetting) => {
console.log(`[WorkletNode] Updating WorkletNode Setting,`, this.setting, setting) console.log(`[WorkletNode] Updating WorkletNode Setting,`, this.setting, setting);
let recreateSocketIoRequired = false let recreateSocketIoRequired = false;
if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) { if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
recreateSocketIoRequired = true recreateSocketIoRequired = true;
} }
this.setting = setting this.setting = setting;
if (recreateSocketIoRequired) { if (recreateSocketIoRequired) {
this.createSocketIO() this.createSocketIO();
} }
} };
getSettings = (): WorkletNodeSetting => { getSettings = (): WorkletNodeSetting => {
return this.setting return this.setting;
} };
getSocketId = () => { getSocketId = () => {
return this.socket?.id return this.socket?.id;
} };
// 処理 // 処理
private createSocketIO = () => { private createSocketIO = () => {
if (this.socket) { if (this.socket) {
this.socket.close() this.socket.close();
} }
if (this.setting.protocol === "sio") { if (this.setting.protocol === "sio") {
this.socket = io(this.setting.serverUrl + "/test"); this.socket = io(this.setting.serverUrl + "/test");
this.socket.on('connect_error', (err) => { this.socket.on("connect_error", (err) => {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`) this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`);
})
this.socket.on('connect', () => {
console.log(`[SIO] connect to ${this.setting.serverUrl}`)
console.log(`[SIO] ${this.socket?.id}`)
}); });
this.socket.on('close', function (socket) { this.socket.on("connect", () => {
console.log(`[SIO] close ${socket.id}`) console.log(`[SIO] connect to ${this.setting.serverUrl}`);
console.log(`[SIO] ${this.socket?.id}`);
});
this.socket.on("close", function (socket) {
console.log(`[SIO] close ${socket.id}`);
}); });
this.socket.on("message", (response: any[]) => {
this.socket.on('message', (response: any[]) => { console.log("message:", response);
console.log("message:", response)
}); });
this.socket.on('response', (response: any[]) => { this.socket.on("response", (response: any[]) => {
const cur = Date.now();
const cur = Date.now() const responseTime = cur - response[0];
const responseTime = cur - response[0] const result = response[1] as ArrayBuffer;
const result = response[1] as ArrayBuffer const perf = response[2];
const perf = response[2]
// Quick hack for server device mode // Quick hack for server device mode
if (response[0] == 0) { if (response[0] == 0) {
this.listener.notifyResponseTime(Math.round(perf[0] * 1000), perf.slice(1, 4)) this.listener.notifyResponseTime(Math.round(perf[0] * 1000), perf.slice(1, 4));
return return;
} }
if (result.byteLength < 128 * 2) { if (result.byteLength < 128 * 2) {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`) this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`);
} else { } else {
if (this.outputNode != null) { if (this.outputNode != null) {
this.outputNode.postReceivedVoice(response[1]) this.outputNode.postReceivedVoice(response[1]);
} else { } else {
this.postReceivedVoice(response[1]) this.postReceivedVoice(response[1]);
} }
this.listener.notifyResponseTime(responseTime, perf) this.listener.notifyResponseTime(responseTime, perf);
} }
}); });
} }
} };
postReceivedVoice = (data: ArrayBuffer) => { postReceivedVoice = (data: ArrayBuffer) => {
// Int16 to Float // Int16 to Float
const i16Data = new Int16Array(data) const i16Data = new Int16Array(data);
const f32Data = new Float32Array(i16Data.length) const f32Data = new Float32Array(i16Data.length);
// console.log(`[worklet] f32DataLength${f32Data.length} i16DataLength${i16Data.length}`) // console.log(`[worklet] f32DataLength${f32Data.length} i16DataLength${i16Data.length}`)
i16Data.forEach((x, i) => { i16Data.forEach((x, i) => {
const float = (x >= 0x8000) ? -(0x10000 - x) / 0x8000 : x / 0x7FFF; const float = x >= 0x8000 ? -(0x10000 - x) / 0x8000 : x / 0x7fff;
f32Data[i] = float f32Data[i] = float;
}) });
// アップサンプリング // アップサンプリング
let upSampledBuffer: Float32Array | null = null let upSampledBuffer: Float32Array | null = null;
if (this.setting.sendingSampleRate == 48000) { if (this.setting.sendingSampleRate == 48000) {
upSampledBuffer = f32Data upSampledBuffer = f32Data;
} else { } else {
upSampledBuffer = new Float32Array(f32Data.length * 2) upSampledBuffer = new Float32Array(f32Data.length * 2);
for (let i = 0; i < f32Data.length; i++) { for (let i = 0; i < f32Data.length; i++) {
const currentFrame = f32Data[i] const currentFrame = f32Data[i];
const nextFrame = i + 1 < f32Data.length ? f32Data[i + 1] : f32Data[i] const nextFrame = i + 1 < f32Data.length ? f32Data[i + 1] : f32Data[i];
upSampledBuffer[i * 2] = currentFrame upSampledBuffer[i * 2] = currentFrame;
upSampledBuffer[i * 2 + 1] = (currentFrame + nextFrame) / 2 upSampledBuffer[i * 2 + 1] = (currentFrame + nextFrame) / 2;
} }
} }
@ -143,15 +137,14 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
voice: upSampledBuffer, voice: upSampledBuffer,
numTrancateTreshold: 0, numTrancateTreshold: 0,
volTrancateThreshold: 0, volTrancateThreshold: 0,
volTrancateLength: 0 volTrancateLength: 0,
} };
this.port.postMessage(req) this.port.postMessage(req);
if (this.isOutputRecording) { if (this.isOutputRecording) {
this.recordingOutputChunk.push(upSampledBuffer) this.recordingOutputChunk.push(upSampledBuffer);
} }
};
}
private _averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) { private _averageDownsampleBuffer(buffer: Float32Array, originalSampleRate: number, destinationSamplerate: number) {
if (originalSampleRate == destinationSamplerate) { if (originalSampleRate == destinationSamplerate) {
@ -168,7 +161,8 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
while (offsetResult < result.length) { while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio); var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
// Use average value of skipped samples // Use average value of skipped samples
var accum = 0, count = 0; var accum = 0,
count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) { for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
accum += buffer[i]; accum += buffer[i];
count++; count++;
@ -185,112 +179,102 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
// console.log(`[Node:handleMessage_] `, event.data.volume); // console.log(`[Node:handleMessage_] `, event.data.volume);
if (event.data.responseType === "start_ok") { if (event.data.responseType === "start_ok") {
if (this.startPromiseResolve) { if (this.startPromiseResolve) {
this.startPromiseResolve() this.startPromiseResolve();
this.startPromiseResolve = null this.startPromiseResolve = null;
} }
} else if (event.data.responseType === "stop_ok") { } else if (event.data.responseType === "stop_ok") {
if (this.stopPromiseResolve) { if (this.stopPromiseResolve) {
this.stopPromiseResolve() this.stopPromiseResolve();
this.stopPromiseResolve = null this.stopPromiseResolve = null;
} }
} else if (event.data.responseType === "volume") { } else if (event.data.responseType === "volume") {
this.listener.notifyVolume(event.data.volume as number) this.listener.notifyVolume(event.data.volume as number);
} else if (event.data.responseType === "inputData") { } else if (event.data.responseType === "inputData") {
const inputData = event.data.inputData as Float32Array const inputData = event.data.inputData as Float32Array;
// console.log("receive input data", inputData) // console.log("receive input data", inputData)
// ダウンサンプリング // ダウンサンプリング
let downsampledBuffer: Float32Array | null = null let downsampledBuffer: Float32Array | null = null;
if (this.setting.sendingSampleRate == 48000) { if (this.setting.sendingSampleRate == 48000) {
downsampledBuffer = inputData downsampledBuffer = inputData;
} else if (this.setting.downSamplingMode == DownSamplingMode.decimate) { } else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
//////// (Kind 1) 間引き ////////// //////// (Kind 1) 間引き //////////
//// 48000Hz で入ってくるので間引いて24000Hzに変換する。 //// 48000Hz で入ってくるので間引いて24000Hzに変換する。
downsampledBuffer = new Float32Array(inputData.length / 2); downsampledBuffer = new Float32Array(inputData.length / 2);
for (let i = 0; i < inputData.length; i++) { for (let i = 0; i < inputData.length; i++) {
if (i % 2 == 0) { if (i % 2 == 0) {
downsampledBuffer[i / 2] = inputData[i] downsampledBuffer[i / 2] = inputData[i];
} }
} }
} else { } else {
//////// (Kind 2) 平均 ////////// //////// (Kind 2) 平均 //////////
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000) // downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
downsampledBuffer = this._averageDownsampleBuffer(inputData, 48000, this.setting.sendingSampleRate) downsampledBuffer = this._averageDownsampleBuffer(inputData, 48000, this.setting.sendingSampleRate);
} }
// Float to Int16 // Float to Int16
const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2) const arrayBuffer = new ArrayBuffer(downsampledBuffer.length * 2);
const dataView = new DataView(arrayBuffer); const dataView = new DataView(arrayBuffer);
for (let i = 0; i < downsampledBuffer.length; i++) { for (let i = 0; i < downsampledBuffer.length; i++) {
let s = Math.max(-1, Math.min(1, downsampledBuffer[i])); let s = Math.max(-1, Math.min(1, downsampledBuffer[i]));
s = s < 0 ? s * 0x8000 : s * 0x7FFF s = s < 0 ? s * 0x8000 : s * 0x7fff;
dataView.setInt16(i * 2, s, true); dataView.setInt16(i * 2, s, true);
} }
// バッファリング // バッファリング
this.requestChunks.push(arrayBuffer) this.requestChunks.push(arrayBuffer);
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。 //// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
if (this.requestChunks.length < this.setting.inputChunkNum) { if (this.requestChunks.length < this.setting.inputChunkNum) {
return return;
} }
// リクエスト用の入れ物を作成 // リクエスト用の入れ物を作成
const windowByteLength = this.requestChunks.reduce((prev, cur) => { const windowByteLength = this.requestChunks.reduce((prev, cur) => {
return prev + cur.byteLength return prev + cur.byteLength;
}, 0) }, 0);
const newBuffer = new Uint8Array(windowByteLength); const newBuffer = new Uint8Array(windowByteLength);
// リクエストのデータをセット // リクエストのデータをセット
this.requestChunks.reduce((prev, cur) => { this.requestChunks.reduce((prev, cur) => {
newBuffer.set(new Uint8Array(cur), prev) newBuffer.set(new Uint8Array(cur), prev);
return prev + cur.byteLength return prev + cur.byteLength;
}, 0) }, 0);
this.sendBuffer(newBuffer);
this.requestChunks = [];
this.sendBuffer(newBuffer) this.listener.notifySendBufferingTime(Date.now() - this.bufferStart);
this.requestChunks = [] this.bufferStart = Date.now();
this.listener.notifySendBufferingTime(Date.now() - this.bufferStart)
this.bufferStart = Date.now()
} else { } else {
console.warn(`[worklet_node][voice-changer-worklet-processor] unknown response ${event.data.responseType}`, event.data) console.warn(`[worklet_node][voice-changer-worklet-processor] unknown response ${event.data.responseType}`, event.data);
} }
} }
private sendBuffer = async (newBuffer: Uint8Array) => { private sendBuffer = async (newBuffer: Uint8Array) => {
const timestamp = Date.now() const timestamp = Date.now();
if (this.setting.protocol === "sio") { if (this.setting.protocol === "sio") {
if (!this.socket) { if (!this.socket) {
console.warn(`sio is not initialized`) console.warn(`sio is not initialized`);
return return;
} }
// console.log("emit!") // console.log("emit!")
this.socket.emit('request_message', [ this.socket.emit("request_message", [timestamp, newBuffer.buffer]);
timestamp,
newBuffer.buffer]);
} else { } else {
const res = await postVoice( const res = await postVoice(this.setting.serverUrl + "/test", timestamp, newBuffer.buffer);
this.setting.serverUrl + "/test",
timestamp,
newBuffer.buffer)
if (res.byteLength < 128 * 2) { if (res.byteLength < 128 * 2) {
this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`) this.listener.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`);
} else { } else {
if (this.outputNode != null) { if (this.outputNode != null) {
this.outputNode.postReceivedVoice(res) this.outputNode.postReceivedVoice(res);
} else { } else {
this.postReceivedVoice(res) this.postReceivedVoice(res);
} }
this.listener.notifyResponseTime(Date.now() - timestamp) this.listener.notifyResponseTime(Date.now() - timestamp);
} }
} }
} };
configure = (setting: WorkletSetting) => { configure = (setting: WorkletSetting) => {
const req: VoiceChangerWorkletProcessorRequest = { const req: VoiceChangerWorkletProcessorRequest = {
@ -298,105 +282,100 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
voice: new Float32Array(1), voice: new Float32Array(1),
numTrancateTreshold: setting.numTrancateTreshold, numTrancateTreshold: setting.numTrancateTreshold,
volTrancateThreshold: setting.volTrancateThreshold, volTrancateThreshold: setting.volTrancateThreshold,
volTrancateLength: setting.volTrancateLength volTrancateLength: setting.volTrancateLength,
} };
this.port.postMessage(req) this.port.postMessage(req);
} };
start = async () => { start = async () => {
const p = new Promise<void>((resolve) => { const p = new Promise<void>((resolve) => {
this.startPromiseResolve = resolve this.startPromiseResolve = resolve;
}) });
const req: VoiceChangerWorkletProcessorRequest = { const req: VoiceChangerWorkletProcessorRequest = {
requestType: "start", requestType: "start",
voice: new Float32Array(1), voice: new Float32Array(1),
numTrancateTreshold: 0, numTrancateTreshold: 0,
volTrancateThreshold: 0, volTrancateThreshold: 0,
volTrancateLength: 0 volTrancateLength: 0,
} };
this.port.postMessage(req) this.port.postMessage(req);
await p await p;
};
}
stop = async () => { stop = async () => {
const p = new Promise<void>((resolve) => { const p = new Promise<void>((resolve) => {
this.stopPromiseResolve = resolve this.stopPromiseResolve = resolve;
}) });
const req: VoiceChangerWorkletProcessorRequest = { const req: VoiceChangerWorkletProcessorRequest = {
requestType: "stop", requestType: "stop",
voice: new Float32Array(1), voice: new Float32Array(1),
numTrancateTreshold: 0, numTrancateTreshold: 0,
volTrancateThreshold: 0, volTrancateThreshold: 0,
volTrancateLength: 0 volTrancateLength: 0,
} };
this.port.postMessage(req) this.port.postMessage(req);
await p await p;
} };
trancateBuffer = () => { trancateBuffer = () => {
const req: VoiceChangerWorkletProcessorRequest = { const req: VoiceChangerWorkletProcessorRequest = {
requestType: "trancateBuffer", requestType: "trancateBuffer",
voice: new Float32Array(1), voice: new Float32Array(1),
numTrancateTreshold: 0, numTrancateTreshold: 0,
volTrancateThreshold: 0, volTrancateThreshold: 0,
volTrancateLength: 0 volTrancateLength: 0,
} };
this.port.postMessage(req) this.port.postMessage(req);
} };
startOutputRecording = () => { startOutputRecording = () => {
this.recordingOutputChunk = [] this.recordingOutputChunk = [];
this.isOutputRecording = true this.isOutputRecording = true;
} };
stopOutputRecording = () => { stopOutputRecording = () => {
this.isOutputRecording = false this.isOutputRecording = false;
const dataSize = this.recordingOutputChunk.reduce((prev, cur) => { const dataSize = this.recordingOutputChunk.reduce((prev, cur) => {
return prev + cur.length return prev + cur.length;
}, 0) }, 0);
const samples = new Float32Array(dataSize); const samples = new Float32Array(dataSize);
let sampleIndex = 0 let sampleIndex = 0;
for (let i = 0; i < this.recordingOutputChunk.length; i++) { for (let i = 0; i < this.recordingOutputChunk.length; i++) {
for (let j = 0; j < this.recordingOutputChunk[i].length; j++) { for (let j = 0; j < this.recordingOutputChunk[i].length; j++) {
samples[sampleIndex] = this.recordingOutputChunk[i][j]; samples[sampleIndex] = this.recordingOutputChunk[i][j];
sampleIndex++; sampleIndex++;
} }
} }
return samples return samples;
} };
} }
export const postVoice = async (url: string, timestamp: number, buffer: ArrayBuffer) => {
export const postVoice = async (
url: string,
timestamp: number,
buffer: ArrayBuffer) => {
const obj = { const obj = {
timestamp, timestamp,
buffer: Buffer.from(buffer).toString('base64') buffer: Buffer.from(buffer).toString("base64"),
}; };
const body = JSON.stringify(obj); const body = JSON.stringify(obj);
const res = await fetch(`${url}`, { const res = await fetch(`${url}`, {
method: "POST", method: "POST",
headers: { headers: {
'Accept': 'application/json', Accept: "application/json",
'Content-Type': 'application/json' "Content-Type": "application/json",
}, },
body: body body: body,
}) });
try { try {
const receivedJson = await res.json() const receivedJson = await res.json();
const changedVoiceBase64 = receivedJson["changedVoiceBase64"] const changedVoiceBase64 = receivedJson["changedVoiceBase64"];
const buf = Buffer.from(changedVoiceBase64, "base64") const buf = Buffer.from(changedVoiceBase64, "base64");
const ab = new ArrayBuffer(buf.length); const ab = new ArrayBuffer(buf.length);
const view = new Uint8Array(ab); const view = new Uint8Array(ab);
for (let i = 0; i < buf.length; ++i) { for (let i = 0; i < buf.length; ++i) {
view[i] = buf[i]; view[i] = buf[i];
} }
return ab return ab;
} catch (e) { } catch (e) {
console.log("Exception:", e) console.log("Exception:", e);
return new ArrayBuffer(10); return new ArrayBuffer(10);
} }
} };

662
client/lib/src/const.ts
View File

@ -1,20 +1,18 @@
// (★1) chunk sizeは 128サンプル, 256byte(int16)と定義。 // (★1) chunk sizeは 128サンプル, 256byte(int16)と定義。
// (★2) 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理。 // (★2) 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理。
// 24000sample -> 1sec, 128sample(1chunk) -> 5.333msec // 24000sample -> 1sec, 128sample(1chunk) -> 5.333msec
// 187.5chunk -> 1sec // 187.5chunk -> 1sec
export const VoiceChangerType = { export const VoiceChangerType = {
"MMVCv15": "MMVCv15", MMVCv15: "MMVCv15",
"MMVCv13": "MMVCv13", MMVCv13: "MMVCv13",
"so-vits-svc-40": "so-vits-svc-40", "so-vits-svc-40": "so-vits-svc-40",
"DDSP-SVC": "DDSP-SVC", "DDSP-SVC": "DDSP-SVC",
"RVC": "RVC", RVC: "RVC",
"Diffusion-SVC":"Diffusion-SVC", "Diffusion-SVC": "Diffusion-SVC",
"Beatrice": "Beatrice" Beatrice: "Beatrice",
} as const;
} as const export type VoiceChangerType = (typeof VoiceChangerType)[keyof typeof VoiceChangerType];
export type VoiceChangerType = typeof VoiceChangerType[keyof typeof VoiceChangerType]
/////////////////////// ///////////////////////
// サーバセッティング // サーバセッティング
@ -22,342 +20,333 @@ export type VoiceChangerType = typeof VoiceChangerType[keyof typeof VoiceChanger
export const InputSampleRate = { export const InputSampleRate = {
"48000": 48000, "48000": 48000,
"44100": 44100, "44100": 44100,
"24000": 24000 "24000": 24000,
} as const } as const;
export type InputSampleRate = typeof InputSampleRate[keyof typeof InputSampleRate] export type InputSampleRate = (typeof InputSampleRate)[keyof typeof InputSampleRate];
export const ModelSamplingRate = { export const ModelSamplingRate = {
"48000": 48000, "48000": 48000,
"40000": 40000, "40000": 40000,
"32000": 32000 "32000": 32000,
} as const } as const;
export type ModelSamplingRate = typeof InputSampleRate[keyof typeof InputSampleRate] export type ModelSamplingRate = (typeof InputSampleRate)[keyof typeof InputSampleRate];
export const CrossFadeOverlapSize = { export const CrossFadeOverlapSize = {
"1024": 1024, "1024": 1024,
"2048": 2048, "2048": 2048,
"4096": 4096, "4096": 4096,
} as const } as const;
export type CrossFadeOverlapSize = typeof CrossFadeOverlapSize[keyof typeof CrossFadeOverlapSize] export type CrossFadeOverlapSize = (typeof CrossFadeOverlapSize)[keyof typeof CrossFadeOverlapSize];
export const F0Detector = { export const F0Detector = {
"dio": "dio", dio: "dio",
"harvest": "harvest", harvest: "harvest",
"crepe": "crepe", crepe: "crepe",
"crepe_full": "crepe_full", crepe_full: "crepe_full",
"crepe_tiny": "crepe_tiny", crepe_tiny: "crepe_tiny",
"rmvpe": "rmvpe", rmvpe: "rmvpe",
"rmvpe_onnx": "rmvpe_onnx", rmvpe_onnx: "rmvpe_onnx",
} as const } as const;
export type F0Detector = typeof F0Detector[keyof typeof F0Detector] export type F0Detector = (typeof F0Detector)[keyof typeof F0Detector];
export const DiffMethod = { export const DiffMethod = {
"pndm": "pndm", pndm: "pndm",
"dpm-solver": "dpm-solver", "dpm-solver": "dpm-solver",
} as const } as const;
export type DiffMethod = typeof DiffMethod[keyof typeof DiffMethod] export type DiffMethod = (typeof DiffMethod)[keyof typeof DiffMethod];
export const RVCModelType = { export const RVCModelType = {
"pyTorchRVC": "pyTorchRVC", pyTorchRVC: "pyTorchRVC",
"pyTorchRVCNono": "pyTorchRVCNono", pyTorchRVCNono: "pyTorchRVCNono",
"pyTorchRVCv2": "pyTorchRVCv2", pyTorchRVCv2: "pyTorchRVCv2",
"pyTorchRVCv2Nono": "pyTorchRVCv2Nono", pyTorchRVCv2Nono: "pyTorchRVCv2Nono",
"pyTorchWebUI": "pyTorchWebUI", pyTorchWebUI: "pyTorchWebUI",
"pyTorchWebUINono": "pyTorchWebUINono", pyTorchWebUINono: "pyTorchWebUINono",
"onnxRVC": "onnxRVC", onnxRVC: "onnxRVC",
"onnxRVCNono": "onnxRVCNono", onnxRVCNono: "onnxRVCNono",
} as const } as const;
export type RVCModelType = typeof RVCModelType[keyof typeof RVCModelType] export type RVCModelType = (typeof RVCModelType)[keyof typeof RVCModelType];
export const ServerSettingKey = { export const ServerSettingKey = {
"passThrough":"passThrough", passThrough: "passThrough",
"srcId": "srcId", srcId: "srcId",
"dstId": "dstId", dstId: "dstId",
"gpu": "gpu", gpu: "gpu",
"crossFadeOffsetRate": "crossFadeOffsetRate", crossFadeOffsetRate: "crossFadeOffsetRate",
"crossFadeEndRate": "crossFadeEndRate", crossFadeEndRate: "crossFadeEndRate",
"crossFadeOverlapSize": "crossFadeOverlapSize", crossFadeOverlapSize: "crossFadeOverlapSize",
"framework": "framework", framework: "framework",
"onnxExecutionProvider": "onnxExecutionProvider", onnxExecutionProvider: "onnxExecutionProvider",
"f0Factor": "f0Factor", f0Factor: "f0Factor",
"f0Detector": "f0Detector", f0Detector: "f0Detector",
"recordIO": "recordIO", recordIO: "recordIO",
"enableServerAudio": "enableServerAudio", enableServerAudio: "enableServerAudio",
"serverAudioStated": "serverAudioStated", serverAudioStated: "serverAudioStated",
"serverAudioSampleRate": "serverAudioSampleRate", serverAudioSampleRate: "serverAudioSampleRate",
"serverInputAudioSampleRate": "serverInputAudioSampleRate", serverInputAudioSampleRate: "serverInputAudioSampleRate",
"serverOutputAudioSampleRate": "serverOutputAudioSampleRate", serverOutputAudioSampleRate: "serverOutputAudioSampleRate",
"serverMonitorAudioSampleRate": "serverMonitorAudioSampleRate", serverMonitorAudioSampleRate: "serverMonitorAudioSampleRate",
"serverInputAudioBufferSize": "serverInputAudioBufferSize", serverInputAudioBufferSize: "serverInputAudioBufferSize",
"serverOutputAudioBufferSize": "serverOutputAudioBufferSize", serverOutputAudioBufferSize: "serverOutputAudioBufferSize",
"serverInputDeviceId": "serverInputDeviceId", serverInputDeviceId: "serverInputDeviceId",
"serverOutputDeviceId": "serverOutputDeviceId", serverOutputDeviceId: "serverOutputDeviceId",
"serverMonitorDeviceId": "serverMonitorDeviceId", serverMonitorDeviceId: "serverMonitorDeviceId",
"serverReadChunkSize": "serverReadChunkSize", serverReadChunkSize: "serverReadChunkSize",
"serverInputAudioGain": "serverInputAudioGain", serverInputAudioGain: "serverInputAudioGain",
"serverOutputAudioGain": "serverOutputAudioGain", serverOutputAudioGain: "serverOutputAudioGain",
"serverMonitorAudioGain": "serverMonitorAudioGain", serverMonitorAudioGain: "serverMonitorAudioGain",
"tran": "tran", tran: "tran",
"noiseScale": "noiseScale", noiseScale: "noiseScale",
"predictF0": "predictF0", predictF0: "predictF0",
"silentThreshold": "silentThreshold", silentThreshold: "silentThreshold",
"extraConvertSize": "extraConvertSize", extraConvertSize: "extraConvertSize",
"clusterInferRatio": "clusterInferRatio", clusterInferRatio: "clusterInferRatio",
"indexRatio": "indexRatio", indexRatio: "indexRatio",
"protect": "protect", protect: "protect",
"rvcQuality": "rvcQuality", rvcQuality: "rvcQuality",
"modelSamplingRate": "modelSamplingRate", modelSamplingRate: "modelSamplingRate",
"silenceFront": "silenceFront", silenceFront: "silenceFront",
"modelSlotIndex": "modelSlotIndex", modelSlotIndex: "modelSlotIndex",
"useEnhancer": "useEnhancer", useEnhancer: "useEnhancer",
"useDiff": "useDiff", useDiff: "useDiff",
// "useDiffDpm": "useDiffDpm", // "useDiffDpm": "useDiffDpm",
"diffMethod": "diffMethod", diffMethod: "diffMethod",
"useDiffSilence": "useDiffSilence", useDiffSilence: "useDiffSilence",
"diffAcc": "diffAcc", diffAcc: "diffAcc",
"diffSpkId": "diffSpkId", diffSpkId: "diffSpkId",
"kStep": "kStep", kStep: "kStep",
"threshold": "threshold", threshold: "threshold",
"speedUp": "speedUp", speedUp: "speedUp",
"skipDiffusion": "skipDiffusion", skipDiffusion: "skipDiffusion",
"inputSampleRate": "inputSampleRate",
"enableDirectML": "enableDirectML",
} as const
export type ServerSettingKey = typeof ServerSettingKey[keyof typeof ServerSettingKey]
inputSampleRate: "inputSampleRate",
enableDirectML: "enableDirectML",
} as const;
export type ServerSettingKey = (typeof ServerSettingKey)[keyof typeof ServerSettingKey];
export type VoiceChangerServerSetting = { export type VoiceChangerServerSetting = {
passThrough: boolean passThrough: boolean;
srcId: number, srcId: number;
dstId: number, dstId: number;
gpu: number, gpu: number;
crossFadeOffsetRate: number, crossFadeOffsetRate: number;
crossFadeEndRate: number, crossFadeEndRate: number;
crossFadeOverlapSize: CrossFadeOverlapSize, crossFadeOverlapSize: CrossFadeOverlapSize;
f0Factor: number f0Factor: number;
f0Detector: F0Detector // dio or harvest f0Detector: F0Detector; // dio or harvest
recordIO: number // 0:off, 1:on recordIO: number; // 0:off, 1:on
enableServerAudio: number // 0:off, 1:on enableServerAudio: number; // 0:off, 1:on
serverAudioStated: number // 0:off, 1:on serverAudioStated: number; // 0:off, 1:on
serverAudioSampleRate: number serverAudioSampleRate: number;
serverInputAudioSampleRate: number serverInputAudioSampleRate: number;
serverOutputAudioSampleRate: number serverOutputAudioSampleRate: number;
serverMonitorAudioSampleRate: number serverMonitorAudioSampleRate: number;
serverInputAudioBufferSize: number serverInputAudioBufferSize: number;
serverOutputAudioBufferSize: number serverOutputAudioBufferSize: number;
serverInputDeviceId: number serverInputDeviceId: number;
serverOutputDeviceId: number serverOutputDeviceId: number;
serverMonitorDeviceId: number serverMonitorDeviceId: number;
serverReadChunkSize: number serverReadChunkSize: number;
serverInputAudioGain: number serverInputAudioGain: number;
serverOutputAudioGain: number serverOutputAudioGain: number;
serverMonitorAudioGain: number serverMonitorAudioGain: number;
tran: number; // so-vits-svc
noiseScale: number; // so-vits-svc
predictF0: number; // so-vits-svc
silentThreshold: number; // so-vits-svc
extraConvertSize: number; // so-vits-svc
clusterInferRatio: number; // so-vits-svc
tran: number // so-vits-svc indexRatio: number; // RVC
noiseScale: number // so-vits-svc protect: number; // RVC
predictF0: number // so-vits-svc rvcQuality: number; // 0:low, 1:high
silentThreshold: number // so-vits-svc silenceFront: number; // 0:off, 1:on
extraConvertSize: number// so-vits-svc modelSamplingRate: ModelSamplingRate; // 32000,40000,48000
clusterInferRatio: number // so-vits-svc modelSlotIndex: number;
indexRatio: number // RVC useEnhancer: number; // DDSP-SVC
protect: number // RVC useDiff: number; // DDSP-SVC
rvcQuality: number // 0:low, 1:high
silenceFront: number // 0:off, 1:on
modelSamplingRate: ModelSamplingRate // 32000,40000,48000
modelSlotIndex: number,
useEnhancer: number// DDSP-SVC
useDiff: number// DDSP-SVC
// useDiffDpm: number// DDSP-SVC // useDiffDpm: number// DDSP-SVC
diffMethod: DiffMethod, // DDSP-SVC diffMethod: DiffMethod; // DDSP-SVC
useDiffSilence: number// DDSP-SVC useDiffSilence: number; // DDSP-SVC
diffAcc: number// DDSP-SVC diffAcc: number; // DDSP-SVC
diffSpkId: number// DDSP-SVC diffSpkId: number; // DDSP-SVC
kStep: number// DDSP-SVC kStep: number; // DDSP-SVC
threshold: number// DDSP-SVC threshold: number; // DDSP-SVC
speedUp: number // Diffusion-SVC speedUp: number; // Diffusion-SVC
skipDiffusion: number // Diffusion-SVC 0:off, 1:on skipDiffusion: number; // Diffusion-SVC 0:off, 1:on
inputSampleRate: InputSampleRate inputSampleRate: InputSampleRate;
enableDirectML: number enableDirectML: number;
} };
type ModelSlot = { type ModelSlot = {
slotIndex: number slotIndex: number;
voiceChangerType: VoiceChangerType voiceChangerType: VoiceChangerType;
name: string, name: string;
description: string, description: string;
credit: string, credit: string;
termsOfUseUrl: string, termsOfUseUrl: string;
iconFile: string iconFile: string;
speakers: { [key: number]: string } speakers: { [key: number]: string };
} };
export type RVCModelSlot = ModelSlot & { export type RVCModelSlot = ModelSlot & {
modelFile: string modelFile: string;
indexFile: string, indexFile: string;
defaultIndexRatio: number, defaultIndexRatio: number;
defaultProtect: number, defaultProtect: number;
defaultTune: number, defaultTune: number;
modelType: RVCModelType, modelType: RVCModelType;
embChannels: number, embChannels: number;
f0: boolean, f0: boolean;
samplingRate: number samplingRate: number;
deprecated: boolean deprecated: boolean;
} };
export type MMVCv13ModelSlot = ModelSlot & { export type MMVCv13ModelSlot = ModelSlot & {
modelFile: string modelFile: string;
configFile: string, configFile: string;
srcId: number srcId: number;
dstId: number dstId: number;
samplingRate: number samplingRate: number;
speakers: { [key: number]: string } speakers: { [key: number]: string };
} };
export type MMVCv15ModelSlot = ModelSlot & { export type MMVCv15ModelSlot = ModelSlot & {
modelFile: string modelFile: string;
configFile: string, configFile: string;
srcId: number srcId: number;
dstId: number dstId: number;
f0Factor: number f0Factor: number;
samplingRate: number samplingRate: number;
f0: { [key: number]: number } f0: { [key: number]: number };
} };
export type SoVitsSvc40ModelSlot = ModelSlot & { export type SoVitsSvc40ModelSlot = ModelSlot & {
modelFile: string modelFile: string;
configFile: string, configFile: string;
clusterFile: string, clusterFile: string;
dstId: number dstId: number;
samplingRate: number samplingRate: number;
defaultTune: number defaultTune: number;
defaultClusterInferRatio: number defaultClusterInferRatio: number;
noiseScale: number noiseScale: number;
speakers: { [key: number]: string } speakers: { [key: number]: string };
} };
export type DDSPSVCModelSlot = ModelSlot & { export type DDSPSVCModelSlot = ModelSlot & {
modelFile: string modelFile: string;
configFile: string, configFile: string;
diffModelFile: string diffModelFile: string;
diffConfigFile: string diffConfigFile: string;
dstId: number dstId: number;
samplingRate: number samplingRate: number;
defaultTune: number
enhancer: boolean
diffusion: boolean
acc: number
kstep: number
speakers: { [key: number]: string }
}
defaultTune: number;
enhancer: boolean;
diffusion: boolean;
acc: number;
kstep: number;
speakers: { [key: number]: string };
};
export type DiffusionSVCModelSlot = ModelSlot & { export type DiffusionSVCModelSlot = ModelSlot & {
modelFile: string modelFile: string;
dstId: number dstId: number;
samplingRate: number samplingRate: number;
defaultTune: number
defaultKstep : number
defaultSpeedup: number
kStepMax: number
nLayers: number
nnLayers: number
speakers: { [key: number]: string }
}
defaultTune: number;
defaultKstep: number;
defaultSpeedup: number;
kStepMax: number;
nLayers: number;
nnLayers: number;
speakers: { [key: number]: string };
};
export type BeatriceModelSlot = ModelSlot & { export type BeatriceModelSlot = ModelSlot & {
modelFile: string modelFile: string;
dstId: number dstId: number;
speakers: { [key: number]: string } speakers: { [key: number]: string };
} };
export type ModelSlotUnion = RVCModelSlot | MMVCv13ModelSlot | MMVCv15ModelSlot | SoVitsSvc40ModelSlot | DDSPSVCModelSlot | DiffusionSVCModelSlot | BeatriceModelSlot export type ModelSlotUnion = RVCModelSlot | MMVCv13ModelSlot | MMVCv15ModelSlot | SoVitsSvc40ModelSlot | DDSPSVCModelSlot | DiffusionSVCModelSlot | BeatriceModelSlot;
type ServerAudioDevice = { type ServerAudioDevice = {
kind: "audioinput" | "audiooutput", kind: "audioinput" | "audiooutput";
index: number, index: number;
name: string name: string;
hostAPI: string hostAPI: string;
} };
export type ServerInfo = VoiceChangerServerSetting & { export type ServerInfo = VoiceChangerServerSetting & {
// コンフィグ対象外 (getInfoで取得のみ可能な情報) // コンフィグ対象外 (getInfoで取得のみ可能な情報)
status: string status: string;
modelSlots: ModelSlotUnion[] modelSlots: ModelSlotUnion[];
serverAudioInputDevices: ServerAudioDevice[] serverAudioInputDevices: ServerAudioDevice[];
serverAudioOutputDevices: ServerAudioDevice[] serverAudioOutputDevices: ServerAudioDevice[];
sampleModels: (RVCSampleModel|DiffusionSVCSampleModel)[] sampleModels: (RVCSampleModel | DiffusionSVCSampleModel)[];
gpus: { gpus: {
id: number, id: number;
name: string, name: string;
memory: number, memory: number;
}[] }[];
maxInputLength: number // MMVCv15 maxInputLength: number; // MMVCv15
voiceChangerParams: { voiceChangerParams: {
model_dir: string model_dir: string;
} };
} };
export type SampleModel = { export type SampleModel = {
id: string id: string;
voiceChangerType: VoiceChangerType voiceChangerType: VoiceChangerType;
lang: string lang: string;
tag: string[] tag: string[];
name: string name: string;
modelUrl: string modelUrl: string;
termsOfUseUrl: string termsOfUseUrl: string;
icon: string icon: string;
credit: string credit: string;
description: string description: string;
sampleRate: number sampleRate: number;
modelType: string modelType: string;
f0: boolean f0: boolean;
} };
export type RVCSampleModel =SampleModel & {
indexUrl: string
featureUrl: string
}
export type DiffusionSVCSampleModel =SampleModel & {
numOfDiffLayers: number
numOfNativeLayers: number
maxKStep: number
}
export type RVCSampleModel = SampleModel & {
indexUrl: string;
featureUrl: string;
};
export type DiffusionSVCSampleModel = SampleModel & {
numOfDiffLayers: number;
numOfNativeLayers: number;
maxKStep: number;
};
export const DefaultServerSetting: ServerInfo = { export const DefaultServerSetting: ServerInfo = {
// VC Common // VC Common
passThrough: false, passThrough: false,
inputSampleRate: 48000, inputSampleRate: 48000,
@ -388,7 +377,6 @@ export const DefaultServerSetting: ServerInfo = {
dstId: 1, dstId: 1,
gpu: 0, gpu: 0,
f0Factor: 1.0, f0Factor: 1.0,
f0Detector: F0Detector.rmvpe_onnx, f0Detector: F0Detector.rmvpe_onnx,
@ -421,101 +409,99 @@ export const DefaultServerSetting: ServerInfo = {
skipDiffusion: 1, skipDiffusion: 1,
enableDirectML: 0, enableDirectML: 0,
// //
status: "ok", status: "ok",
modelSlots: [], modelSlots: [],
serverAudioInputDevices: [], serverAudioInputDevices: [],
serverAudioOutputDevices: [], serverAudioOutputDevices: [],
maxInputLength: 128 * 2048, maxInputLength: 128 * 2048,
voiceChangerParams: { voiceChangerParams: {
model_dir: "" model_dir: "",
} },
} };
/////////////////////// ///////////////////////
// Workletセッティング // Workletセッティング
/////////////////////// ///////////////////////
export type WorkletSetting = { export type WorkletSetting = {
numTrancateTreshold: number, numTrancateTreshold: number;
volTrancateThreshold: number, volTrancateThreshold: number;
volTrancateLength: number volTrancateLength: number;
} };
/////////////////////// ///////////////////////
// Worklet Nodeセッティング // Worklet Nodeセッティング
/////////////////////// ///////////////////////
export const Protocol = { export const Protocol = {
"sio": "sio", sio: "sio",
"rest": "rest", rest: "rest",
} as const } as const;
export type Protocol = typeof Protocol[keyof typeof Protocol] export type Protocol = (typeof Protocol)[keyof typeof Protocol];
export const SendingSampleRate = { export const SendingSampleRate = {
"48000": 48000, "48000": 48000,
"44100": 44100, "44100": 44100,
"24000": 24000 "24000": 24000,
} as const } as const;
export type SendingSampleRate = typeof SendingSampleRate[keyof typeof SendingSampleRate] export type SendingSampleRate = (typeof SendingSampleRate)[keyof typeof SendingSampleRate];
export const DownSamplingMode = { export const DownSamplingMode = {
"decimate": "decimate", decimate: "decimate",
"average": "average" average: "average",
} as const } as const;
export type DownSamplingMode = typeof DownSamplingMode[keyof typeof DownSamplingMode] export type DownSamplingMode = (typeof DownSamplingMode)[keyof typeof DownSamplingMode];
export type WorkletNodeSetting = { export type WorkletNodeSetting = {
serverUrl: string, serverUrl: string;
protocol: Protocol, protocol: Protocol;
sendingSampleRate: SendingSampleRate, sendingSampleRate: SendingSampleRate;
inputChunkNum: number, inputChunkNum: number;
downSamplingMode: DownSamplingMode, downSamplingMode: DownSamplingMode;
} };
/////////////////////// ///////////////////////
// クライアントセッティング // クライアントセッティング
/////////////////////// ///////////////////////
export const SampleRate = { export const SampleRate = {
"48000": 48000, "48000": 48000,
} as const } as const;
export type SampleRate = typeof SampleRate[keyof typeof SampleRate] export type SampleRate = (typeof SampleRate)[keyof typeof SampleRate];
export type VoiceChangerClientSetting = { export type VoiceChangerClientSetting = {
audioInput: string | MediaStream | null, audioInput: string | MediaStream | null;
sampleRate: SampleRate, // 48000Hz sampleRate: SampleRate; // 48000Hz
echoCancel: boolean, echoCancel: boolean;
noiseSuppression: boolean, noiseSuppression: boolean;
noiseSuppression2: boolean noiseSuppression2: boolean;
inputGain: number inputGain: number;
outputGain: number outputGain: number;
monitorGain: number monitorGain: number;
passThroughConfirmationSkip: boolean passThroughConfirmationSkip: boolean;
} };
/////////////////////// ///////////////////////
// Client セッティング // Client セッティング
/////////////////////// ///////////////////////
export type ClientSetting = { export type ClientSetting = {
workletSetting: WorkletSetting workletSetting: WorkletSetting;
workletNodeSetting: WorkletNodeSetting workletNodeSetting: WorkletNodeSetting;
voiceChangerClientSetting: VoiceChangerClientSetting voiceChangerClientSetting: VoiceChangerClientSetting;
} };
export const DefaultClientSettng: ClientSetting = { export const DefaultClientSettng: ClientSetting = {
workletSetting: { workletSetting: {
numTrancateTreshold: 100, numTrancateTreshold: 100,
volTrancateThreshold: 0.0005, volTrancateThreshold: 0.0005,
volTrancateLength: 32 volTrancateLength: 32,
}, },
workletNodeSetting: { workletNodeSetting: {
serverUrl: "", serverUrl: "",
protocol: "sio", protocol: "sio",
sendingSampleRate: 48000, sendingSampleRate: 48000,
inputChunkNum: 48, inputChunkNum: 48,
downSamplingMode: "average" downSamplingMode: "average",
}, },
voiceChangerClientSetting: { voiceChangerClientSetting: {
audioInput: null, audioInput: null,
@ -526,10 +512,9 @@ export const DefaultClientSettng: ClientSetting = {
inputGain: 1.0, inputGain: 1.0,
outputGain: 1.0, outputGain: 1.0,
monitorGain: 1.0, monitorGain: 1.0,
passThroughConfirmationSkip: false passThroughConfirmationSkip: false,
} },
} };
//////////////////////////////////// ////////////////////////////////////
// Exceptions // Exceptions
@ -538,36 +523,33 @@ export const VOICE_CHANGER_CLIENT_EXCEPTION = {
ERR_SIO_CONNECT_FAILED: "ERR_SIO_CONNECT_FAILED", ERR_SIO_CONNECT_FAILED: "ERR_SIO_CONNECT_FAILED",
ERR_SIO_INVALID_RESPONSE: "ERR_SIO_INVALID_RESPONSE", ERR_SIO_INVALID_RESPONSE: "ERR_SIO_INVALID_RESPONSE",
ERR_REST_INVALID_RESPONSE: "ERR_REST_INVALID_RESPONSE", ERR_REST_INVALID_RESPONSE: "ERR_REST_INVALID_RESPONSE",
ERR_MIC_STREAM_NOT_INITIALIZED: "ERR_MIC_STREAM_NOT_INITIALIZED" ERR_MIC_STREAM_NOT_INITIALIZED: "ERR_MIC_STREAM_NOT_INITIALIZED",
} as const;
} as const export type VOICE_CHANGER_CLIENT_EXCEPTION = (typeof VOICE_CHANGER_CLIENT_EXCEPTION)[keyof typeof VOICE_CHANGER_CLIENT_EXCEPTION];
export type VOICE_CHANGER_CLIENT_EXCEPTION = typeof VOICE_CHANGER_CLIENT_EXCEPTION[keyof typeof VOICE_CHANGER_CLIENT_EXCEPTION]
//////////////////////////////////// ////////////////////////////////////
// indexedDB // indexedDB
//////////////////////////////////// ////////////////////////////////////
export const INDEXEDDB_DB_APP_NAME = "INDEXEDDB_KEY_VOICE_CHANGER" export const INDEXEDDB_DB_APP_NAME = "INDEXEDDB_KEY_VOICE_CHANGER";
export const INDEXEDDB_DB_NAME = "INDEXEDDB_KEY_VOICE_CHANGER_DB" export const INDEXEDDB_DB_NAME = "INDEXEDDB_KEY_VOICE_CHANGER_DB";
export const INDEXEDDB_KEY_CLIENT = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_CLIENT" export const INDEXEDDB_KEY_CLIENT = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_CLIENT";
export const INDEXEDDB_KEY_SERVER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_SERVER" export const INDEXEDDB_KEY_SERVER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_SERVER";
export const INDEXEDDB_KEY_MODEL_DATA = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_MODEL_DATA" export const INDEXEDDB_KEY_MODEL_DATA = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_MODEL_DATA";
// ONNX // ONNX
export type OnnxExporterInfo = { export type OnnxExporterInfo = {
"status": string status: string;
"path": string path: string;
"filename": string filename: string;
} };
// Merge // Merge
export type MergeElement = { export type MergeElement = {
slotIndex: number slotIndex: number;
strength: number strength: number;
} };
export type MergeModelRequest = { export type MergeModelRequest = {
voiceChangerType: VoiceChangerType voiceChangerType: VoiceChangerType;
command: "mix", command: "mix";
files: MergeElement[] files: MergeElement[];
} };

2
client/lib/src/exceptions.ts
View File

@ -1,5 +1,5 @@
export class ModelLoadException extends Error { export class ModelLoadException extends Error {
public causeFileType: string = "" public causeFileType: string = "";
constructor(causeFileType: string) { constructor(causeFileType: string) {
super(`Model Load Exception:${causeFileType}`); super(`Model Load Exception:${causeFileType}`);
this.causeFileType = causeFileType; this.causeFileType = causeFileType;

299
client/lib/src/hooks/useClient.ts
View File

@ -1,269 +1,258 @@
import { useEffect, useMemo, useRef, useState } from "react" import { useEffect, useMemo, useRef, useState } from "react";
import { VoiceChangerClient } from "../VoiceChangerClient" import { VoiceChangerClient } from "../VoiceChangerClient";
import { useClientSetting } from "./useClientSetting" import { useClientSetting } from "./useClientSetting";
import { IndexedDBStateAndMethod, useIndexedDB } from "./useIndexedDB" import { IndexedDBStateAndMethod, useIndexedDB } from "./useIndexedDB";
import { ServerSettingState, useServerSetting } from "./useServerSetting" import { ServerSettingState, useServerSetting } from "./useServerSetting";
import { useWorkletNodeSetting } from "./useWorkletNodeSetting" import { useWorkletNodeSetting } from "./useWorkletNodeSetting";
import { useWorkletSetting } from "./useWorkletSetting" import { useWorkletSetting } from "./useWorkletSetting";
import { ClientSetting, DefaultClientSettng, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "../const" import { ClientSetting, DefaultClientSettng, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "../const";
export type UseClientProps = { export type UseClientProps = {
audioContext: AudioContext | null audioContext: AudioContext | null;
} };
export type ClientState = { export type ClientState = {
initialized: boolean initialized: boolean;
setting: ClientSetting, setting: ClientSetting;
// 各種設定I/Fへの参照 // 各種設定I/Fへの参照
setVoiceChangerClientSetting: (_voiceChangerClientSetting: VoiceChangerClientSetting) => void setVoiceChangerClientSetting: (_voiceChangerClientSetting: VoiceChangerClientSetting) => void;
setServerUrl: (url: string) => void; setServerUrl: (url: string) => void;
start: () => Promise<void> start: () => Promise<void>;
stop: () => Promise<void> stop: () => Promise<void>;
reloadClientSetting: () => Promise<void> reloadClientSetting: () => Promise<void>;
setWorkletNodeSetting: (_workletNodeSetting: WorkletNodeSetting) => void setWorkletNodeSetting: (_workletNodeSetting: WorkletNodeSetting) => void;
startOutputRecording: () => void startOutputRecording: () => void;
stopOutputRecording: () => Promise<Float32Array> stopOutputRecording: () => Promise<Float32Array>;
trancateBuffer: () => Promise<void> trancateBuffer: () => Promise<void>;
setWorkletSetting: (_workletSetting: WorkletSetting) => void setWorkletSetting: (_workletSetting: WorkletSetting) => void;
// workletSetting: WorkletSetting // workletSetting: WorkletSetting
// workletSetting: WorkletSettingState // workletSetting: WorkletSettingState
// clientSetting: ClientSettingState // clientSetting: ClientSettingState
// workletNodeSetting: WorkletNodeSettingState // workletNodeSetting: WorkletNodeSettingState
serverSetting: ServerSettingState serverSetting: ServerSettingState;
indexedDBState: IndexedDBStateAndMethod indexedDBState: IndexedDBStateAndMethod;
// モニタリングデータ // モニタリングデータ
bufferingTime: number; bufferingTime: number;
volume: number; volume: number;
performance: PerformanceData performance: PerformanceData;
updatePerformance: (() => Promise<void>) | null updatePerformance: (() => Promise<void>) | null;
// setClientType: (val: ClientType) => void // setClientType: (val: ClientType) => void
// 情報取得 // 情報取得
getInfo: () => Promise<void> getInfo: () => Promise<void>;
// 設定クリア // 設定クリア
clearSetting: () => Promise<void> clearSetting: () => Promise<void>;
// AudioOutputElement 設定 // AudioOutputElement 設定
setAudioOutputElementId: (elemId: string) => void setAudioOutputElementId: (elemId: string) => void;
setAudioMonitorElementId: (elemId: string) => void setAudioMonitorElementId: (elemId: string) => void;
ioErrorCount: number ioErrorCount: number;
resetIoErrorCount: () => void resetIoErrorCount: () => void;
} };
export type PerformanceData = { export type PerformanceData = {
responseTime: number responseTime: number;
preprocessTime: number preprocessTime: number;
mainprocessTime: number mainprocessTime: number;
postprocessTime: number postprocessTime: number;
} };
const InitialPerformanceData: PerformanceData = { const InitialPerformanceData: PerformanceData = {
responseTime: 0, responseTime: 0,
preprocessTime: 0, preprocessTime: 0,
mainprocessTime: 0, mainprocessTime: 0,
postprocessTime: 0 postprocessTime: 0,
} };
export const useClient = (props: UseClientProps): ClientState => { export const useClient = (props: UseClientProps): ClientState => {
const [initialized, setInitialized] = useState<boolean>(false);
const [initialized, setInitialized] = useState<boolean>(false) const [setting, setSetting] = useState<ClientSetting>(DefaultClientSettng);
const [setting, setSetting] = useState<ClientSetting>(DefaultClientSettng) // (1-1) クライアント
// (1-1) クライアント const voiceChangerClientRef = useRef<VoiceChangerClient | null>(null);
const voiceChangerClientRef = useRef<VoiceChangerClient | null>(null) const [voiceChangerClient, setVoiceChangerClient] = useState<VoiceChangerClient | null>(voiceChangerClientRef.current);
const [voiceChangerClient, setVoiceChangerClient] = useState<VoiceChangerClient | null>(voiceChangerClientRef.current)
//// クライアント初期化待ち用フラグ //// クライアント初期化待ち用フラグ
const initializedResolveRef = useRef<(value: void | PromiseLike<void>) => void>() const initializedResolveRef = useRef<(value: void | PromiseLike<void>) => void>();
const initializedPromise = useMemo(() => { const initializedPromise = useMemo(() => {
return new Promise<void>((resolve) => { return new Promise<void>((resolve) => {
initializedResolveRef.current = resolve initializedResolveRef.current = resolve;
}) });
}, []) }, []);
// (1-2) 各種設定I/F // (1-2) 各種設定I/F
const voiceChangerClientSetting = useClientSetting({ voiceChangerClient, voiceChangerClientSetting: setting.voiceChangerClientSetting }) const voiceChangerClientSetting = useClientSetting({ voiceChangerClient, voiceChangerClientSetting: setting.voiceChangerClientSetting });
const workletNodeSetting = useWorkletNodeSetting({ voiceChangerClient: voiceChangerClient, workletNodeSetting: setting.workletNodeSetting }) const workletNodeSetting = useWorkletNodeSetting({ voiceChangerClient: voiceChangerClient, workletNodeSetting: setting.workletNodeSetting });
useWorkletSetting({ voiceChangerClient, workletSetting: setting.workletSetting }) useWorkletSetting({ voiceChangerClient, workletSetting: setting.workletSetting });
const serverSetting = useServerSetting({ voiceChangerClient }) const serverSetting = useServerSetting({ voiceChangerClient });
const indexedDBState = useIndexedDB({ clientType: null }) const indexedDBState = useIndexedDB({ clientType: null });
// (1-3) モニタリングデータ // (1-3) モニタリングデータ
const [bufferingTime, setBufferingTime] = useState<number>(0) const [bufferingTime, setBufferingTime] = useState<number>(0);
const [performance, setPerformance] = useState<PerformanceData>(InitialPerformanceData) const [performance, setPerformance] = useState<PerformanceData>(InitialPerformanceData);
const [volume, setVolume] = useState<number>(0) const [volume, setVolume] = useState<number>(0);
const [ioErrorCount, setIoErrorCount] = useState<number>(0) const [ioErrorCount, setIoErrorCount] = useState<number>(0);
//// Server Audio Deviceを使うとき、モニタリングデータはpolling //// Server Audio Deviceを使うとき、モニタリングデータはpolling
const updatePerformance = useMemo(() => { const updatePerformance = useMemo(() => {
if (!voiceChangerClientRef.current) { if (!voiceChangerClientRef.current) {
return null return null;
} }
return async () => { return async () => {
if (voiceChangerClientRef.current) { if (voiceChangerClientRef.current) {
const performance = await voiceChangerClientRef.current!.getPerformance() const performance = await voiceChangerClientRef.current!.getPerformance();
const responseTime = performance[0] const responseTime = performance[0];
const preprocessTime = performance[1] const preprocessTime = performance[1];
const mainprocessTime = performance[2] const mainprocessTime = performance[2];
const postprocessTime = performance[3] const postprocessTime = performance[3];
setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime }) setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime });
} else { } else {
const responseTime = 0 const responseTime = 0;
const preprocessTime = 0 const preprocessTime = 0;
const mainprocessTime = 0 const mainprocessTime = 0;
const postprocessTime = 0 const postprocessTime = 0;
setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime }) setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime });
} }
} };
}, [voiceChangerClientRef.current]) }, [voiceChangerClientRef.current]);
// (1-4) エラーステータス // (1-4) エラーステータス
const ioErrorCountRef = useRef<number>(0) const ioErrorCountRef = useRef<number>(0);
const resetIoErrorCount = () => { const resetIoErrorCount = () => {
ioErrorCountRef.current = 0 ioErrorCountRef.current = 0;
setIoErrorCount(ioErrorCountRef.current) setIoErrorCount(ioErrorCountRef.current);
} };
// 設定データ管理 // 設定データ管理
const { setItem, getItem } = useIndexedDB({ clientType: null }) const { setItem, getItem } = useIndexedDB({ clientType: null });
// 設定データの更新と保存 // 設定データの更新と保存
const _setSetting = (_setting: ClientSetting) => { const _setSetting = (_setting: ClientSetting) => {
const storeData = { ..._setting } const storeData = { ..._setting };
storeData.voiceChangerClientSetting = { ...storeData.voiceChangerClientSetting } storeData.voiceChangerClientSetting = { ...storeData.voiceChangerClientSetting };
if (typeof storeData.voiceChangerClientSetting.audioInput != "string") { if (typeof storeData.voiceChangerClientSetting.audioInput != "string") {
storeData.voiceChangerClientSetting.audioInput = "none" storeData.voiceChangerClientSetting.audioInput = "none";
} }
setItem("clientSetting", storeData) setItem("clientSetting", storeData);
setSetting(_setting) setSetting(_setting);
} };
// 設定データ初期化 // 設定データ初期化
useEffect(() => { useEffect(() => {
if (!voiceChangerClient) { if (!voiceChangerClient) {
return return;
} }
const loadCache = async () => { const loadCache = async () => {
const _setting = await getItem("clientSetting") as ClientSetting const _setting = (await getItem("clientSetting")) as ClientSetting;
if (_setting) { if (_setting) {
setSetting(_setting) setSetting(_setting);
serverSetting.reloadServerInfo() serverSetting.reloadServerInfo();
} }
} };
loadCache() loadCache();
}, [voiceChangerClient]) }, [voiceChangerClient]);
// (2-1) クライアント初期化処理 // (2-1) クライアント初期化処理
useEffect(() => { useEffect(() => {
const initialized = async () => { const initialized = async () => {
if (!props.audioContext) { if (!props.audioContext) {
return return;
} }
const voiceChangerClient = new VoiceChangerClient(props.audioContext, true, { const voiceChangerClient = new VoiceChangerClient(props.audioContext, true, {
notifySendBufferingTime: (val: number) => { notifySendBufferingTime: (val: number) => {
setBufferingTime(val) setBufferingTime(val);
}, },
notifyResponseTime: (val: number, perf?: number[]) => { notifyResponseTime: (val: number, perf?: number[]) => {
const responseTime = val const responseTime = val;
const preprocessTime = perf ? Math.ceil(perf[0] * 1000) : 0 const preprocessTime = perf ? Math.ceil(perf[0] * 1000) : 0;
const mainprocessTime = perf ? Math.ceil(perf[1] * 1000) : 0 const mainprocessTime = perf ? Math.ceil(perf[1] * 1000) : 0;
const postprocessTime = perf ? Math.ceil(perf[2] * 1000) : 0 const postprocessTime = perf ? Math.ceil(perf[2] * 1000) : 0;
setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime }) setPerformance({ responseTime, preprocessTime, mainprocessTime, postprocessTime });
}, },
notifyException: (mes: string) => { notifyException: (mes: string) => {
if (mes.length > 0) { if (mes.length > 0) {
console.log(`error:${mes}`) console.log(`error:${mes}`);
ioErrorCountRef.current += 1 ioErrorCountRef.current += 1;
setIoErrorCount(ioErrorCountRef.current) setIoErrorCount(ioErrorCountRef.current);
} }
}, },
notifyVolume: (vol: number) => { notifyVolume: (vol: number) => {
setVolume(vol) setVolume(vol);
} },
}) });
await voiceChangerClient.isInitialized() await voiceChangerClient.isInitialized();
voiceChangerClientRef.current = voiceChangerClient voiceChangerClientRef.current = voiceChangerClient;
setVoiceChangerClient(voiceChangerClientRef.current) setVoiceChangerClient(voiceChangerClientRef.current);
console.log("[useClient] client initialized") console.log("[useClient] client initialized");
// const audio = document.getElementById(props.audioOutputElementId) as HTMLAudioElement // const audio = document.getElementById(props.audioOutputElementId) as HTMLAudioElement
// audio.srcObject = voiceChangerClientRef.current.stream // audio.srcObject = voiceChangerClientRef.current.stream
// audio.play() // audio.play()
initializedResolveRef.current!() initializedResolveRef.current!();
setInitialized(true) setInitialized(true);
} };
initialized() initialized();
}, [props.audioContext]) }, [props.audioContext]);
const setAudioOutputElementId = (elemId: string) => { const setAudioOutputElementId = (elemId: string) => {
if (!voiceChangerClientRef.current) { if (!voiceChangerClientRef.current) {
console.warn("[voiceChangerClient] is not ready for set audio output.") console.warn("[voiceChangerClient] is not ready for set audio output.");
return return;
} }
const audio = document.getElementById(elemId) as HTMLAudioElement const audio = document.getElementById(elemId) as HTMLAudioElement;
if (audio.paused) { if (audio.paused) {
audio.srcObject = voiceChangerClientRef.current.stream audio.srcObject = voiceChangerClientRef.current.stream;
audio.play() audio.play();
} }
} };
const setAudioMonitorElementId = (elemId: string) => { const setAudioMonitorElementId = (elemId: string) => {
if (!voiceChangerClientRef.current) { if (!voiceChangerClientRef.current) {
console.warn("[voiceChangerClient] is not ready for set audio output.") console.warn("[voiceChangerClient] is not ready for set audio output.");
return return;
} }
const audio = document.getElementById(elemId) as HTMLAudioElement const audio = document.getElementById(elemId) as HTMLAudioElement;
if (audio.paused) { if (audio.paused) {
audio.srcObject = voiceChangerClientRef.current.monitorStream audio.srcObject = voiceChangerClientRef.current.monitorStream;
audio.play() audio.play();
} }
} };
// (2-2) 情報リロード // (2-2) 情報リロード
const getInfo = useMemo(() => { const getInfo = useMemo(() => {
return async () => { return async () => {
await initializedPromise await initializedPromise;
await voiceChangerClientSetting.reloadClientSetting() // 実質的な処理の意味はない await voiceChangerClientSetting.reloadClientSetting(); // 実質的な処理の意味はない
await serverSetting.reloadServerInfo() await serverSetting.reloadServerInfo();
} };
}, [voiceChangerClientSetting.reloadClientSetting, serverSetting.reloadServerInfo]) }, [voiceChangerClientSetting.reloadClientSetting, serverSetting.reloadServerInfo]);
const clearSetting = async () => { const clearSetting = async () => {
// TBD // TBD
} };
// 設定変更 // 設定変更
const setVoiceChangerClientSetting = (_voiceChangerClientSetting: VoiceChangerClientSetting) => { const setVoiceChangerClientSetting = (_voiceChangerClientSetting: VoiceChangerClientSetting) => {
setting.voiceChangerClientSetting = _voiceChangerClientSetting setting.voiceChangerClientSetting = _voiceChangerClientSetting;
console.log("setting.voiceChangerClientSetting", setting.voiceChangerClientSetting) console.log("setting.voiceChangerClientSetting", setting.voiceChangerClientSetting);
// workletSettingIF.setSetting(_workletSetting) // workletSettingIF.setSetting(_workletSetting)
_setSetting({ ...setting }) _setSetting({ ...setting });
} };
const setWorkletNodeSetting = (_workletNodeSetting: WorkletNodeSetting) => { const setWorkletNodeSetting = (_workletNodeSetting: WorkletNodeSetting) => {
setting.workletNodeSetting = _workletNodeSetting setting.workletNodeSetting = _workletNodeSetting;
console.log("setting.workletNodeSetting", setting.workletNodeSetting) console.log("setting.workletNodeSetting", setting.workletNodeSetting);
// workletSettingIF.setSetting(_workletSetting) // workletSettingIF.setSetting(_workletSetting)
_setSetting({ ...setting }) _setSetting({ ...setting });
} };
const setWorkletSetting = (_workletSetting: WorkletSetting) => { const setWorkletSetting = (_workletSetting: WorkletSetting) => {
setting.workletSetting = _workletSetting setting.workletSetting = _workletSetting;
console.log("setting.workletSetting", setting.workletSetting) console.log("setting.workletSetting", setting.workletSetting);
// workletSettingIF.setSetting(_workletSetting) // workletSettingIF.setSetting(_workletSetting)
_setSetting({ ...setting }) _setSetting({ ...setting });
} };
return { return {
initialized, initialized,
@ -302,6 +291,6 @@ export const useClient = (props: UseClientProps): ClientState => {
setAudioMonitorElementId, setAudioMonitorElementId,
ioErrorCount, ioErrorCount,
resetIoErrorCount resetIoErrorCount,
} };
} };

81
client/lib/src/hooks/useClientSetting.ts
View File

@ -1,49 +1,46 @@
import { useState, useMemo, useEffect } from "react" import { useState, useMemo, useEffect } from "react";
import { VoiceChangerClientSetting } from "../const" import { VoiceChangerClientSetting } from "../const";
import { VoiceChangerClient } from "../VoiceChangerClient" import { VoiceChangerClient } from "../VoiceChangerClient";
export type UseClientSettingProps = { export type UseClientSettingProps = {
voiceChangerClient: VoiceChangerClient | null voiceChangerClient: VoiceChangerClient | null;
voiceChangerClientSetting: VoiceChangerClientSetting voiceChangerClientSetting: VoiceChangerClientSetting;
} };
export type ClientSettingState = { export type ClientSettingState = {
setServerUrl: (url: string) => void; setServerUrl: (url: string) => void;
start: () => Promise<void> start: () => Promise<void>;
stop: () => Promise<void> stop: () => Promise<void>;
reloadClientSetting: () => Promise<void> reloadClientSetting: () => Promise<void>;
} };
export const useClientSetting = (props: UseClientSettingProps): ClientSettingState => { export const useClientSetting = (props: UseClientSettingProps): ClientSettingState => {
// 更新比較用 // 更新比較用
const [voiceChangerClientSetting, setVoiceChangerClientSetting] = useState<VoiceChangerClientSetting>(props.voiceChangerClientSetting) const [voiceChangerClientSetting, setVoiceChangerClientSetting] = useState<VoiceChangerClientSetting>(props.voiceChangerClientSetting);
useEffect(() => { useEffect(() => {
const update = async () => { const update = async () => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
for (let k in props.voiceChangerClientSetting) { for (let k in props.voiceChangerClientSetting) {
const cur_v = voiceChangerClientSetting[k as keyof VoiceChangerClientSetting] const cur_v = voiceChangerClientSetting[k as keyof VoiceChangerClientSetting];
const new_v = props.voiceChangerClientSetting[k as keyof VoiceChangerClientSetting] const new_v = props.voiceChangerClientSetting[k as keyof VoiceChangerClientSetting];
if (cur_v != new_v) { if (cur_v != new_v) {
setVoiceChangerClientSetting(props.voiceChangerClientSetting) setVoiceChangerClientSetting(props.voiceChangerClientSetting);
await props.voiceChangerClient.updateClientSetting(props.voiceChangerClientSetting) await props.voiceChangerClient.updateClientSetting(props.voiceChangerClientSetting);
break break;
} }
} }
} };
update() update();
}, [props.voiceChangerClient, props.voiceChangerClientSetting]) }, [props.voiceChangerClient, props.voiceChangerClientSetting]);
const setServerUrl = useMemo(() => { const setServerUrl = useMemo(() => {
return (url: string) => { return (url: string) => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
props.voiceChangerClient.setServerUrl(url, true) props.voiceChangerClient.setServerUrl(url, true);
} };
}, [props.voiceChangerClient]) }, [props.voiceChangerClient]);
////////////// //////////////
// 操作 // 操作
@ -51,29 +48,29 @@ export const useClientSetting = (props: UseClientSettingProps): ClientSettingSta
// (1) start // (1) start
const start = useMemo(() => { const start = useMemo(() => {
return async () => { return async () => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
await props.voiceChangerClient.start() await props.voiceChangerClient.start();
} };
}, [props.voiceChangerClient]) }, [props.voiceChangerClient]);
// (2) stop // (2) stop
const stop = useMemo(() => { const stop = useMemo(() => {
return async () => { return async () => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
await props.voiceChangerClient.stop() await props.voiceChangerClient.stop();
} };
}, [props.voiceChangerClient]) }, [props.voiceChangerClient]);
const reloadClientSetting = useMemo(() => { const reloadClientSetting = useMemo(() => {
return async () => { return async () => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
await props.voiceChangerClient.getClientSettings() await props.voiceChangerClient.getClientSettings();
} };
}, [props.voiceChangerClient]) }, [props.voiceChangerClient]);
return { return {
setServerUrl, setServerUrl,
start, start,
stop, stop,
reloadClientSetting reloadClientSetting,
} };
} };

68
client/lib/src/hooks/useIndexedDB.ts
View File

@ -3,67 +3,65 @@ import { useMemo } from "react";
import { INDEXEDDB_DB_APP_NAME, INDEXEDDB_DB_NAME } from "../const"; import { INDEXEDDB_DB_APP_NAME, INDEXEDDB_DB_NAME } from "../const";
export type UseIndexedDBProps = { export type UseIndexedDBProps = {
clientType: null clientType: null;
} };
export type IndexedDBState = { export type IndexedDBState = {
dummy: string dummy: string;
} };
export type IndexedDBStateAndMethod = IndexedDBState & { export type IndexedDBStateAndMethod = IndexedDBState & {
setItem: (key: string, value: unknown) => Promise<void>, setItem: (key: string, value: unknown) => Promise<void>;
getItem: (key: string) => Promise<unknown> getItem: (key: string) => Promise<unknown>;
removeItem: (key: string) => Promise<void> removeItem: (key: string) => Promise<void>;
removeDB: () => Promise<void> removeDB: () => Promise<void>;
} };
export const useIndexedDB = (props: UseIndexedDBProps): IndexedDBStateAndMethod => { export const useIndexedDB = (props: UseIndexedDBProps): IndexedDBStateAndMethod => {
const clientType = props.clientType || "default" const clientType = props.clientType || "default";
localForage.config({ localForage.config({
driver: localForage.INDEXEDDB, driver: localForage.INDEXEDDB,
name: INDEXEDDB_DB_APP_NAME, name: INDEXEDDB_DB_APP_NAME,
version: 1.0, version: 1.0,
storeName: `${INDEXEDDB_DB_NAME}`, storeName: `${INDEXEDDB_DB_NAME}`,
description: 'appStorage' description: "appStorage",
});
})
const setItem = useMemo(() => { const setItem = useMemo(() => {
return async (key: string, value: unknown) => { return async (key: string, value: unknown) => {
const clientKey = `${clientType}_${key}` const clientKey = `${clientType}_${key}`;
await localForage.setItem(clientKey, value) await localForage.setItem(clientKey, value);
} };
}, [props.clientType]) }, [props.clientType]);
const getItem = useMemo(() => { const getItem = useMemo(() => {
return async (key: string) => { return async (key: string) => {
const clientKey = `${clientType}_${key}` const clientKey = `${clientType}_${key}`;
return await localForage.getItem(clientKey) return await localForage.getItem(clientKey);
} };
}, [props.clientType]) }, [props.clientType]);
const removeItem = useMemo(() => { const removeItem = useMemo(() => {
return async (key: string) => { return async (key: string) => {
const clientKey = `${clientType}_${key}` const clientKey = `${clientType}_${key}`;
console.log("remove key:", clientKey) console.log("remove key:", clientKey);
return await localForage.removeItem(clientKey) return await localForage.removeItem(clientKey);
} };
}, [props.clientType]) }, [props.clientType]);
const removeDB = useMemo(() => { const removeDB = useMemo(() => {
return async () => { return async () => {
const keys = await localForage.keys() const keys = await localForage.keys();
for (const key of keys) { for (const key of keys) {
console.log("remove key:", key) console.log("remove key:", key);
await localForage.removeItem(key) await localForage.removeItem(key);
} }
} };
}, [props.clientType]) }, [props.clientType]);
return { return {
dummy: "", dummy: "",
setItem, setItem,
getItem, getItem,
removeItem, removeItem,
removeDB removeDB,
} };
} };

265
client/lib/src/hooks/useServerSetting.ts
View File

@ -1,215 +1,208 @@
import { useState, useMemo } from "react" import { useState, useMemo } from "react";
import { VoiceChangerServerSetting, ServerInfo, ServerSettingKey, OnnxExporterInfo, MergeModelRequest, VoiceChangerType, DefaultServerSetting } from "../const" import { VoiceChangerServerSetting, ServerInfo, ServerSettingKey, OnnxExporterInfo, MergeModelRequest, VoiceChangerType, DefaultServerSetting } from "../const";
import { VoiceChangerClient } from "../VoiceChangerClient" import { VoiceChangerClient } from "../VoiceChangerClient";
export const ModelAssetName = { export const ModelAssetName = {
iconFile: "iconFile" iconFile: "iconFile",
} as const } as const;
export type ModelAssetName = typeof ModelAssetName[keyof typeof ModelAssetName] export type ModelAssetName = (typeof ModelAssetName)[keyof typeof ModelAssetName];
export const ModelFileKind = { export const ModelFileKind = {
"mmvcv13Config": "mmvcv13Config", mmvcv13Config: "mmvcv13Config",
"mmvcv13Model": "mmvcv13Model", mmvcv13Model: "mmvcv13Model",
"mmvcv15Config": "mmvcv15Config", mmvcv15Config: "mmvcv15Config",
"mmvcv15Model": "mmvcv15Model", mmvcv15Model: "mmvcv15Model",
"mmvcv15Correspondence": "mmvcv15Correspondence", mmvcv15Correspondence: "mmvcv15Correspondence",
"soVitsSvc40Config": "soVitsSvc40Config", soVitsSvc40Config: "soVitsSvc40Config",
"soVitsSvc40Model": "soVitsSvc40Model", soVitsSvc40Model: "soVitsSvc40Model",
"soVitsSvc40Cluster": "soVitsSvc40Cluster", soVitsSvc40Cluster: "soVitsSvc40Cluster",
"rvcModel": "rvcModel", rvcModel: "rvcModel",
"rvcIndex": "rvcIndex", rvcIndex: "rvcIndex",
"ddspSvcModel": "ddspSvcModel", ddspSvcModel: "ddspSvcModel",
"ddspSvcModelConfig": "ddspSvcModelConfig", ddspSvcModelConfig: "ddspSvcModelConfig",
"ddspSvcDiffusion": "ddspSvcDiffusion", ddspSvcDiffusion: "ddspSvcDiffusion",
"ddspSvcDiffusionConfig": "ddspSvcDiffusionConfig", ddspSvcDiffusionConfig: "ddspSvcDiffusionConfig",
"diffusionSVCModel": "diffusionSVCModel", diffusionSVCModel: "diffusionSVCModel",
"beatriceModel": "beatriceModel", beatriceModel: "beatriceModel",
} as const;
} as const export type ModelFileKind = (typeof ModelFileKind)[keyof typeof ModelFileKind];
export type ModelFileKind = typeof ModelFileKind[keyof typeof ModelFileKind]
export type ModelFile = { export type ModelFile = {
file: File, file: File;
kind: ModelFileKind kind: ModelFileKind;
dir: string dir: string;
} };
export type ModelUploadSetting = { export type ModelUploadSetting = {
voiceChangerType: VoiceChangerType, voiceChangerType: VoiceChangerType;
slot: number slot: number;
isSampleMode: boolean isSampleMode: boolean;
sampleId: string | null sampleId: string | null;
files: ModelFile[] files: ModelFile[];
params: any params: any;
} };
export type ModelFileForServer = Omit<ModelFile, "file"> & { export type ModelFileForServer = Omit<ModelFile, "file"> & {
name: string, name: string;
kind: ModelFileKind kind: ModelFileKind;
} };
export type ModelUploadSettingForServer = Omit<ModelUploadSetting, "files"> & { export type ModelUploadSettingForServer = Omit<ModelUploadSetting, "files"> & {
files: ModelFileForServer[] files: ModelFileForServer[];
} };
type AssetUploadSetting = { type AssetUploadSetting = {
slot: number slot: number;
name: ModelAssetName name: ModelAssetName;
file: string file: string;
} };
export type UseServerSettingProps = { export type UseServerSettingProps = {
voiceChangerClient: VoiceChangerClient | null voiceChangerClient: VoiceChangerClient | null;
} };
export type ServerSettingState = { export type ServerSettingState = {
serverSetting: ServerInfo serverSetting: ServerInfo;
updateServerSettings: (setting: ServerInfo) => Promise<void> updateServerSettings: (setting: ServerInfo) => Promise<void>;
reloadServerInfo: () => Promise<void>; reloadServerInfo: () => Promise<void>;
uploadModel: (setting: ModelUploadSetting) => Promise<void> uploadModel: (setting: ModelUploadSetting) => Promise<void>;
uploadProgress: number uploadProgress: number;
isUploading: boolean isUploading: boolean;
getOnnx: () => Promise<OnnxExporterInfo> getOnnx: () => Promise<OnnxExporterInfo>;
mergeModel: (request: MergeModelRequest) => Promise<ServerInfo> mergeModel: (request: MergeModelRequest) => Promise<ServerInfo>;
updateModelDefault: () => Promise<ServerInfo> updateModelDefault: () => Promise<ServerInfo>;
updateModelInfo: (slot: number, key: string, val: string) => Promise<ServerInfo> updateModelInfo: (slot: number, key: string, val: string) => Promise<ServerInfo>;
uploadAssets: (slot: number, name: ModelAssetName, file: File) => Promise<void> uploadAssets: (slot: number, name: ModelAssetName, file: File) => Promise<void>;
} };
export const useServerSetting = (props: UseServerSettingProps): ServerSettingState => { export const useServerSetting = (props: UseServerSettingProps): ServerSettingState => {
const [serverSetting, setServerSetting] = useState<ServerInfo>(DefaultServerSetting) const [serverSetting, setServerSetting] = useState<ServerInfo>(DefaultServerSetting);
////////////// //////////////
// 設定 // 設定
///////////// /////////////
const updateServerSettings = useMemo(() => { const updateServerSettings = useMemo(() => {
return async (setting: ServerInfo) => { return async (setting: ServerInfo) => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
for (let i = 0; i < Object.values(ServerSettingKey).length; i++) { for (let i = 0; i < Object.values(ServerSettingKey).length; i++) {
const k = Object.values(ServerSettingKey)[i] as keyof VoiceChangerServerSetting const k = Object.values(ServerSettingKey)[i] as keyof VoiceChangerServerSetting;
const cur_v = serverSetting[k] const cur_v = serverSetting[k];
const new_v = setting[k] const new_v = setting[k];
if (cur_v != new_v) { if (cur_v != new_v) {
const res = await props.voiceChangerClient.updateServerSettings(k, "" + new_v) const res = await props.voiceChangerClient.updateServerSettings(k, "" + new_v);
setServerSetting(res) setServerSetting(res);
} }
} }
} };
}, [props.voiceChangerClient, serverSetting]) }, [props.voiceChangerClient, serverSetting]);
////////////// //////////////
// 操作 // 操作
///////////// /////////////
const [uploadProgress, setUploadProgress] = useState<number>(0) const [uploadProgress, setUploadProgress] = useState<number>(0);
const [isUploading, setIsUploading] = useState<boolean>(false) const [isUploading, setIsUploading] = useState<boolean>(false);
// (e) モデルアップロード // (e) モデルアップロード
const _uploadFile2 = useMemo(() => { const _uploadFile2 = useMemo(() => {
return async (file: File, onprogress: (progress: number, end: boolean) => void, dir: string = "") => { return async (file: File, onprogress: (progress: number, end: boolean) => void, dir: string = "") => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
const num = await props.voiceChangerClient.uploadFile2(dir, file, onprogress) const num = await props.voiceChangerClient.uploadFile2(dir, file, onprogress);
const res = await props.voiceChangerClient.concatUploadedFile(dir + file.name, num) const res = await props.voiceChangerClient.concatUploadedFile(dir + file.name, num);
console.log("uploaded", num, res) console.log("uploaded", num, res);
} };
}, [props.voiceChangerClient]) }, [props.voiceChangerClient]);
// 新しいアップローダ // 新しいアップローダ
const uploadModel = useMemo(() => { const uploadModel = useMemo(() => {
return async (setting: ModelUploadSetting) => { return async (setting: ModelUploadSetting) => {
if (!props.voiceChangerClient) { if (!props.voiceChangerClient) {
return return;
} }
setUploadProgress(0) setUploadProgress(0);
setIsUploading(true) setIsUploading(true);
if (setting.isSampleMode == false) { if (setting.isSampleMode == false) {
const progRate = 1 / setting.files.length const progRate = 1 / setting.files.length;
for (let i = 0; i < setting.files.length; i++) { for (let i = 0; i < setting.files.length; i++) {
const progOffset = 100 * i * progRate const progOffset = 100 * i * progRate;
await _uploadFile2(setting.files[i].file, (progress: number, _end: boolean) => { await _uploadFile2(
setUploadProgress(progress * progRate + progOffset) setting.files[i].file,
}, setting.files[i].dir) (progress: number, _end: boolean) => {
setUploadProgress(progress * progRate + progOffset);
},
setting.files[i].dir
);
} }
} }
const params: ModelUploadSettingForServer = { const params: ModelUploadSettingForServer = {
...setting, files: setting.files.map((f) => { return { name: f.file.name, kind: f.kind, dir: f.dir } }) ...setting,
} files: setting.files.map((f) => {
return { name: f.file.name, kind: f.kind, dir: f.dir };
}),
};
const loadPromise = props.voiceChangerClient.loadModel( const loadPromise = props.voiceChangerClient.loadModel(0, false, JSON.stringify(params));
0, await loadPromise;
false,
JSON.stringify(params),
)
await loadPromise
setUploadProgress(0) setUploadProgress(0);
setIsUploading(false) setIsUploading(false);
reloadServerInfo() reloadServerInfo();
};
} }, [props.voiceChangerClient]);
}, [props.voiceChangerClient])
const uploadAssets = useMemo(() => { const uploadAssets = useMemo(() => {
return async (slot: number, name: ModelAssetName, file: File) => { return async (slot: number, name: ModelAssetName, file: File) => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
await _uploadFile2(file, (progress: number, _end: boolean) => { await _uploadFile2(file, (progress: number, _end: boolean) => {
console.log(progress, _end) console.log(progress, _end);
}) });
const assetUploadSetting: AssetUploadSetting = { const assetUploadSetting: AssetUploadSetting = {
slot, slot,
name, name,
file: file.name file: file.name,
} };
await props.voiceChangerClient.uploadAssets(JSON.stringify(assetUploadSetting)) await props.voiceChangerClient.uploadAssets(JSON.stringify(assetUploadSetting));
reloadServerInfo() reloadServerInfo();
} };
}, [props.voiceChangerClient]) }, [props.voiceChangerClient]);
const reloadServerInfo = useMemo(() => { const reloadServerInfo = useMemo(() => {
return async () => { return async () => {
if (!props.voiceChangerClient) return;
if (!props.voiceChangerClient) return const res = await props.voiceChangerClient.getServerSettings();
const res = await props.voiceChangerClient.getServerSettings() setServerSetting(res);
setServerSetting(res) };
} }, [props.voiceChangerClient]);
}, [props.voiceChangerClient])
const getOnnx = async () => { const getOnnx = async () => {
return props.voiceChangerClient!.getOnnx() return props.voiceChangerClient!.getOnnx();
} };
const mergeModel = async (request: MergeModelRequest) => { const mergeModel = async (request: MergeModelRequest) => {
const serverInfo = await props.voiceChangerClient!.mergeModel(request) const serverInfo = await props.voiceChangerClient!.mergeModel(request);
setServerSetting(serverInfo) setServerSetting(serverInfo);
return serverInfo return serverInfo;
} };
const updateModelDefault = async () => { const updateModelDefault = async () => {
const serverInfo = await props.voiceChangerClient!.updateModelDefault() const serverInfo = await props.voiceChangerClient!.updateModelDefault();
setServerSetting(serverInfo) setServerSetting(serverInfo);
return serverInfo return serverInfo;
} };
const updateModelInfo = async (slot: number, key: string, val: string) => { const updateModelInfo = async (slot: number, key: string, val: string) => {
const serverInfo = await props.voiceChangerClient!.updateModelInfo(slot, key, val) const serverInfo = await props.voiceChangerClient!.updateModelInfo(slot, key, val);
setServerSetting(serverInfo) setServerSetting(serverInfo);
return serverInfo return serverInfo;
} };
return { return {
serverSetting, serverSetting,
@ -223,6 +216,6 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
mergeModel, mergeModel,
updateModelDefault, updateModelDefault,
updateModelInfo, updateModelInfo,
uploadAssets uploadAssets,
} };
} };

70
client/lib/src/hooks/useWorkletNodeSetting.ts
View File

@ -1,67 +1,63 @@
import { useState, useMemo, useEffect } from "react" import { useState, useMemo, useEffect } from "react";
import { WorkletNodeSetting } from "../const"
import { VoiceChangerClient } from "../VoiceChangerClient"
import { WorkletNodeSetting } from "../const";
import { VoiceChangerClient } from "../VoiceChangerClient";
export type UseWorkletNodeSettingProps = { export type UseWorkletNodeSettingProps = {
voiceChangerClient: VoiceChangerClient | null voiceChangerClient: VoiceChangerClient | null;
workletNodeSetting: WorkletNodeSetting workletNodeSetting: WorkletNodeSetting;
} };
export type WorkletNodeSettingState = { export type WorkletNodeSettingState = {
startOutputRecording: () => void startOutputRecording: () => void;
stopOutputRecording: () => Promise<Float32Array> stopOutputRecording: () => Promise<Float32Array>;
trancateBuffer: () => Promise<void> trancateBuffer: () => Promise<void>;
} };
export const useWorkletNodeSetting = (props: UseWorkletNodeSettingProps): WorkletNodeSettingState => { export const useWorkletNodeSetting = (props: UseWorkletNodeSettingProps): WorkletNodeSettingState => {
// 更新比較用 // 更新比較用
const [workletNodeSetting, _setWorkletNodeSetting] = useState<WorkletNodeSetting>(props.workletNodeSetting) const [workletNodeSetting, _setWorkletNodeSetting] = useState<WorkletNodeSetting>(props.workletNodeSetting);
////////////// //////////////
// 設定 // 設定
///////////// /////////////
useEffect(() => { useEffect(() => {
if (!props.voiceChangerClient) return;
if (!props.voiceChangerClient) return
for (let k in props.workletNodeSetting) { for (let k in props.workletNodeSetting) {
const cur_v = workletNodeSetting[k as keyof WorkletNodeSetting] const cur_v = workletNodeSetting[k as keyof WorkletNodeSetting];
const new_v = props.workletNodeSetting[k as keyof WorkletNodeSetting] const new_v = props.workletNodeSetting[k as keyof WorkletNodeSetting];
if (cur_v != new_v) { if (cur_v != new_v) {
_setWorkletNodeSetting(props.workletNodeSetting) _setWorkletNodeSetting(props.workletNodeSetting);
props.voiceChangerClient.updateWorkletNodeSetting(props.workletNodeSetting) props.voiceChangerClient.updateWorkletNodeSetting(props.workletNodeSetting);
break break;
} }
} }
}, [props.voiceChangerClient, props.workletNodeSetting]);
}, [props.voiceChangerClient, props.workletNodeSetting])
const startOutputRecording = useMemo(() => { const startOutputRecording = useMemo(() => {
return () => { return () => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
props.voiceChangerClient.startOutputRecording() props.voiceChangerClient.startOutputRecording();
} };
}, [props.voiceChangerClient]) }, [props.voiceChangerClient]);
const stopOutputRecording = useMemo(() => { const stopOutputRecording = useMemo(() => {
return async () => { return async () => {
if (!props.voiceChangerClient) return new Float32Array() if (!props.voiceChangerClient) return new Float32Array();
return props.voiceChangerClient.stopOutputRecording() return props.voiceChangerClient.stopOutputRecording();
} };
}, [props.voiceChangerClient]) }, [props.voiceChangerClient]);
const trancateBuffer = useMemo(() => { const trancateBuffer = useMemo(() => {
return async () => { return async () => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
props.voiceChangerClient.trancateBuffer() props.voiceChangerClient.trancateBuffer();
} };
}, [props.voiceChangerClient]) }, [props.voiceChangerClient]);
return { return {
startOutputRecording, startOutputRecording,
stopOutputRecording, stopOutputRecording,
trancateBuffer trancateBuffer,
} };
} };

25
client/lib/src/hooks/useWorkletSetting.ts
View File

@ -1,27 +1,24 @@
import { useState, useEffect } from "react" import { useState, useEffect } from "react";
import { WorkletSetting } from "../const"; import { WorkletSetting } from "../const";
import { VoiceChangerClient } from "../VoiceChangerClient"; import { VoiceChangerClient } from "../VoiceChangerClient";
export type UseWorkletSettingProps = { export type UseWorkletSettingProps = {
voiceChangerClient: VoiceChangerClient | null voiceChangerClient: VoiceChangerClient | null;
workletSetting: WorkletSetting workletSetting: WorkletSetting;
} };
export type WorkletSettingState = { export type WorkletSettingState = {
// setting: WorkletSetting; // setting: WorkletSetting;
// setSetting: (setting: WorkletSetting) => void; // setSetting: (setting: WorkletSetting) => void;
};
}
export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSettingState => { export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSettingState => {
const [setting, _setSetting] = useState<WorkletSetting>(props.workletSetting) const [setting, _setSetting] = useState<WorkletSetting>(props.workletSetting);
useEffect(() => { useEffect(() => {
if (!props.voiceChangerClient) return if (!props.voiceChangerClient) return;
props.voiceChangerClient.configureWorklet(setting) props.voiceChangerClient.configureWorklet(setting);
}, [props.voiceChangerClient, props.workletSetting]) }, [props.voiceChangerClient, props.workletSetting]);
// // 設定 _setSettingがトリガでuseEffectが呼ばれて、workletに設定が飛ぶ // // 設定 _setSettingがトリガでuseEffectが呼ばれて、workletに設定が飛ぶ
// const setSetting = useMemo(() => { // const setSetting = useMemo(() => {
@ -34,5 +31,5 @@ export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSetting
return { return {
// setting, // setting,
// setSetting, // setSetting,
} };
} };

14
client/lib/src/index.ts
View File

@ -1,7 +1,7 @@
export * from "./const" export * from "./const";
export * from "./exceptions" export * from "./exceptions";
export * from "./VoiceChangerClient" export * from "./VoiceChangerClient";
export * from "./util" export * from "./util";
export * from "./hooks/useClient" export * from "./hooks/useClient";
export * from "./hooks/useIndexedDB" export * from "./hooks/useIndexedDB";
export * from "./hooks/useServerSetting" export * from "./hooks/useServerSetting";

40
client/lib/src/util.ts
View File

@ -17,32 +17,32 @@ export const fileSelector = async (regex: string) => {
const p = new Promise<File>((resolve, reject) => { const p = new Promise<File>((resolve, reject) => {
fileInput.onchange = (e) => { fileInput.onchange = (e) => {
if (e.target instanceof HTMLInputElement == false) { if (e.target instanceof HTMLInputElement == false) {
console.log("invalid target!", e.target) console.log("invalid target!", e.target);
reject("invalid target") reject("invalid target");
return null return null;
} }
const target = e.target as HTMLInputElement const target = e.target as HTMLInputElement;
if (!target.files || target.files.length == 0) { if (!target.files || target.files.length == 0) {
reject("no file selected") reject("no file selected");
return null return null;
} }
if (regex != "" && target.files[0].type.match(regex)) { if (regex != "" && target.files[0].type.match(regex)) {
reject(`not target file type ${target.files[0].type}`); reject(`not target file type ${target.files[0].type}`);
return null return null;
} }
resolve(target.files[0]) resolve(target.files[0]);
return null return null;
}; };
fileInput.click(); fileInput.click();
}); });
return await p return await p;
} };
export const fileSelectorAsDataURL = async (regex: string) => { export const fileSelectorAsDataURL = async (regex: string) => {
const f = await fileSelector(regex) const f = await fileSelector(regex);
if (!f) { if (!f) {
return f return f;
} }
const url = await new Promise<string>((resolve) => { const url = await new Promise<string>((resolve) => {
@ -52,15 +52,13 @@ export const fileSelectorAsDataURL = async (regex: string) => {
resolve(reader.result as string); resolve(reader.result as string);
}; };
reader.readAsDataURL(f); reader.readAsDataURL(f);
}) });
return url return url;
} };
export const validateUrl = (url: string) => { export const validateUrl = (url: string) => {
if (url?.endsWith("/")) { if (url?.endsWith("/")) {
return url.substring(0, url.length - 1) return url.substring(0, url.length - 1);
} }
return url return url;
} };

2
client/lib/src/utils/BlockingQueue.ts
View File

@ -38,4 +38,4 @@ export class BlockingQueue<T> {
get length() { get length() {
return this._promises.length - this._resolvers.length; return this._promises.length - this._resolvers.length;
} }
} }

54
client/lib/tsconfig.json
View File

@ -1,33 +1,33 @@
{ {
"compilerOptions": { "compilerOptions": {
"target": "ES2020", "target": "ES2020",
"declaration": true, "declaration": true,
"outDir": "./dist", "outDir": "./dist",
/* ファイル名の大文字小文字を区別 */ /* ファイル名の大文字小文字を区別 */
"forceConsistentCasingInFileNames": true, "forceConsistentCasingInFileNames": true,
/* 型チェック関係のオプション */ /* 型チェック関係のオプション */
"strict": true, "strict": true,
"noImplicitAny": true, "noImplicitAny": true,
"strictNullChecks": true, "strictNullChecks": true,
"noUnusedLocals": true, "noUnusedLocals": true,
"noUnusedParameters": true, "noUnusedParameters": true,
"noImplicitReturns": true, "noImplicitReturns": true,
/* Module解決方法 */ /* Module解決方法 */
"moduleResolution": "node", "moduleResolution": "node",
"esModuleInterop": true, "esModuleInterop": true,
// "isolatedModules": true, // "isolatedModules": true,
"allowSyntheticDefaultImports": true, "allowSyntheticDefaultImports": true,
// /* 型チェックだけさせたいので出力なし */ // /* 型チェックだけさせたいので出力なし */
// "noEmit": true, // "noEmit": true,
/* For avoid WebGL2 error */ /* For avoid WebGL2 error */
/* https://stackoverflow.com/questions/52846622/error-ts2430-interface-webglrenderingcontext-incorrectly-extends-interface-w */ /* https://stackoverflow.com/questions/52846622/error-ts2430-interface-webglrenderingcontext-incorrectly-extends-interface-w */
"skipLibCheck": true "skipLibCheck": true
}, },
/* tscコマンドで読み込むファイルを指定 */ /* tscコマンドで読み込むファイルを指定 */
"include": ["src/**/*.ts"], "include": ["src/**/*.ts"],
"exclude": ["node_modules"] "exclude": ["node_modules"]
} }

56
client/lib/tsconfig.worklet.json
View File

@ -1,33 +1,33 @@
{ {
"compilerOptions": { "compilerOptions": {
"target": "ES2020", "target": "ES2020",
"lib": ["ES2020"], "lib": ["ES2020"],
"outDir": "./worklet/dist", "outDir": "./worklet/dist",
"declaration": true, "declaration": true,
/* ファイル名の大文字小文字を区別 */ /* ファイル名の大文字小文字を区別 */
"forceConsistentCasingInFileNames": true, "forceConsistentCasingInFileNames": true,
/* 型チェック関係のオプション */ /* 型チェック関係のオプション */
"strict": true, "strict": true,
"noImplicitAny": true, "noImplicitAny": true,
"strictNullChecks": true, "strictNullChecks": true,
"noUnusedLocals": true, "noUnusedLocals": true,
"noUnusedParameters": true, "noUnusedParameters": true,
"noImplicitReturns": true, "noImplicitReturns": true,
/* Module解決方法 */ /* Module解決方法 */
"moduleResolution": "node", "moduleResolution": "node",
"esModuleInterop": true, "esModuleInterop": true,
// "isolatedModules": true, // "isolatedModules": true,
"allowSyntheticDefaultImports": true, "allowSyntheticDefaultImports": true,
// /* 型チェックだけさせたいので出力なし */ // /* 型チェックだけさせたいので出力なし */
// "noEmit": true, // "noEmit": true,
/* For avoid WebGL2 error */ /* For avoid WebGL2 error */
/* https://stackoverflow.com/questions/52846622/error-ts2430-interface-webglrenderingcontext-incorrectly-extends-interface-w */ /* https://stackoverflow.com/questions/52846622/error-ts2430-interface-webglrenderingcontext-incorrectly-extends-interface-w */
"skipLibCheck": true "skipLibCheck": true
}, },
/* tscコマンドで読み込むファイルを指定 */ /* tscコマンドで読み込むファイルを指定 */
"include": ["worklet/src/*.ts"], "include": ["worklet/src/*.ts"],
"exclude": ["node_modules"] "exclude": ["node_modules"]
} }

8
client/lib/webpack.common.js
View File

@ -5,8 +5,8 @@ module.exports = {
resolve: { resolve: {
extensions: [".ts", ".js"], extensions: [".ts", ".js"],
fallback: { fallback: {
"buffer": require.resolve("buffer/") buffer: require.resolve("buffer/"),
} },
}, },
module: { module: {
rules: [ rules: [
@ -32,11 +32,11 @@ module.exports = {
plugins: [ plugins: [
new webpack.ProvidePlugin({ new webpack.ProvidePlugin({
Buffer: ['buffer', 'Buffer'], Buffer: ["buffer", "Buffer"],
}), }),
], ],
externals: { externals: {
react: "react", react: "react",
"react-dom": "reactDOM", "react-dom": "reactDOM",
} },
}; };

8
client/lib/webpack.dev.js
View File

@ -1,6 +1,6 @@
const { merge } = require('webpack-merge'); const { merge } = require("webpack-merge");
const common = require('./webpack.common.js') const common = require("./webpack.common.js");
module.exports = merge(common, { module.exports = merge(common, {
mode: 'development', mode: "development",
}) });

8
client/lib/webpack.prod.js
View File

@ -1,6 +1,6 @@
const { merge } = require('webpack-merge'); const { merge } = require("webpack-merge");
const common = require('./webpack.common.js') const common = require("./webpack.common.js");
module.exports = merge(common, { module.exports = merge(common, {
mode: 'production', mode: "production",
}) });

2
client/lib/webpack.worklet.common.js
View File

@ -27,5 +27,5 @@ module.exports = {
], ],
}, },
], ],
} },
}; };

10
client/lib/webpack.worklet.dev.js
View File

@ -1,7 +1,7 @@
const { merge } = require('webpack-merge'); const { merge } = require("webpack-merge");
const common = require('./webpack.worklet.common.js') const common = require("./webpack.worklet.common.js");
const worklet = merge(common, { const worklet = merge(common, {
mode: 'development', mode: "development",
}) });
module.exports = [worklet]; module.exports = [worklet];

9
client/lib/webpack.worklet.prod.js
View File

@ -1,8 +1,7 @@
const { merge } = require('webpack-merge'); const { merge } = require("webpack-merge");
const common = require('./webpack.worklet.common.js') const common = require("./webpack.worklet.common.js");
const worklet = merge(common, { const worklet = merge(common, {
mode: 'production', mode: "production",
}) });
module.exports = [worklet]; module.exports = [worklet];

146
client/lib/worklet/src/voice-changer-worklet-processor.ts
View File

@ -1,50 +1,47 @@
export const RequestType = { export const RequestType = {
"voice": "voice", voice: "voice",
"config": "config", config: "config",
"start": "start", start: "start",
"stop": "stop", stop: "stop",
"trancateBuffer": "trancateBuffer", trancateBuffer: "trancateBuffer",
} as const } as const;
export type RequestType = typeof RequestType[keyof typeof RequestType] export type RequestType = (typeof RequestType)[keyof typeof RequestType];
export const ResponseType = { export const ResponseType = {
"volume": "volume", volume: "volume",
"inputData": "inputData", inputData: "inputData",
"start_ok": "start_ok", start_ok: "start_ok",
"stop_ok": "stop_ok", stop_ok: "stop_ok",
} as const } as const;
export type ResponseType = typeof ResponseType[keyof typeof ResponseType] export type ResponseType = (typeof ResponseType)[keyof typeof ResponseType];
export type VoiceChangerWorkletProcessorRequest = { export type VoiceChangerWorkletProcessorRequest = {
requestType: RequestType, requestType: RequestType;
voice: Float32Array, voice: Float32Array;
numTrancateTreshold: number numTrancateTreshold: number;
volTrancateThreshold: number volTrancateThreshold: number;
volTrancateLength: number volTrancateLength: number;
} };
export type VoiceChangerWorkletProcessorResponse = { export type VoiceChangerWorkletProcessorResponse = {
responseType: ResponseType, responseType: ResponseType;
volume?: number, volume?: number;
recordData?: Float32Array[] recordData?: Float32Array[];
inputData?: Float32Array inputData?: Float32Array;
} };
class VoiceChangerWorkletProcessor extends AudioWorkletProcessor { class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
private BLOCK_SIZE = 128 private BLOCK_SIZE = 128;
private initialized = false; private initialized = false;
private volume = 0 private volume = 0;
private numTrancateTreshold = 150 private numTrancateTreshold = 150;
// private volTrancateThreshold = 0.0005 // private volTrancateThreshold = 0.0005
// private volTrancateLength = 32 // private volTrancateLength = 32
// private volTrancateCount = 0 // private volTrancateCount = 0
private isRecording = false private isRecording = false;
playBuffer: Float32Array[] = [] playBuffer: Float32Array[] = [];
/** /**
* @constructor * @constructor
*/ */
@ -56,73 +53,72 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
calcVol = (data: Float32Array, prevVol: number) => { calcVol = (data: Float32Array, prevVol: number) => {
const sum = data.reduce((prev, cur) => { const sum = data.reduce((prev, cur) => {
return prev + cur * cur return prev + cur * cur;
}, 0) }, 0);
const rms = Math.sqrt(sum / data.length) const rms = Math.sqrt(sum / data.length);
return Math.max(rms, prevVol * 0.95) return Math.max(rms, prevVol * 0.95);
} };
trancateBuffer = () => { trancateBuffer = () => {
console.log("[worklet] Buffer truncated") console.log("[worklet] Buffer truncated");
while (this.playBuffer.length > 2) { while (this.playBuffer.length > 2) {
this.playBuffer.shift() this.playBuffer.shift();
} }
} };
handleMessage(event: any) { handleMessage(event: any) {
const request = event.data as VoiceChangerWorkletProcessorRequest const request = event.data as VoiceChangerWorkletProcessorRequest;
if (request.requestType === "config") { if (request.requestType === "config") {
this.numTrancateTreshold = request.numTrancateTreshold this.numTrancateTreshold = request.numTrancateTreshold;
// this.volTrancateLength = request.volTrancateLength // this.volTrancateLength = request.volTrancateLength
// this.volTrancateThreshold = request.volTrancateThreshold // this.volTrancateThreshold = request.volTrancateThreshold
console.log("[worklet] worklet configured", request) console.log("[worklet] worklet configured", request);
return return;
} else if (request.requestType === "start") { } else if (request.requestType === "start") {
if (this.isRecording) { if (this.isRecording) {
console.warn("[worklet] recoring is already started") console.warn("[worklet] recoring is already started");
return return;
} }
this.isRecording = true this.isRecording = true;
const startResponse: VoiceChangerWorkletProcessorResponse = { const startResponse: VoiceChangerWorkletProcessorResponse = {
responseType: "start_ok", responseType: "start_ok",
} };
this.port.postMessage(startResponse); this.port.postMessage(startResponse);
return return;
} else if (request.requestType === "stop") { } else if (request.requestType === "stop") {
if (!this.isRecording) { if (!this.isRecording) {
console.warn("[worklet] recoring is not started") console.warn("[worklet] recoring is not started");
return return;
} }
this.isRecording = false this.isRecording = false;
const stopResponse: VoiceChangerWorkletProcessorResponse = { const stopResponse: VoiceChangerWorkletProcessorResponse = {
responseType: "stop_ok", responseType: "stop_ok",
} };
this.port.postMessage(stopResponse); this.port.postMessage(stopResponse);
return return;
} else if (request.requestType === "trancateBuffer") { } else if (request.requestType === "trancateBuffer") {
this.trancateBuffer() this.trancateBuffer();
return return;
} }
if (this.playBuffer.length > this.numTrancateTreshold) { if (this.playBuffer.length > this.numTrancateTreshold) {
this.trancateBuffer() this.trancateBuffer();
} }
const f32Data = request.voice const f32Data = request.voice;
const chunkNum = f32Data.length / this.BLOCK_SIZE const chunkNum = f32Data.length / this.BLOCK_SIZE;
for (let i = 0; i < chunkNum; i++) { for (let i = 0; i < chunkNum; i++) {
const block = f32Data.slice(i * this.BLOCK_SIZE, (i + 1) * this.BLOCK_SIZE) const block = f32Data.slice(i * this.BLOCK_SIZE, (i + 1) * this.BLOCK_SIZE);
this.playBuffer.push(block) this.playBuffer.push(block);
} }
} }
pushData = (inputData: Float32Array) => { pushData = (inputData: Float32Array) => {
const volumeResponse: VoiceChangerWorkletProcessorResponse = { const volumeResponse: VoiceChangerWorkletProcessorResponse = {
responseType: ResponseType.inputData, responseType: ResponseType.inputData,
inputData: inputData inputData: inputData,
} };
this.port.postMessage(volumeResponse); this.port.postMessage(volumeResponse);
} };
process(_inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record<string, Float32Array>) { process(_inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record<string, Float32Array>) {
if (!this.initialized) { if (!this.initialized) {
@ -132,13 +128,13 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
if (this.isRecording) { if (this.isRecording) {
if (_inputs.length > 0 && _inputs[0].length > 0) { if (_inputs.length > 0 && _inputs[0].length > 0) {
this.pushData(_inputs[0][0]) this.pushData(_inputs[0][0]);
} }
} }
if (this.playBuffer.length === 0) { if (this.playBuffer.length === 0) {
// console.log("[worklet] no play buffer") // console.log("[worklet] no play buffer")
return true return true;
} }
//// 一定期間無音状態が続いている場合はスキップ。 //// 一定期間無音状態が続いている場合はスキップ。
@ -155,7 +151,6 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
// this.volTrancateCount = 0 // this.volTrancateCount = 0
// } // }
// // V.1.5.0よりsilent skipで音飛びするようになったので無効化 // // V.1.5.0よりsilent skipで音飛びするようになったので無効化
// if (this.volTrancateCount < this.volTrancateLength || this.volTrancateLength < 0) { // if (this.volTrancateCount < this.volTrancateLength || this.volTrancateLength < 0) {
// break // break
@ -164,22 +159,19 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
// // console.log("silent...skip") // // console.log("silent...skip")
// } // }
// } // }
let voice = this.playBuffer.shift() let voice = this.playBuffer.shift();
if (voice) { if (voice) {
this.volume = this.calcVol(voice, this.volume) this.volume = this.calcVol(voice, this.volume);
const volumeResponse: VoiceChangerWorkletProcessorResponse = { const volumeResponse: VoiceChangerWorkletProcessorResponse = {
responseType: ResponseType.volume, responseType: ResponseType.volume,
volume: this.volume volume: this.volume,
} };
this.port.postMessage(volumeResponse); this.port.postMessage(volumeResponse);
outputs[0][0].set(voice) outputs[0][0].set(voice);
if (outputs[0].length == 2) { if (outputs[0].length == 2) {
outputs[0][1].set(voice) outputs[0][1].set(voice);
} }
} }
return true; return true;