diff --git a/client/demo/src/000_index.tsx b/client/demo/src/000_index.tsx index 7771a417..df326b80 100644 --- a/client/demo/src/000_index.tsx +++ b/client/demo/src/000_index.tsx @@ -1,10 +1,8 @@ import * as React from "react"; import { createRoot } from "react-dom/client"; import "./css/App.css" -import { useEffect, useMemo, useRef, useState } from "react"; +import { useMemo, } from "react"; import { useMicrophoneOptions } from "./100_options_microphone"; -import { VoiceChnagerClient, createDummyMediaStream } from "@dannadori/voice-changer-client-js" -import { AUDIO_ELEMENT_FOR_PLAY_RESULT } from "./const"; const container = document.getElementById("app")!; const root = createRoot(container); diff --git a/client/demo/src/100_options_microphone.tsx b/client/demo/src/100_options_microphone.tsx index 3370c1db..323ac667 100644 --- a/client/demo/src/100_options_microphone.tsx +++ b/client/demo/src/100_options_microphone.tsx @@ -1,6 +1,6 @@ import * as React from "react"; import { useEffect, useMemo, useState } from "react"; -import { AUDIO_ELEMENT_FOR_PLAY_RESULT, CHROME_EXTENSION } from "./const"; +import { AUDIO_ELEMENT_FOR_PLAY_RESULT } from "./const"; import { useServerSetting } from "./101_server_setting"; import { useDeviceSetting } from "./102_device_setting"; import { useConvertSetting } from "./104_convert_setting"; @@ -8,6 +8,7 @@ import { useAdvancedSetting } from "./105_advanced_setting"; import { useSpeakerSetting } from "./103_speaker_setting"; import { useClient } from "./hooks/useClient"; import { useServerControl } from "./106_server_control"; +import { ServerSettingKey } from "@dannadori/voice-changer-client-js"; @@ -19,8 +20,7 @@ export const useMicrophoneOptions = () => { }) const serverSetting = useServerSetting({ - uploadFile: clientState.uploadFile, - changeOnnxExcecutionProvider: clientState.changeOnnxExcecutionProvider + clientState }) const deviceSetting = useDeviceSetting(audioContext) const speakerSetting = useSpeakerSetting() @@ -31,6 +31,7 @@ export const useMicrophoneOptions = () => { const serverControl = useServerControl({ convertStart: async () => { await clientState.start(serverSetting.mmvcServerUrl, serverSetting.protocol) }, convertStop: async () => { clientState.stop() }, + getInfo: clientState.getInfo, volume: clientState.volume, bufferingTime: clientState.bufferingTime, responseTime: clientState.responseTime @@ -47,13 +48,73 @@ export const useMicrophoneOptions = () => { document.addEventListener('mousedown', createAudioContext); }, []) - - + // 101 ServerSetting + //// サーバ変更 useEffect(() => { - console.log("input Cahngaga!") + if (!clientState.clientInitialized) return + clientState.setServerUrl(serverSetting.mmvcServerUrl) + }, [serverSetting.mmvcServerUrl]) + //// プロトコル変更 + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.setProtocol(serverSetting.protocol) + }, [serverSetting.protocol]) + //// フレームワーク変更 + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.updateSettings(ServerSettingKey.framework, serverSetting.framework) + }, [serverSetting.framework]) + //// OnnxExecutionProvider変更 + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.updateSettings(ServerSettingKey.onnxExecutionProvider, serverSetting.onnxExecutionProvider) + }, [serverSetting.onnxExecutionProvider]) + + // 102 DeviceSetting + //// 入力情報の設定 + useEffect(() => { + if (!clientState.clientInitialized) return clientState.changeInput(deviceSetting.audioInput, convertSetting.bufferSize, advancedSetting.vfForceDisabled) }, [clientState.clientInitialized, deviceSetting.audioInput, convertSetting.bufferSize, advancedSetting.vfForceDisabled]) + // 103 SpeakerSetting + // 音声変換元、変換先の設定 + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.updateSettings(ServerSettingKey.srcId, speakerSetting.srcId) + }, [speakerSetting.srcId]) + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.updateSettings(ServerSettingKey.dstId, speakerSetting.dstId) + }, [speakerSetting.dstId]) + + // 104 ConvertSetting + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.setInputChunkNum(convertSetting.inputChunkNum) + }, [convertSetting.inputChunkNum]) + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.updateSettings(ServerSettingKey.convertChunkNum, convertSetting.convertChunkNum) + }, [convertSetting.convertChunkNum]) + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.updateSettings(ServerSettingKey.gpu, convertSetting.gpu) + }, [convertSetting.gpu]) + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.updateSettings(ServerSettingKey.crossFadeOffsetRate, convertSetting.crossFadeOffsetRate) + }, [convertSetting.crossFadeOffsetRate]) + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.updateSettings(ServerSettingKey.crossFadeEndRate, convertSetting.crossFadeEndRate) + }, [convertSetting.crossFadeEndRate]) + + // 105 AdvancedSetting + useEffect(() => { + if (!clientState.clientInitialized) return + clientState.setVoiceChangerMode(advancedSetting.voiceChangerMode) + }, [advancedSetting.voiceChangerMode]) // // const [options, setOptions] = useState(InitMicrophoneOptionsState) diff --git a/client/demo/src/101_server_setting.tsx b/client/demo/src/101_server_setting.tsx index aa1a7724..ed6cbccf 100644 --- a/client/demo/src/101_server_setting.tsx +++ b/client/demo/src/101_server_setting.tsx @@ -1,10 +1,10 @@ -import { DefaultVoiceChangerOptions, OnnxExecutionProvider, Protocol, Framework, fileSelector, getInfo, loadModel } from "@dannadori/voice-changer-client-js" -import React, { useEffect } from "react" +import { DefaultVoiceChangerOptions, OnnxExecutionProvider, Protocol, Framework, fileSelector, ServerSettingKey } from "@dannadori/voice-changer-client-js" +import React from "react" import { useMemo, useState } from "react" +import { ClientState } from "./hooks/useClient" export type UseServerSettingProps = { - uploadFile: (baseUrl: string, file: File, onprogress: (progress: number, end: boolean) => void) => Promise - changeOnnxExcecutionProvider: (baseUrl: string, provider: OnnxExecutionProvider) => Promise + clientState: ClientState } export type ServerSettingState = { @@ -80,24 +80,21 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta return } if (pyTorchModel) { - await props.uploadFile(mmvcServerUrl, pyTorchModel, (progress: number, end: boolean) => { + await props.clientState.uploadFile(pyTorchModel, (progress: number, end: boolean) => { console.log(progress, end) }) } if (onnxModel) { - await props.uploadFile(mmvcServerUrl, onnxModel, (progress: number, end: boolean) => { + await props.clientState.uploadFile(onnxModel, (progress: number, end: boolean) => { console.log(progress, end) }) } - await props.uploadFile(mmvcServerUrl, configFile, (progress: number, end: boolean) => { + await props.clientState.uploadFile(configFile, (progress: number, end: boolean) => { console.log(progress, end) }) - const res = await getInfo(mmvcServerUrl) - console.log(res) - - const res2 = await loadModel(mmvcServerUrl, configFile, pyTorchModel, onnxModel) - console.log(res2) + await props.clientState.loadModel(configFile, pyTorchModel, onnxModel) + console.log("loaded") } return ( @@ -148,7 +145,7 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta ) - }, [pyTorchModel, configFile, onnxModel, mmvcServerUrl, props.uploadFile]) + }, [pyTorchModel, configFile, onnxModel, mmvcServerUrl]) const protocolRow = useMemo(() => { const onProtocolChanged = async (val: Protocol) => { @@ -201,7 +198,6 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta return } const onOnnxExecutionProviderChanged = async (val: OnnxExecutionProvider) => { - await props.changeOnnxExcecutionProvider(mmvcServerUrl, val) setOnnxExecutionProvider(val) } return ( diff --git a/client/demo/src/102_device_setting.tsx b/client/demo/src/102_device_setting.tsx index c791fe7a..460f2f32 100644 --- a/client/demo/src/102_device_setting.tsx +++ b/client/demo/src/102_device_setting.tsx @@ -83,13 +83,9 @@ export const useDeviceSetting = (audioContext: AudioContext | null): DeviceSetti useEffect(() => { - console.log("iiiiinnnppu1") if (!audioContext) { - console.log("iiiiinnnppu2") return } - - console.log("iiiiinnnppu3") if (audioInputForGUI == "none") { const ms = createDummyMediaStream(audioContext) setAudioInput(ms) @@ -168,6 +164,7 @@ export const useDeviceSetting = (audioContext: AudioContext | null): DeviceSetti }, [outputAudioDeviceInfo, audioOutputForGUI]) useEffect(() => { + if (audioOutputForGUI == "none") return [AUDIO_ELEMENT_FOR_PLAY_RESULT, AUDIO_ELEMENT_FOR_TEST_ORIGINAL].forEach(x => { const audio = document.getElementById(x) as HTMLAudioElement if (audio) { diff --git a/client/demo/src/103_speaker_setting.tsx b/client/demo/src/103_speaker_setting.tsx index 565e1901..543a01c9 100644 --- a/client/demo/src/103_speaker_setting.tsx +++ b/client/demo/src/103_speaker_setting.tsx @@ -88,7 +88,7 @@ export const useSpeakerSetting = () => { ) - }, [speakers]) + }, [speakers, editSpeakerTargetId, editSpeakerTargetName]) const speakerSetting = useMemo(() => { diff --git a/client/demo/src/105_advanced_setting.tsx b/client/demo/src/105_advanced_setting.tsx index 68474e59..a8bf4d2b 100644 --- a/client/demo/src/105_advanced_setting.tsx +++ b/client/demo/src/105_advanced_setting.tsx @@ -4,14 +4,14 @@ import React, { useMemo, useState } from "react" export type AdvancedSettingState = { advancedSetting: JSX.Element; vfForceDisabled: boolean; - voiceChangeMode: VoiceChangerMode; + voiceChangerMode: VoiceChangerMode; } export const useAdvancedSetting = (): AdvancedSettingState => { const [vfForceDisabled, setVfForceDisabled] = useState(false) - const [voiceChangeMode, setVoiceChangeMode] = useState("realtime") + const [voiceChangerMode, setVoiceChangerMode] = useState("realtime") const vfForceDisableRow = useMemo(() => { return ( @@ -31,7 +31,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
Voice Change Mode
- { setVoiceChangerMode(e.target.value as VoiceChangerMode) }}> { Object.values(VoiceChangerMode).map(x => { return @@ -60,7 +60,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => { return { advancedSetting, vfForceDisabled, - voiceChangeMode, + voiceChangerMode, } } diff --git a/client/demo/src/106_server_control.tsx b/client/demo/src/106_server_control.tsx index a49500fb..bdfd054f 100644 --- a/client/demo/src/106_server_control.tsx +++ b/client/demo/src/106_server_control.tsx @@ -1,8 +1,10 @@ +import { ServerInfo } from "@dannadori/voice-changer-client-js" import React, { useMemo, useState } from "react" export type UseServerControlProps = { convertStart: () => Promise convertStop: () => Promise + getInfo: () => Promise volume: number, bufferingTime: number, responseTime: number @@ -53,6 +55,29 @@ export const useServerControl = (props: UseServerControlProps) => { }, [props.volume, props.bufferingTime, props.responseTime]) + + const infoRow = useMemo(() => { + const onReloadClicked = async () => { + const info = await props.getInfo() + console.log("info", info) + } + return ( + <> +
+
Info:
+
vol(rms):{props.volume.toFixed(4)}
+
buf(ms):{props.bufferingTime}
+
res(ms):{props.responseTime}
+
+
reload
+
+
+ + ) + }, [props.getInfo]) + + + const serverControl = useMemo(() => { return ( <> @@ -63,9 +88,10 @@ export const useServerControl = (props: UseServerControlProps) => {
{startButtonRow} {performanceRow} + {infoRow} ) - }, [startButtonRow, performanceRow]) + }, [startButtonRow, performanceRow, infoRow]) return { serverControl, diff --git a/client/demo/src/hooks/useClient.ts b/client/demo/src/hooks/useClient.ts index 3b61000a..a2286331 100644 --- a/client/demo/src/hooks/useClient.ts +++ b/client/demo/src/hooks/useClient.ts @@ -1,4 +1,4 @@ -import { BufferSize, createDummyMediaStream, Protocol, VoiceChangerMode, VoiceChangerRequestParamas, VoiceChnagerClient, uploadLargeFile, concatUploadedFile, OnnxExecutionProvider, setOnnxExecutionProvider } from "@dannadori/voice-changer-client-js" +import { BufferSize, createDummyMediaStream, Protocol, ServerSettingKey, VoiceChangerMode, VoiceChnagerClient } from "@dannadori/voice-changer-client-js" import { useEffect, useMemo, useRef, useState } from "react" export type UseClientProps = { @@ -11,14 +11,26 @@ export type ClientState = { bufferingTime: number; responseTime: number; volume: number; + // Client Setting + setServerUrl: (mmvcServerUrl: string) => Promise + setProtocol: (protocol: Protocol) => Promise + setInputChunkNum: (num: number) => Promise + setVoiceChangerMode: (val: VoiceChangerMode) => Promise + + // Client Control start: (mmvcServerUrl: string, protocol: Protocol) => Promise; stop: () => Promise; + + // Device Setting changeInput: (audioInput: MediaStream | string, bufferSize: BufferSize, vfForceDisable: boolean) => Promise - changeInputChunkNum: (inputChunkNum: number) => void - changeVoiceChangeMode: (voiceChangerMode: VoiceChangerMode) => void - changeRequestParams: (params: VoiceChangerRequestParamas) => void - uploadFile: (baseUrl: string, file: File, onprogress: (progress: number, end: boolean) => void) => Promise - changeOnnxExcecutionProvider: (baseUrl: string, provider: OnnxExecutionProvider) => Promise + + // Server Setting + uploadFile: (file: File, onprogress: (progress: number, end: boolean) => void) => Promise + loadModel: (configFile: File, pyTorchModelFile: File | null, onnxModelFile: File | null) => Promise + updateSettings: (key: ServerSettingKey, val: string | number) => Promise + + // Information + getInfo: () => Promise } export const useClient = (props: UseClientProps): ClientState => { @@ -53,7 +65,7 @@ export const useClient = (props: UseClientProps): ClientState => { }) await voiceChangerClient.isInitialized() voiceChangerClientRef.current = voiceChangerClient - console.log("client initialized!!") + console.log("[useClient] client initialized") setClientInitialized(true) const audio = document.getElementById(props.audioOutputElementId) as HTMLAudioElement @@ -63,13 +75,60 @@ export const useClient = (props: UseClientProps): ClientState => { initialized() }, [props.audioContext]) - const start = useMemo(() => { - return async (mmvcServerUrl: string, protocol: Protocol) => { + // Client Setting + const setServerUrl = useMemo(() => { + return async (mmvcServerUrl: string) => { if (!voiceChangerClientRef.current) { console.log("client is not initialized") return } - voiceChangerClientRef.current.setServerUrl(mmvcServerUrl, protocol, true) + voiceChangerClientRef.current.setServerUrl(mmvcServerUrl, true) + voiceChangerClientRef.current.stop() + } + }, []) + + const setProtocol = useMemo(() => { + return async (protocol: Protocol) => { + if (!voiceChangerClientRef.current) { + console.log("client is not initialized") + return + } + voiceChangerClientRef.current.setProtocol(protocol) + voiceChangerClientRef.current.stop() + } + }, []) + + const setInputChunkNum = useMemo(() => { + return async (num: number) => { + if (!voiceChangerClientRef.current) { + console.log("client is not initialized") + return + } + voiceChangerClientRef.current.setInputChunkNum(num) + voiceChangerClientRef.current.stop() + } + }, []) + + const setVoiceChangerMode = useMemo(() => { + return async (val: VoiceChangerMode) => { + if (!voiceChangerClientRef.current) { + console.log("client is not initialized") + return + } + voiceChangerClientRef.current.setVoiceChangerMode(val) + voiceChangerClientRef.current.stop() + } + }, []) + + + // Client Control + const start = useMemo(() => { + return async (mmvcServerUrl: string) => { + if (!voiceChangerClientRef.current) { + console.log("client is not initialized") + return + } + voiceChangerClientRef.current.setServerUrl(mmvcServerUrl, true) voiceChangerClientRef.current.start() } }, []) @@ -83,82 +142,92 @@ export const useClient = (props: UseClientProps): ClientState => { } }, []) + + // Device Setting const changeInput = useMemo(() => { return async (audioInput: MediaStream | string, bufferSize: BufferSize, vfForceDisable: boolean) => { if (!voiceChangerClientRef.current || !props.audioContext) { - console.log("not initialized", voiceChangerClientRef.current, props.audioContext) + console.log("[useClient] not initialized", voiceChangerClientRef.current, props.audioContext) return } if (!audioInput || audioInput == "none") { - console.log("setup! 1") + console.log("[useClient] setup!(1)", audioInput) const ms = createDummyMediaStream(props.audioContext) await voiceChangerClientRef.current.setup(ms, bufferSize, vfForceDisable) } else { - console.log("setup! 2") + console.log("[useClient] setup!(2)", audioInput) await voiceChangerClientRef.current.setup(audioInput, bufferSize, vfForceDisable) } } }, [props.audioContext]) - const changeInputChunkNum = useMemo(() => { - return (inputChunkNum: number) => { - if (!voiceChangerClientRef.current) { - // console.log("client is not initialized") - return - } - voiceChangerClientRef.current.setInputChunkNum(inputChunkNum) - } - }, []) - - const changeVoiceChangeMode = useMemo(() => { - return (voiceChangerMode: VoiceChangerMode) => { - if (!voiceChangerClientRef.current) { - // console.log("client is not initialized") - return - } - voiceChangerClientRef.current.setVoiceChangerMode(voiceChangerMode) - } - }, []) - - const changeRequestParams = useMemo(() => { - return (params: VoiceChangerRequestParamas) => { - if (!voiceChangerClientRef.current) { - // console.log("client is not initialized") - return - } - voiceChangerClientRef.current.setRequestParams(params) - } - }, []) + // Server Setting const uploadFile = useMemo(() => { - return async (baseUrl: string, file: File, onprogress: (progress: number, end: boolean) => void) => { - const num = await uploadLargeFile(baseUrl, file, onprogress) - const res = await concatUploadedFile(baseUrl, file, num) + return async (file: File, onprogress: (progress: number, end: boolean) => void) => { + if (!voiceChangerClientRef.current) { + throw "[useClient] Client Not Initialized." + } + const num = await voiceChangerClientRef.current.uploadFile(file, onprogress) + const res = await voiceChangerClientRef.current.concatUploadedFile(file, num) console.log("upload", num, res) } }, []) - const changeOnnxExcecutionProvider = useMemo(() => { - return async (baseUrl: string, provider: OnnxExecutionProvider) => { - setOnnxExecutionProvider(baseUrl, provider) + const loadModel = useMemo(() => { + return async (configFile: File, pyTorchModelFile: File | null, onnxModelFile: File | null) => { + if (!voiceChangerClientRef.current) { + throw "[useClient] Client Not Initialized." + } + await voiceChangerClientRef.current.loadModel(configFile, pyTorchModelFile, onnxModelFile) + console.log("load model") } }, []) + const updateSettings = useMemo(() => { + return async (key: ServerSettingKey, val: string | number) => { + if (!voiceChangerClientRef.current) { + throw "[useClient] Client Not Initialized." + } + return await voiceChangerClientRef.current.updateServerSettings(key, "" + val) + } + }, []) + + // Information + const getInfo = useMemo(() => { + return async () => { + if (!voiceChangerClientRef.current) { + throw "[useClient] Client Not Initialized." + } + const serverSettings = await voiceChangerClientRef.current.getServerSettings() + const clientSettings = await voiceChangerClientRef.current.getClientSettings() + console.log(serverSettings, clientSettings) + } + }, []) + + return { clientInitialized, bufferingTime, responseTime, volume, + setServerUrl, + setProtocol, + setInputChunkNum, + setVoiceChangerMode, + start, stop, - uploadFile, + changeInput, - changeInputChunkNum, - changeVoiceChangeMode, - changeRequestParams, - changeOnnxExcecutionProvider, + + uploadFile, + loadModel, + updateSettings, + + getInfo, } } \ No newline at end of file diff --git a/client/lib/src/AudioStreamer.ts b/client/lib/src/AudioStreamer.ts index 252af791..3761d0f1 100644 --- a/client/lib/src/AudioStreamer.ts +++ b/client/lib/src/AudioStreamer.ts @@ -1,7 +1,7 @@ import { io, Socket } from "socket.io-client"; import { DefaultEventsMap } from "@socket.io/component-emitter"; import { Duplex, DuplexOptions } from "readable-stream"; -import { DefaultVoiceChangerRequestParamas, Protocol, VoiceChangerMode, VoiceChangerRequestParamas, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const"; +import { Protocol, VoiceChangerMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const"; export type Callbacks = { onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer) => void @@ -11,6 +11,14 @@ export type AudioStreamerListeners = { notifyResponseTime: (time: number) => void notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void } + +export type AudioStreamerSettings = { + serverUrl: string; + protocol: Protocol; + inputChunkNum: number; + voiceChangerMode: VoiceChangerMode; +} + export class AudioStreamer extends Duplex { private callbacks: Callbacks private audioStreamerListeners: AudioStreamerListeners @@ -18,8 +26,7 @@ export class AudioStreamer extends Duplex { private serverUrl = "" private socket: Socket | null = null private voiceChangerMode: VoiceChangerMode = "realtime" - private requestParamas: VoiceChangerRequestParamas = DefaultVoiceChangerRequestParamas - private inputChunkNum = 10 + private inputChunkNum = 128 private requestChunks: ArrayBuffer[] = [] private recordChunks: ArrayBuffer[] = [] private isRecording = false @@ -58,16 +65,15 @@ export class AudioStreamer extends Duplex { } // Option Change - setServerUrl = (serverUrl: string, mode: Protocol) => { + setServerUrl = (serverUrl: string) => { this.serverUrl = serverUrl - this.protocol = mode console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.protocol}`) - this.createSocketIO()// mode check is done in the method. } - - setRequestParams = (val: VoiceChangerRequestParamas) => { - this.requestParamas = val + setProtocol = (mode: Protocol) => { + this.protocol = mode + console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.protocol}`) + this.createSocketIO()// mode check is done in the method. } setInputChunkNum = (num: number) => { @@ -78,6 +84,15 @@ export class AudioStreamer extends Duplex { this.voiceChangerMode = val } + getSettings = (): AudioStreamerSettings => { + return { + serverUrl: this.serverUrl, + protocol: this.protocol, + inputChunkNum: this.inputChunkNum, + voiceChangerMode: this.voiceChangerMode + } + } + // Main Process //// Pipe from mic stream @@ -191,7 +206,7 @@ export class AudioStreamer extends Duplex { const timestamp = Date.now() // console.log("REQUEST_MESSAGE:", [this.gpu, this.srcId, this.dstId, timestamp, newBuffer.buffer]) // console.log("SERVER_URL", this.serverUrl, this.protocol) - const convertChunkNum = this.voiceChangerMode === "realtime" ? this.requestParamas.convertChunkNum : 0 + // const convertChunkNum = this.voiceChangerMode === "realtime" ? this.requestParamas.convertChunkNum : 0 if (this.protocol === "sio") { if (!this.socket) { console.warn(`sio is not initialized`) @@ -199,26 +214,26 @@ export class AudioStreamer extends Duplex { } // console.log("emit!") this.socket.emit('request_message', [ - this.requestParamas.gpu, - this.requestParamas.srcId, - this.requestParamas.dstId, + // this.requestParamas.gpu, + // this.requestParamas.srcId, + // this.requestParamas.dstId, timestamp, - convertChunkNum, - this.requestParamas.crossFadeLowerValue, - this.requestParamas.crossFadeOffsetRate, - this.requestParamas.crossFadeEndRate, + // convertChunkNum, + // this.requestParamas.crossFadeLowerValue, + // this.requestParamas.crossFadeOffsetRate, + // this.requestParamas.crossFadeEndRate, newBuffer.buffer]); } else { const res = await postVoice( this.serverUrl + "/test", - this.requestParamas.gpu, - this.requestParamas.srcId, - this.requestParamas.dstId, + // this.requestParamas.gpu, + // this.requestParamas.srcId, + // this.requestParamas.dstId, timestamp, - convertChunkNum, - this.requestParamas.crossFadeLowerValue, - this.requestParamas.crossFadeOffsetRate, - this.requestParamas.crossFadeEndRate, + // convertChunkNum, + // this.requestParamas.crossFadeLowerValue, + // this.requestParamas.crossFadeOffsetRate, + // this.requestParamas.crossFadeEndRate, newBuffer.buffer) if (res.byteLength < 128 * 2) { @@ -233,24 +248,24 @@ export class AudioStreamer extends Duplex { export const postVoice = async ( url: string, - gpu: number, - srcId: number, - dstId: number, + // gpu: number, + // srcId: number, + // dstId: number, timestamp: number, - convertChunkNum: number, - crossFadeLowerValue: number, - crossFadeOffsetRate: number, - crossFadeEndRate: number, + // convertChunkNum: number, + // crossFadeLowerValue: number, + // crossFadeOffsetRate: number, + // crossFadeEndRate: number, buffer: ArrayBuffer) => { const obj = { - gpu, - srcId, - dstId, + // gpu, + // srcId, + // dstId, timestamp, - convertChunkNum, - crossFadeLowerValue, - crossFadeOffsetRate, - crossFadeEndRate, + // convertChunkNum, + // crossFadeLowerValue, + // crossFadeOffsetRate, + // crossFadeEndRate, buffer: Buffer.from(buffer).toString('base64') }; const body = JSON.stringify(obj); diff --git a/client/lib/src/ServerConfigurator.ts b/client/lib/src/ServerConfigurator.ts new file mode 100644 index 00000000..baba504d --- /dev/null +++ b/client/lib/src/ServerConfigurator.ts @@ -0,0 +1,132 @@ +import { ServerInfo, ServerSettingKey } from "./const"; + + +type FileChunk = { + hash: number, + chunk: Blob +} +export class ServerConfigurator { + private serverUrl = "" + getSettings = async () => { + const url = this.serverUrl + "/info" + const info = await new Promise((resolve) => { + const request = new Request(url, { + method: 'GET', + }); + fetch(request).then(async (response) => { + const json = await response.json() as ServerInfo + resolve(json) + }) + }) + return info + } + + setServerUrl = (serverUrl: string) => { + this.serverUrl = serverUrl + console.log(`[ServerConfigurator] Server URL: ${this.serverUrl}`) + } + + updateSettings = async (key: ServerSettingKey, val: string) => { + const url = this.serverUrl + "/update_setteings" + const p = new Promise((resolve) => { + const formData = new FormData(); + formData.append("key", key); + formData.append("val", val); + const request = new Request(url, { + method: 'POST', + body: formData, + }); + fetch(request).then(async (response) => { + console.log(await response.json()) + resolve() + }) + }) + const info = await p + return info + } + + uploadFile = async (file: File, onprogress: (progress: number, end: boolean) => void) => { + const url = this.serverUrl + "/upload_file" + onprogress(0, false) + const size = 1024 * 1024; + const fileChunks: FileChunk[] = []; + let index = 0; // index値 + for (let cur = 0; cur < file.size; cur += size) { + fileChunks.push({ + hash: index++, + chunk: file.slice(cur, cur + size), + }); + } + + const chunkNum = fileChunks.length + console.log("FILE_CHUNKS:", chunkNum, fileChunks) + + + while (true) { + const promises: Promise[] = [] + for (let i = 0; i < 10; i++) { + const chunk = fileChunks.shift() + if (!chunk) { + break + } + const p = new Promise((resolve) => { + const formData = new FormData(); + formData.append("file", chunk.chunk); + formData.append("filename", `${file.name}_${chunk.hash}`); + const request = new Request(url, { + method: 'POST', + body: formData, + }); + fetch(request).then(async (response) => { + console.log(await response.text()) + resolve() + }) + }) + + promises.push(p) + } + await Promise.all(promises) + if (fileChunks.length == 0) { + break + } + onprogress(Math.floor(((chunkNum - fileChunks.length) / (chunkNum + 1)) * 100), false) + } + return chunkNum + } + + concatUploadedFile = async (file: File, chunkNum: number) => { + const url = this.serverUrl + "/concat_uploaded_file" + new Promise((resolve) => { + const formData = new FormData(); + formData.append("filename", file.name); + formData.append("filenameChunkNum", "" + chunkNum); + const request = new Request(url, { + method: 'POST', + body: formData, + }); + fetch(request).then(async (response) => { + console.log(await response.text()) + resolve() + }) + }) + } + + loadModel = async (configFile: File, pyTorchModelFile: File | null, onnxModelFile: File | null) => { + const url = this.serverUrl + "/load_model" + const loadP = new Promise((resolve) => { + const formData = new FormData(); + formData.append("pyTorchModelFilename", pyTorchModelFile?.name || "-"); + formData.append("onnxModelFilename", onnxModelFile?.name || "-"); + formData.append("configFilename", configFile.name); + const request = new Request(url, { + method: 'POST', + body: formData, + }); + fetch(request).then(async (response) => { + console.log(await response.text()) + resolve() + }) + }) + await loadP + } +} \ No newline at end of file diff --git a/client/lib/src/VoiceChangerClient.ts b/client/lib/src/VoiceChangerClient.ts index aba247ee..ed416ce0 100644 --- a/client/lib/src/VoiceChangerClient.ts +++ b/client/lib/src/VoiceChangerClient.ts @@ -3,9 +3,10 @@ import { VoiceChangerWorkletNode, VolumeListener } from "./VoiceChangerWorkletNo import workerjs from "raw-loader!../worklet/dist/index.js"; import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js"; import { createDummyMediaStream, validateUrl } from "./util"; -import { BufferSize, DefaultVoiceChangerOptions, DefaultVoiceChangerRequestParamas, Protocol, VoiceChangerMode, VoiceChangerRequestParamas, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const"; +import { BufferSize, DefaultVoiceChangerOptions, Protocol, ServerSettingKey, VoiceChangerMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const"; import MicrophoneStream from "microphone-stream"; import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreamer"; +import { ServerConfigurator } from "./ServerConfigurator"; // オーディオデータの流れ @@ -15,6 +16,7 @@ import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreame export class VoiceChnagerClient { + private configurator: ServerConfigurator private ctx: AudioContext private vfEnable = false private vf: VoiceFocusDeviceTransformer | null = null @@ -61,6 +63,7 @@ export class VoiceChnagerClient { } constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners, volumeListener: VolumeListener) { + this.configurator = new ServerConfigurator() this.ctx = ctx this.vfEnable = vfEnable this.promiseForInitialize = new Promise(async (resolve) => { @@ -72,7 +75,7 @@ export class VoiceChnagerClient { this.vcNode.connect(this.currentMediaStreamAudioDestinationNode) // vc node -> output node // (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる) this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, }) - this.audioStreamer.setRequestParams(DefaultVoiceChangerRequestParamas) + // this.audioStreamer.setRequestParams(DefaultVoiceChangerRequestParamas) this.audioStreamer.setInputChunkNum(DefaultVoiceChangerOptions.inputChunkNum) this.audioStreamer.setVoiceChangerMode(DefaultVoiceChangerOptions.voiceChangerMode) @@ -168,7 +171,7 @@ export class VoiceChnagerClient { return this._isVoiceChanging } // Audio Streamer Settingg - setServerUrl = (serverUrl: string, mode: Protocol, openTab: boolean = false) => { + setServerUrl = (serverUrl: string, openTab: boolean = false) => { const url = validateUrl(serverUrl) const pageUrl = `${location.protocol}//${location.host}` console.log("SERVER CHECK", url, pageUrl) @@ -183,11 +186,12 @@ export class VoiceChnagerClient { } } } - this.audioStreamer.setServerUrl(validateUrl(serverUrl), mode) + this.audioStreamer.setServerUrl(url) + this.configurator.setServerUrl(url) } - setRequestParams = (val: VoiceChangerRequestParamas) => { - this.audioStreamer.setRequestParams(val) + setProtocol = (mode: Protocol) => { + this.audioStreamer.setProtocol(mode) } setInputChunkNum = (num: number) => { @@ -198,5 +202,28 @@ export class VoiceChnagerClient { this.audioStreamer.setVoiceChangerMode(val) } + // Configurator Method + uploadFile = (file: File, onprogress: (progress: number, end: boolean) => void) => { + return this.configurator.uploadFile(file, onprogress) + } + concatUploadedFile = (file: File, chunkNum: number) => { + return this.configurator.concatUploadedFile(file, chunkNum) + } + loadModel = (configFile: File, pyTorchModelFile: File | null, onnxModelFile: File | null) => { + return this.configurator.loadModel(configFile, pyTorchModelFile, onnxModelFile) + } + updateServerSettings = (key: ServerSettingKey, val: string) => { + return this.configurator.updateSettings(key, val) + } + + // Information + getClientSettings = () => { + return this.audioStreamer.getSettings() + } + getServerSettings = () => { + return this.configurator.getSettings() + } + + } \ No newline at end of file diff --git a/client/lib/src/const.ts b/client/lib/src/const.ts index a66cab07..eabd9e8c 100644 --- a/client/lib/src/const.ts +++ b/client/lib/src/const.ts @@ -36,6 +36,15 @@ export type Speaker = { "name": string, } + +export type ServerInfo = { + pyTorchModelFile: string, + onnxModelFile: string, + configFile: string, + providers: string[] +} + + // Consts export const Protocol = { @@ -80,6 +89,18 @@ export const Framework = { } export type Framework = typeof Framework[keyof typeof Framework] +export const ServerSettingKey = { + "srcId": "srcId", + "dstId": "dstId", + "convertChunkNum": "convertChunkNum", + "gpu": "gpu", + "crossFadeOffsetRate": "crossFadeOffsetRate", + "crossFadeEndRate": "crossFadeEndRate", + "framework": "framework", + "onnxExecutionProvider": "onnxExecutionProvider" +} as const +export type ServerSettingKey = typeof ServerSettingKey[keyof typeof ServerSettingKey] + // Defaults export const DefaultVoiceChangerRequestParamas: VoiceChangerRequestParamas = { convertChunkNum: 32, //（★１） diff --git a/client/lib/src/index.ts b/client/lib/src/index.ts index 66907402..d545210f 100644 --- a/client/lib/src/index.ts +++ b/client/lib/src/index.ts @@ -1,4 +1,3 @@ export * from "./const" export * from "./VoiceChangerClient" export * from "./util" -export * from "./uploader" \ No newline at end of file diff --git a/client/lib/src/uploader.ts b/client/lib/src/uploader.ts deleted file mode 100644 index e7676def..00000000 --- a/client/lib/src/uploader.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { OnnxExecutionProvider } from "./const" -import { validateUrl } from "./util" - - -type FileChunk = { - hash: number, - chunk: Blob -} - -export type ServerInfo = { - pyTorchModelFile: string, - onnxModelFile: string, - configFile: string, - providers: string[] -} - -export const getInfo = async (baseUrl: string) => { - const url = validateUrl(baseUrl) + "/info" - const info = await new Promise((resolve) => { - const request = new Request(url, { - method: 'GET', - }); - fetch(request).then(async (response) => { - const json = await response.json() as ServerInfo - resolve(json) - }) - }) - return info -} - - -export const uploadLargeFile = async (baseUrl: string, file: File, onprogress: (progress: number, end: boolean) => void) => { - const url = validateUrl(baseUrl) + "/upload_file" - onprogress(0, false) - const size = 1024 * 1024; - const fileChunks: FileChunk[] = []; - let index = 0; // index値 - for (let cur = 0; cur < file.size; cur += size) { - fileChunks.push({ - hash: index++, - chunk: file.slice(cur, cur + size), - }); - } - - const chunkNum = fileChunks.length - console.log("FILE_CHUNKS:", chunkNum, fileChunks) - - - while (true) { - const promises: Promise[] = [] - for (let i = 0; i < 10; i++) { - const chunk = fileChunks.shift() - if (!chunk) { - break - } - const p = new Promise((resolve) => { - const formData = new FormData(); - formData.append("file", chunk.chunk); - formData.append("filename", `${file.name}_${chunk.hash}`); - const request = new Request(url, { - method: 'POST', - body: formData, - }); - fetch(request).then(async (response) => { - console.log(await response.text()) - resolve() - }) - }) - - promises.push(p) - } - await Promise.all(promises) - if (fileChunks.length == 0) { - break - } - onprogress(Math.floor(((chunkNum - fileChunks.length) / (chunkNum + 1)) * 100), false) - } - return chunkNum -} - -export const concatUploadedFile = async (baseUrl: string, file: File, chunkNum: number) => { - const url = validateUrl(baseUrl) + "/concat_uploaded_file" - new Promise((resolve) => { - const formData = new FormData(); - formData.append("filename", file.name); - formData.append("filenameChunkNum", "" + chunkNum); - const request = new Request(url, { - method: 'POST', - body: formData, - }); - fetch(request).then(async (response) => { - console.log(await response.text()) - resolve() - }) - }) -} - -export const loadModel = async (baseUrl: string, configFile: File, pyTorchModelFile: File | null, onnxModelFile: File | null) => { - const url = validateUrl(baseUrl) + "/load_model" - const loadP = new Promise((resolve) => { - const formData = new FormData(); - formData.append("pyTorchModelFilename", pyTorchModelFile?.name || "-"); - formData.append("onnxModelFilename", onnxModelFile?.name || "-"); - formData.append("configFilename", configFile.name); - const request = new Request(url, { - method: 'POST', - body: formData, - }); - fetch(request).then(async (response) => { - console.log(await response.text()) - resolve() - }) - }) - await loadP -} - -export const setOnnxExecutionProvider = async (baseUrl: string, provider: OnnxExecutionProvider) => { - const url = validateUrl(baseUrl) + "/set_onnx_provider" - const loadP = new Promise((resolve) => { - const formData = new FormData(); - formData.append("provider", provider); - const request = new Request(url, { - method: 'POST', - body: formData, - }); - fetch(request).then(async (response) => { - console.log(await response.json()) - resolve() - }) - }) - await loadP -} - -// export const uploadModelProps = async (baseUrl: string, modelFile: File, configFile: File, onprogress: (progress: number, end: boolean) => void) => { -// const uploadURL = DEBUG ? `${DEBUG_BASE_URL}/upload_file` : `${baseUrl}/upload_file` -// const loadModelURL = DEBUG ? `${DEBUG_BASE_URL}/load_model` : `${baseUrl}/load_model` -// onprogress(0, false) - -// const chunkNum = await uploadLargeFile(baseUrl, modelFile, (progress: number, _end: boolean) => { -// onprogress(progress, false) -// }) -// console.log("model uploaded") - - -// const configP = new Promise((resolve) => { -// const formData = new FormData(); -// formData.append("file", configFile); -// formData.append("filename", configFile.name); -// const request = new Request(uploadURL, { -// method: 'POST', -// body: formData, -// }); -// fetch(request).then(async (response) => { -// console.log(await response.text()) -// resolve() -// }) -// }) - -// await configP -// console.log("config uploaded") - -// const loadP = new Promise((resolve) => { -// const formData = new FormData(); -// formData.append("modelFilename", modelFile.name); -// formData.append("modelFilenameChunkNum", "" + chunkNum); -// formData.append("configFilename", configFile.name); -// const request = new Request(loadModelURL, { -// method: 'POST', -// body: formData, -// }); -// fetch(request).then(async (response) => { -// console.log(await response.text()) -// resolve() -// }) -// }) -// await loadP -// onprogress(100, true) -// console.log("model loaded") -// } - - diff --git a/server/restapi/MMVC_Rest_Fileuploader.py b/server/restapi/MMVC_Rest_Fileuploader.py index 2ab58b22..9008ac65 100644 --- a/server/restapi/MMVC_Rest_Fileuploader.py +++ b/server/restapi/MMVC_Rest_Fileuploader.py @@ -1,5 +1,5 @@ import os,shutil - +from typing import Union from fastapi import APIRouter from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse @@ -20,7 +20,7 @@ class MMVC_Rest_Fileuploader: self.router.add_api_route("/info", self.get_info, methods=["GET"]) self.router.add_api_route("/upload_file", self.post_upload_file, methods=["POST"]) self.router.add_api_route("/concat_uploaded_file", self.post_concat_uploaded_file, methods=["POST"]) - self.router.add_api_route("/set_onnx_provider", self.post_set_onnx_provider, methods=["POST"]) + self.router.add_api_route("/update_setteings",self.post_update_setteings, methods=["POST"]) self.router.add_api_route("/load_model", self.post_load_model, methods=["POST"]) self.router.add_api_route("/load_model_for_train", self.post_load_model_for_train, methods=["POST"]) self.router.add_api_route("/extract_voices", self.post_extract_voices, methods=["POST"]) @@ -35,16 +35,17 @@ class MMVC_Rest_Fileuploader: UPLOAD_DIR, filename, filenameChunkNum, UPLOAD_DIR) return {"concat": f"{modelFilePath}"} - def post_set_onnx_provider(self, provider: str = Form(...)): - res = self.voiceChangerManager.set_onnx_provider(provider) - json_compatible_item_data = jsonable_encoder(res) - return JSONResponse(content=json_compatible_item_data) - def get_info(self): info = self.voiceChangerManager.get_info() json_compatible_item_data = jsonable_encoder(info) return JSONResponse(content=json_compatible_item_data) + def post_update_setteings(self, key:str=Form(...), val:Union[int, str, float]=Form(...)): + print("post_update_setteings", key, val) + info = self.voiceChangerManager.update_setteings(key, val) + json_compatible_item_data = jsonable_encoder(info) + return JSONResponse(content=json_compatible_item_data) + def post_load_model( self, pyTorchModelFilename: str = Form(...), diff --git a/server/sio/MMVC_Namespace.py b/server/sio/MMVC_Namespace.py index 7de618b3..6b8c03c1 100644 --- a/server/sio/MMVC_Namespace.py +++ b/server/sio/MMVC_Namespace.py @@ -30,15 +30,14 @@ class MMVC_Namespace(socketio.AsyncNamespace): crossFadeOffsetRate = float(msg[6]) crossFadeEndRate = float(msg[7]) data = msg[8] - # print(srcId, dstId, timestamp, convertChunkNum, crossFadeLowerValue, crossFadeOffsetRate, crossFadeEndRate) unpackedData = np.array(struct.unpack('<%sh' % (len(data) // struct.calcsize(' convertSize: convertSize = unpackedData.shape[0] * 2 # print("convert Size", convertChunkNum, convertSize) - self._generate_strength(crossFadeOffsetRate, crossFadeEndRate, unpackedData) - data = self. _generate_input(unpackedData, convertSize, srcId) + self._generate_strength(unpackedData) + data = self._generate_input(unpackedData, convertSize) - try: - # if gpu < 0 or (self.gpu_num == 0 and not self.mps_enabled): - if gpu == -2 and hasattr(self, 'onnx_session') == True: - x, x_lengths, spec, spec_lengths, y, y_lengths, sid_src = [x for x in data] - sid_tgt1 = torch.LongTensor([dstId]) - # if spec.size()[2] >= 8: - audio1 = self.onnx_session.run( - ["audio"], - { - "specs": spec.numpy(), - "lengths": spec_lengths.numpy(), - "sid_src": sid_src.numpy(), - "sid_tgt": sid_tgt1.numpy() - })[0][0,0] * self.hps.data.max_wav_value - if hasattr(self, 'np_prev_audio1') == True: - prev = self.np_prev_audio1[-1*unpackedData.shape[0]:] - cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] - # print(prev.shape, self.np_prev_strength.shape, cur.shape, self.np_cur_strength.shape) - powered_prev = prev * self.np_prev_strength - powered_cur = cur * self.np_cur_strength - result = powered_prev + powered_cur - #result = prev * self.np_prev_strength + cur * self.np_cur_strength - else: - cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] - result = cur - self.np_prev_audio1 = audio1 + # try: + # # if gpu < 0 or (self.gpu_num == 0 and not self.mps_enabled): + # if self.gpu == -2 and hasattr(self, 'onnx_session') == True: + # x, x_lengths, spec, spec_lengths, y, y_lengths, sid_src = [x for x in data] + # sid_tgt1 = torch.LongTensor([self.dstId]) + # # if spec.size()[2] >= 8: + # audio1 = self.onnx_session.run( + # ["audio"], + # { + # "specs": spec.numpy(), + # "lengths": spec_lengths.numpy(), + # "sid_src": sid_src.numpy(), + # "sid_tgt": sid_tgt1.numpy() + # })[0][0,0] * self.hps.data.max_wav_value + # if hasattr(self, 'np_prev_audio1') == True: + # prev = self.np_prev_audio1[-1*unpackedData.shape[0]:] + # cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] + # # print(prev.shape, self.np_prev_strength.shape, cur.shape, self.np_cur_strength.shape) + # powered_prev = prev * self.np_prev_strength + # powered_cur = cur * self.np_cur_strength + # result = powered_prev + powered_cur + # #result = prev * self.np_prev_strength + cur * self.np_cur_strength + # else: + # cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] + # result = cur + # self.np_prev_audio1 = audio1 - elif gpu < 0 or self.gpu_num == 0: - with torch.no_grad(): - x, x_lengths, spec, spec_lengths, y, y_lengths, sid_src = [ - x.cpu() for x in data] - sid_tgt1 = torch.LongTensor([dstId]).cpu() - audio1 = (self.net_g.cpu().voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt1)[0][0, 0].data * self.hps.data.max_wav_value) + # elif self.gpu < 0 or self.gpu_num == 0: + # with torch.no_grad(): + # x, x_lengths, spec, spec_lengths, y, y_lengths, sid_src = [ + # x.cpu() for x in data] + # sid_tgt1 = torch.LongTensor([self.dstId]).cpu() + # audio1 = (self.net_g.cpu().voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt1)[0][0, 0].data * self.hps.data.max_wav_value) - if self.prev_strength.device != torch.device('cpu'): - print(f"prev_strength move from {self.prev_strength.device} to cpu") - self.prev_strength = self.prev_strength.cpu() - if self.cur_strength.device != torch.device('cpu'): - print(f"cur_strength move from {self.cur_strength.device} to cpu") - self.cur_strength = self.cur_strength.cpu() + # if self.prev_strength.device != torch.device('cpu'): + # print(f"prev_strength move from {self.prev_strength.device} to cpu") + # self.prev_strength = self.prev_strength.cpu() + # if self.cur_strength.device != torch.device('cpu'): + # print(f"cur_strength move from {self.cur_strength.device} to cpu") + # self.cur_strength = self.cur_strength.cpu() - if hasattr(self, 'prev_audio1') == True and self.prev_audio1.device == torch.device('cpu'): - prev = self.prev_audio1[-1*unpackedData.shape[0]:] - cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] - result = prev * self.prev_strength + cur * self.cur_strength - else: - cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] - result = cur + # if hasattr(self, 'prev_audio1') == True and self.prev_audio1.device == torch.device('cpu'): + # prev = self.prev_audio1[-1*unpackedData.shape[0]:] + # cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] + # result = prev * self.prev_strength + cur * self.cur_strength + # else: + # cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] + # result = cur - self.prev_audio1 = audio1 - result = result.cpu().float().numpy() - # elif self.mps_enabled == True: # MPS doesnt support aten::weight_norm_interface, and PYTORCH_ENABLE_MPS_FALLBACK=1 cause a big dely. - # x, x_lengths, spec, spec_lengths, y, y_lengths, sid_src = [ - # x.to("mps") for x in data] - # sid_tgt1 = torch.LongTensor([dstId]).to("mps") - # audio1 = (self.net_g.to("mps").voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt1)[ - # 0][0, 0].data * self.hps.data.max_wav_value).cpu().float().numpy() + # self.prev_audio1 = audio1 + # result = result.cpu().float().numpy() - else: - with torch.no_grad(): - x, x_lengths, spec, spec_lengths, y, y_lengths, sid_src = [x.cuda(gpu) for x in data] - sid_tgt1 = torch.LongTensor([dstId]).cuda(gpu) - # audio1 = (self.net_g.cuda(gpu).voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt1)[0][0, 0].data * self.hps.data.max_wav_value).cpu().float().numpy() - audio1 = self.net_g.cuda(gpu).voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt1)[0][0, 0].data * self.hps.data.max_wav_value + # else: + # with torch.no_grad(): + # x, x_lengths, spec, spec_lengths, y, y_lengths, sid_src = [x.cuda(self.gpu) for x in data] + # sid_tgt1 = torch.LongTensor([self.dstId]).cuda(self.gpu) + # audio1 = self.net_g.cuda(self.gpu).voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt1)[0][0, 0].data * self.hps.data.max_wav_value - if self.prev_strength.device != torch.device('cuda', gpu): - print(f"prev_strength move from {self.prev_strength.device} to gpu{gpu}") - self.prev_strength = self.prev_strength.cuda(gpu) - if self.cur_strength.device != torch.device('cuda', gpu): - print(f"cur_strength move from {self.cur_strength.device} to gpu{gpu}") - self.cur_strength = self.cur_strength.cuda(gpu) + # if self.prev_strength.device != torch.device('cuda', self.gpu): + # print(f"prev_strength move from {self.prev_strength.device} to gpu{self.gpu}") + # self.prev_strength = self.prev_strength.cuda(self.gpu) + # if self.cur_strength.device != torch.device('cuda', self.gpu): + # print(f"cur_strength move from {self.cur_strength.device} to gpu{self.gpu}") + # self.cur_strength = self.cur_strength.cuda(self.gpu) - if hasattr(self, 'prev_audio1') == True and self.prev_audio1.device == torch.device('cuda', gpu): - prev = self.prev_audio1[-1*unpackedData.shape[0]:] - cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] - result = prev * self.prev_strength + cur * self.cur_strength - # print("merging...", prev.shape, cur.shape) - else: - cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] - result = cur - # print("no merging...", cur.shape) - self.prev_audio1 = audio1 + # if hasattr(self, 'prev_audio1') == True and self.prev_audio1.device == torch.device('cuda', self.gpu): + # prev = self.prev_audio1[-1*unpackedData.shape[0]:] + # cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] + # result = prev * self.prev_strength + cur * self.cur_strength + # # print("merging...", prev.shape, cur.shape) + # else: + # cur = audio1[-2*unpackedData.shape[0]:-1*unpackedData.shape[0]] + # result = cur + # # print("no merging...", cur.shape) + # self.prev_audio1 = audio1 - #print(result) - result = result.cpu().float().numpy() + # #print(result) + # result = result.cpu().float().numpy() - except Exception as e: - print("VC PROCESSING!!!! EXCEPTION!!!", e) - print(traceback.format_exc()) - del self.np_prev_audio1 - del self.prev_audio1 + # except Exception as e: + # print("VC PROCESSING!!!! EXCEPTION!!!", e) + # print(traceback.format_exc()) + # del self.np_prev_audio1 + # del self.prev_audio1 - result = result.astype(np.int16) - # print("on_request result size:",result.shape) - return result + # result = result.astype(np.int16) + # # print("on_request result size:",result.shape) + # return result + return diff --git a/server/voice_changer/VoiceChangerManager.py b/server/voice_changer/VoiceChangerManager.py index 464f6217..768d5a9b 100644 --- a/server/voice_changer/VoiceChangerManager.py +++ b/server/voice_changer/VoiceChangerManager.py @@ -21,16 +21,22 @@ class VoiceChangerManager(): else: return {"no info":"no info"} - def set_onnx_provider(self, provider:str): + def update_setteings(self, key:str, val:any): if hasattr(self, 'voiceChanger'): - return self.voiceChanger.set_onnx_provider(provider) + return self.voiceChanger.update_setteings(key, val) else: - return {"error":"no voice changer"} + return {"no info":"no info"} + + # def set_onnx_provider(self, provider:str): + # if hasattr(self, 'voiceChanger'): + # return self.voiceChanger.set_onnx_provider(provider) + # else: + # return {"error":"no voice changer"} def changeVoice(self, gpu:int, srcId:int, dstId:int, timestamp:int, convertChunkNum:int, crossFadeLowerValue:float, crossFadeOffsetRate:float, crossFadeEndRate:float, unpackedData:any): if hasattr(self, 'voiceChanger') == True: - return self.voiceChanger.on_request(gpu, srcId, dstId, timestamp, convertChunkNum, crossFadeLowerValue, crossFadeOffsetRate, crossFadeEndRate, unpackedData) + return self.voiceChanger.on_request(unpackedData) else: print("Voice Change is not loaded. Did you load a correct model?") return np.zeros(1).astype(np.int16)