diff --git a/client/demo/src/hooks/useClient.ts b/client/demo/src/hooks/useClient.ts index b3eccb94..f08dfaf8 100644 --- a/client/demo/src/hooks/useClient.ts +++ b/client/demo/src/hooks/useClient.ts @@ -11,6 +11,12 @@ export type ClientState = { bufferingTime: number; responseTime: number; volume: number; + + // Setting + + + + // Client Setting setServerUrl: (mmvcServerUrl: string) => Promise setProtocol: (protocol: Protocol) => Promise @@ -36,17 +42,15 @@ export const useClient = (props: UseClientProps): ClientState => { const voiceChangerClientRef = useRef(null) const [clientInitialized, setClientInitialized] = useState(false) - - const [bufferingTime, setBufferingTime] = useState(0) - const [responseTime, setResponseTime] = useState(0) - const [volume, setVolume] = useState(0) - const initializedResolveRef = useRef<(value: void | PromiseLike) => void>() const initializedPromise = useMemo(() => { return new Promise((resolve) => { initializedResolveRef.current = resolve }) }, []) + const [bufferingTime, setBufferingTime] = useState(0) + const [responseTime, setResponseTime] = useState(0) + const [volume, setVolume] = useState(0) useEffect(() => { const initialized = async () => { @@ -88,30 +92,30 @@ export const useClient = (props: UseClientProps): ClientState => { const setServerUrl = useMemo(() => { return async (mmvcServerUrl: string) => { await initializedPromise - voiceChangerClientRef.current.setServerUrl(mmvcServerUrl, true) - voiceChangerClientRef.current.stop() + voiceChangerClientRef.current!.setServerUrl(mmvcServerUrl, true) + voiceChangerClientRef.current!.stop() } }, []) const setProtocol = useMemo(() => { return async (protocol: Protocol) => { await initializedPromise - voiceChangerClientRef.current.setProtocol(protocol) + voiceChangerClientRef.current!.setProtocol(protocol) } }, []) const setInputChunkNum = useMemo(() => { return async (num: number) => { await initializedPromise - voiceChangerClientRef.current.setInputChunkNum(num) + voiceChangerClientRef.current!.setInputChunkNum(num) } }, []) const setVoiceChangerMode = useMemo(() => { return async (val: VoiceChangerMode) => { await initializedPromise - voiceChangerClientRef.current.setVoiceChangerMode(val) - voiceChangerClientRef.current.stop() + voiceChangerClientRef.current!.setVoiceChangerMode(val) + voiceChangerClientRef.current!.stop() } }, []) @@ -120,14 +124,14 @@ export const useClient = (props: UseClientProps): ClientState => { const start = useMemo(() => { return async (mmvcServerUrl: string) => { await initializedPromise - voiceChangerClientRef.current.setServerUrl(mmvcServerUrl, true) - voiceChangerClientRef.current.start() + voiceChangerClientRef.current!.setServerUrl(mmvcServerUrl, true) + voiceChangerClientRef.current!.start() } }, []) const stop = useMemo(() => { return async () => { await initializedPromise - voiceChangerClientRef.current.stop() + voiceChangerClientRef.current!.stop() } }, []) @@ -136,15 +140,14 @@ export const useClient = (props: UseClientProps): ClientState => { const changeInput = useMemo(() => { return async (audioInput: MediaStream | string, bufferSize: BufferSize, vfForceDisable: boolean) => { await initializedPromise - if (!props.audioContext) return if (!audioInput || audioInput == "none") { console.log("[useClient] setup!(1)", audioInput) - const ms = createDummyMediaStream(props.audioContext) - await voiceChangerClientRef.current.setup(ms, bufferSize, vfForceDisable) + const ms = createDummyMediaStream(props.audioContext!) + await voiceChangerClientRef.current!.setup(ms, bufferSize, vfForceDisable) } else { console.log("[useClient] setup!(2)", audioInput) - await voiceChangerClientRef.current.setup(audioInput, bufferSize, vfForceDisable) + await voiceChangerClientRef.current!.setup(audioInput, bufferSize, vfForceDisable) } } }, [props.audioContext]) @@ -155,8 +158,8 @@ export const useClient = (props: UseClientProps): ClientState => { const uploadFile = useMemo(() => { return async (file: File, onprogress: (progress: number, end: boolean) => void) => { await initializedPromise - const num = await voiceChangerClientRef.current.uploadFile(file, onprogress) - const res = await voiceChangerClientRef.current.concatUploadedFile(file, num) + const num = await voiceChangerClientRef.current!.uploadFile(file, onprogress) + const res = await voiceChangerClientRef.current!.concatUploadedFile(file, num) console.log("uploaded", num, res) } }, []) @@ -164,7 +167,7 @@ export const useClient = (props: UseClientProps): ClientState => { const loadModel = useMemo(() => { return async (configFile: File, pyTorchModelFile: File | null, onnxModelFile: File | null) => { await initializedPromise - await voiceChangerClientRef.current.loadModel(configFile, pyTorchModelFile, onnxModelFile) + await voiceChangerClientRef.current!.loadModel(configFile, pyTorchModelFile, onnxModelFile) console.log("loaded model") } }, []) @@ -172,7 +175,7 @@ export const useClient = (props: UseClientProps): ClientState => { const updateSettings = useMemo(() => { return async (key: ServerSettingKey, val: string | number) => { await initializedPromise - return await voiceChangerClientRef.current.updateServerSettings(key, "" + val) + return await voiceChangerClientRef.current!.updateServerSettings(key, "" + val) } }, []) @@ -180,8 +183,8 @@ export const useClient = (props: UseClientProps): ClientState => { const getInfo = useMemo(() => { return async () => { await initializedPromise - const serverSettings = await voiceChangerClientRef.current.getServerSettings() - const clientSettings = await voiceChangerClientRef.current.getClientSettings() + const serverSettings = await voiceChangerClientRef.current!.getServerSettings() + const clientSettings = await voiceChangerClientRef.current!.getClientSettings() console.log(serverSettings, clientSettings) } }, []) diff --git a/client/lib/src/ServerConfigurator.ts b/client/lib/src/ServerConfigurator.ts index b17c4bed..cf308580 100644 --- a/client/lib/src/ServerConfigurator.ts +++ b/client/lib/src/ServerConfigurator.ts @@ -28,7 +28,7 @@ export class ServerConfigurator { updateSettings = async (key: ServerSettingKey, val: string) => { const url = this.serverUrl + "/update_setteings" - const p = new Promise((resolve) => { + const info = await new Promise(async (resolve) => { const formData = new FormData(); formData.append("key", key); formData.append("val", val); @@ -36,12 +36,9 @@ export class ServerConfigurator { method: 'POST', body: formData, }); - fetch(request).then(async (response) => { - console.log(await response.json()) - resolve() - }) + const res = await (await fetch(request)).json() as ServerInfo + resolve(res) }) - const info = await p return info } @@ -113,7 +110,7 @@ export class ServerConfigurator { loadModel = async (configFile: File, pyTorchModelFile: File | null, onnxModelFile: File | null) => { const url = this.serverUrl + "/load_model" - const loadP = new Promise((resolve) => { + const info = new Promise(async (resolve) => { const formData = new FormData(); formData.append("pyTorchModelFilename", pyTorchModelFile?.name || "-"); formData.append("onnxModelFilename", onnxModelFile?.name || "-"); @@ -122,11 +119,9 @@ export class ServerConfigurator { method: 'POST', body: formData, }); - fetch(request).then(async (response) => { - console.log(await response.text()) - resolve() - }) + const res = await (await fetch(request)).json() as ServerInfo + resolve(res) }) - await loadP + return await info } } \ No newline at end of file diff --git a/client/lib/src/VoiceChangerClient.ts b/client/lib/src/VoiceChangerClient.ts index 6a0756f0..b55c41e2 100644 --- a/client/lib/src/VoiceChangerClient.ts +++ b/client/lib/src/VoiceChangerClient.ts @@ -77,7 +77,6 @@ export class VoiceChnagerClient { this.vcNode.connect(this.currentMediaStreamAudioDestinationNode) // vc node -> output node // (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる) this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, }) - // this.audioStreamer.setRequestParams(DefaultVoiceChangerRequestParamas) this.audioStreamer.setInputChunkNum(DefaultVoiceChangerOptions.inputChunkNum) this.audioStreamer.setVoiceChangerMode(DefaultVoiceChangerOptions.voiceChangerMode) diff --git a/client/lib/src/const.ts b/client/lib/src/const.ts index eabd9e8c..33027351 100644 --- a/client/lib/src/const.ts +++ b/client/lib/src/const.ts @@ -44,9 +44,7 @@ export type ServerInfo = { providers: string[] } - // Consts - export const Protocol = { "sio": "sio", "rest": "rest", diff --git a/server/requirements.txt b/server/requirements.txt index 7efb6bff..e4e850be 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -17,4 +17,21 @@ tqdm==4.64.1 Unidecode==1.3.6 uvicorn==0.20.0 websockets==10.4 -onnxruntime==1.13.1 \ No newline at end of file +onnxruntime==1.13.1 +onnxruntime-gpu==1.13.1 +#onnxruntime-openvino==1.13.1 +#onnxruntime-directml==1.13.1 + +#torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113 +#torch==1.12.1+cu113 +#torchaudio==0.12.1+cu113 +#torchvision==0.13.1+cu113 +# conda install pytorch torchvision torchaudio pytorch-cuda=11.6 -c pytorch -c nvidia + +# conda install pytorch==1.13.0+cu116 torchvision torchaudio pytorch-cuda=11.6 -c pytorch -c nvidia + + +#pip install torch==1.13.0+cu116 torchvision==0.13.1+cu116 torchaudio==0.13.0 --extra-index-url https://download.pytorch.org/whl/cu116 + +# conda install pytorch torchvision torchaudio pytorch-cuda=11.7 -c pytorch -c nvidia + diff --git a/server/restapi/MMVC_Rest_Fileuploader.py b/server/restapi/MMVC_Rest_Fileuploader.py index 9008ac65..aa32a8d4 100644 --- a/server/restapi/MMVC_Rest_Fileuploader.py +++ b/server/restapi/MMVC_Rest_Fileuploader.py @@ -28,12 +28,14 @@ class MMVC_Rest_Fileuploader: self.onnx_provider="" def post_upload_file(self, file: UploadFile = File(...), filename: str = Form(...)): - return upload_file(UPLOAD_DIR, file, filename) + res = upload_file(UPLOAD_DIR, file, filename) + json_compatible_item_data = jsonable_encoder(res) + return JSONResponse(content=json_compatible_item_data) def post_concat_uploaded_file(self, filename: str = Form(...), filenameChunkNum: int = Form(...)): - modelFilePath = concat_file_chunks( - UPLOAD_DIR, filename, filenameChunkNum, UPLOAD_DIR) - return {"concat": f"{modelFilePath}"} + res = concat_file_chunks(UPLOAD_DIR, filename, filenameChunkNum, UPLOAD_DIR) + json_compatible_item_data = jsonable_encoder(res) + return JSONResponse(content=json_compatible_item_data) def get_info(self): info = self.voiceChangerManager.get_info() @@ -57,8 +59,10 @@ class MMVC_Rest_Fileuploader: onnxModelFilePath = os.path.join(UPLOAD_DIR, onnxModelFilename) if onnxModelFilename != "-" else None configFilePath = os.path.join(UPLOAD_DIR, configFilename) - self.voiceChangerManager.loadModel(configFilePath, pyTorchModelFilePath, onnxModelFilePath) - return {"load": f"{configFilePath}, {pyTorchModelFilePath}, {onnxModelFilePath}"} + info = self.voiceChangerManager.loadModel(configFilePath, pyTorchModelFilePath, onnxModelFilePath) + json_compatible_item_data = jsonable_encoder(info) + return JSONResponse(content=json_compatible_item_data) + # return {"load": f"{configFilePath}, {pyTorchModelFilePath}, {onnxModelFilePath}"} def post_load_model_for_train( @@ -82,4 +86,5 @@ class MMVC_Rest_Fileuploader: zipFilePath = concat_file_chunks( UPLOAD_DIR, zipFilename, zipFileChunkNum, UPLOAD_DIR) shutil.unpack_archive(zipFilePath, "MMVC_Trainer/dataset/textful/") - return {"Zip file unpacked": f"{zipFilePath}"} \ No newline at end of file + return {"Zip file unpacked": f"{zipFilePath}"} + \ No newline at end of file diff --git a/server/restapi/mods/FileUploader.py b/server/restapi/mods/FileUploader.py index b7bf6db8..7b25372d 100755 --- a/server/restapi/mods/FileUploader.py +++ b/server/restapi/mods/FileUploader.py @@ -9,8 +9,9 @@ def upload_file(upload_dirname:str, file:UploadFile, filename: str): upload_dir = open(os.path.join(upload_dirname, filename),'wb+') shutil.copyfileobj(fileobj, upload_dir) upload_dir.close() - return {"uploaded files": f"{filename} "} - return {"Error": "uploaded file is not found."} + + return {"status":"OK", "msg": f"uploaded files {filename} "} + return {"status":"ERROR", "msg": "uploaded file is not found."} def concat_file_chunks(upload_dirname:str, filename:str, chunkNum:int, dest_dirname:str): target_file_name = os.path.join(dest_dirname, filename) @@ -25,5 +26,5 @@ def concat_file_chunks(upload_dirname:str, filename:str, chunkNum:int, dest_dirn stored_chunk_file.close() os.unlink(chunk_file_path) target_file.close() - return target_file_name + return {"status":"OK", "msg": f"concat files {target_file_name} "} diff --git a/server/voice_changer/VoiceChanger.py b/server/voice_changer/VoiceChanger.py index 9ececb9e..d9d4ec5b 100755 --- a/server/voice_changer/VoiceChanger.py +++ b/server/voice_changer/VoiceChanger.py @@ -40,6 +40,8 @@ class VoiceChanger(): # 初期化 self.settings = VocieChangerSettings(config_file=config) self.unpackedData_length=0 + self.net_g = None + self.onnx_session = None # 共通で使用する情報を収集 self.hps = utils.get_hparams_from_file(config) self.gpu_num = torch.cuda.device_count() @@ -68,8 +70,6 @@ class VoiceChanger(): **self.hps.model) self.net_g.eval() utils.load_checkpoint(pyTorch_model_file, self.net_g, None) - if hasattr(self, "net_g") == False: - self.net_g = None # ONNXモデル生成 if onnx_model_file != None: @@ -79,14 +79,11 @@ class VoiceChanger(): onnx_model_file, providers=providers ) - if hasattr(self, "onnx_session") == False: - self.onnx_session = None + return self.get_info() def destroy(self): - if hasattr(self, "net_g"): - del self.net_g - if hasattr(self, "onnx_session"): - del self.onnx_session + del self.net_g + del self.onnx_session def get_info(self): data = asdict(self.settings) @@ -108,7 +105,6 @@ class VoiceChanger(): self.onnx_session.set_providers(providers=[val], provider_options=provider_options) else: self.onnx_session.set_providers(providers=[val]) - return self.get_info() elif key in self.settings.intData: setattr(self.settings, key, int(val)) if key == "gpu" and val >= 0 and val < self.gpu_num and self.onnx_session != None: @@ -119,16 +115,13 @@ class VoiceChanger(): self.onnx_session.set_providers(providers=["CUDAExecutionProvider"], provider_options=provider_options) if key == "crossFadeOffsetRate" or key == "crossFadeEndRate": self.unpackedData_length = 0 - return self.get_info() elif key in self.settings.floatData: setattr(self.settings, key, float(val)) - return self.get_info() elif key in self.settings.strData: setattr(self.settings, key, str(val)) - return self.get_info() else: print(f"{key} is not mutalbe variable!") - return self.get_info() + return self.get_info() diff --git a/server/voice_changer/VoiceChangerManager.py b/server/voice_changer/VoiceChangerManager.py index 678cdd5b..397a86ee 100644 --- a/server/voice_changer/VoiceChangerManager.py +++ b/server/voice_changer/VoiceChangerManager.py @@ -13,19 +13,25 @@ class VoiceChangerManager(): def loadModel(self, config, model, onnx_model): if hasattr(self, 'voiceChanger') == False: self.voiceChanger = VoiceChanger(config) - self.voiceChanger.loadModel(config, model, onnx_model) + info = self.voiceChanger.loadModel(config, model, onnx_model) + info["status"]="OK" + return info def get_info(self): if hasattr(self, 'voiceChanger'): - return self.voiceChanger.get_info() + info = self.voiceChanger.get_info() + info["status"]="OK" + return info else: - return {"no info":"no info"} + return {"status":"ERROR", "msg":"no model loaded"} def update_setteings(self, key:str, val:any): if hasattr(self, 'voiceChanger'): - return self.voiceChanger.update_setteings(key, val) + info = self.voiceChanger.update_setteings(key, val) + info["status"]="OK" + return info else: - return {"no info":"no info"} + return {"status":"ERROR", "msg":"no model loaded"} def changeVoice(self, unpackedData:any): if hasattr(self, 'voiceChanger') == True: