diff --git a/.gitignore b/.gitignore index 4670c0ee..ac0e8ec5 100644 --- a/.gitignore +++ b/.gitignore @@ -40,9 +40,6 @@ client/lib/worklet/dist docker/cudnn/ -server/hubert_base.pt -server/hubert-soft-0d54a1f4.pt -server/nsf_hifigan/ - +server/weights/ start_trainer.sh diff --git a/server/Exceptions.py b/server/Exceptions.py index 6b89d065..82819c8d 100644 --- a/server/Exceptions.py +++ b/server/Exceptions.py @@ -5,3 +5,8 @@ class NoModeLoadedException(Exception): def __str__(self): return repr(f"No model for {self.framework} loaded. Please confirm the model uploaded.") + + +class ONNXInputArgumentException(Exception): + def __str__(self): + return repr(f"ONNX received invalid argument.") diff --git a/server/MMVCServerSIO.py b/server/MMVCServerSIO.py index 1e175360..35308465 100755 --- a/server/MMVCServerSIO.py +++ b/server/MMVCServerSIO.py @@ -42,11 +42,18 @@ def setupArgParser(): parser.add_argument("--cluster", type=str, help="path to cluster model") parser.add_argument("--internal", type=strtobool, default=False, help="各種パスをmac appの中身に変換") - parser.add_argument("--hubert", type=str, help="path to hubert model") - parser.add_argument("--useHubertOnnx", type=strtobool, default=False, help="use hubert onnx") - parser.add_argument("--hubertSoftPt", type=str, help="path to hubert-soft model(pytorch)") - parser.add_argument("--enhancerPt", type=str, help="path to enhancer model(pytorch)") - parser.add_argument("--enhancerOnnx", type=str, help="path to enhancer model(onnx)") + # parser.add_argument("--hubert", type=str, help="path to hubert model") + # parser.add_argument("--useHubertOnnx", type=strtobool, default=False, help="use hubert onnx") + # parser.add_argument("--hubertSoftPt", type=str, help="path to hubert-soft model(pytorch)") + # parser.add_argument("--enhancerPt", type=str, help="path to enhancer model(pytorch)") + # parser.add_argument("--enhancerOnnx", type=str, help="path to enhancer model(onnx)") + + parser.add_argument("--content_vec_500", type=str, help="path to content_vec_500 model(pytorch)") + parser.add_argument("--content_vec_500_onnx", type=str, help="path to content_vec_500 model(onnx)") + parser.add_argument("--content_vec_500_onnx_on", type=strtobool, default=False, help="use or not onnx for content_vec_500") + parser.add_argument("--hubert_base", type=str, help="path to hubert_base model(pytorch)") + parser.add_argument("--hubert_soft", type=str, help="path to hubert_soft model(pytorch)") + parser.add_argument("--nsf_hifigan", type=str, help="path to nsf_hifigan model(pytorch)") return parser @@ -129,11 +136,12 @@ if args.colab == True: if __name__ == 'MMVCServerSIO': voiceChangerManager = VoiceChangerManager.get_instance({ - "hubert": args.hubert, - "useHubertOnnx": args.useHubertOnnx, - "hubertSoftPt": args.hubertSoftPt, - "enhancerPt": args.enhancerPt, - "enhancerOnnx": args.enhancerOnnx + "content_vec_500": args.content_vec_500, + "content_vec_500_onnx": args.content_vec_500_onnx, + "content_vec_500_onnx_on": args.content_vec_500_onnx_on, + "hubert_base": args.hubert_base, + "hubert_soft": args.hubert_soft, + "nsf_hifigan": args.nsf_hifigan, }) if CONFIG and (MODEL or ONNX_MODEL): if MODEL_TYPE == "MMVCv15" or MODEL_TYPE == "MMVCv13": diff --git a/server/voice_changer/DDSP_SVC/DDSP_SVC.py b/server/voice_changer/DDSP_SVC/DDSP_SVC.py index 5731f753..86a9b6bb 100644 --- a/server/voice_changer/DDSP_SVC/DDSP_SVC.py +++ b/server/voice_changer/DDSP_SVC/DDSP_SVC.py @@ -84,7 +84,7 @@ class DDSP_SVC: print("-------------------hopsize", self.hop_size) # hubert - self.vec_path = self.params["hubertSoftPt"] + self.vec_path = self.params["hubert_soft"] self.encoder = vo.Units_Encoder( self.args.data.encoder, self.vec_path, @@ -115,7 +115,7 @@ class DDSP_SVC: float(1100)) self.volume_extractor = vo.Volume_Extractor(self.hop_size) - self.enhancer_path = self.params["enhancerPt"] + self.enhancer_path = self.params["nsf_hifigan"] self.enhancer = Enhancer(self.args.enhancer.type, self.enhancer_path, device=self.useDevice()) return self.get_info() diff --git a/server/voice_changer/MMVCv15/MMVCv15.py b/server/voice_changer/MMVCv15/MMVCv15.py index 2081be45..36f2020d 100644 --- a/server/voice_changer/MMVCv15/MMVCv15.py +++ b/server/voice_changer/MMVCv15/MMVCv15.py @@ -20,7 +20,7 @@ import pyworld as pw from models import SynthesizerTrn from voice_changer.MMVCv15.client_modules import convert_continuos_f0, spectrogram_torch, get_hparams_from_file, load_checkpoint -from Exceptions import NoModeLoadedException +from Exceptions import NoModeLoadedException, ONNXInputArgumentException providers = ['OpenVINOExecutionProvider', "CUDAExecutionProvider", "DmlExecutionProvider", "CPUExecutionProvider"] @@ -241,11 +241,15 @@ class MMVCv15: return result def inference(self, data): - if self.settings.framework == "ONNX": - audio = self._onnx_inference(data) - else: - audio = self._pyTorch_inference(data) - return audio + try: + if self.settings.framework == "ONNX": + audio = self._onnx_inference(data) + else: + audio = self._pyTorch_inference(data) + return audio + except onnxruntime.capi.onnxruntime_pybind11_state.InvalidArgument as e: + print(e) + raise ONNXInputArgumentException() def __del__(self): del self.net_g diff --git a/server/voice_changer/RVC/RVC.py b/server/voice_changer/RVC/RVC.py index 3a3fc84b..3897d217 100644 --- a/server/voice_changer/RVC/RVC.py +++ b/server/voice_changer/RVC/RVC.py @@ -94,7 +94,7 @@ class RVC: print("[Voice Changer] RVC loading... slot:", self.slot) try: - hubert_path = self.params["hubert"] + hubert_path = self.params["hubert_base"] models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task([hubert_path], suffix="",) model = models[0] model.eval() diff --git a/server/voice_changer/SoVitsSvc40/SoVitsSvc40.py b/server/voice_changer/SoVitsSvc40/SoVitsSvc40.py index 8d274cdf..94b97140 100644 --- a/server/voice_changer/SoVitsSvc40/SoVitsSvc40.py +++ b/server/voice_changer/SoVitsSvc40/SoVitsSvc40.py @@ -83,22 +83,27 @@ class SoVitsSvc40: self.settings.onnxModelFile = props["files"]["onnxModelFilename"] clusterTorchModel = props["files"]["clusterTorchModelFilename"] + content_vec_path = self.params["content_vec_500"] + content_vec_hubert_onnx_path = self.params["content_vec_500_onnx"] + content_vec_hubert_onnx_on = self.params["content_vec_500_onnx_on"] + hubert_base_path = self.params["hubert_base"] + # hubert model try: - hubert_path = self.params["hubert"] - useHubertOnnx = self.params["useHubertOnnx"] - self.useHubertOnnx = useHubertOnnx - if useHubertOnnx == True: + if os.path.exists(content_vec_path) == False: + content_vec_path = hubert_base_path + + if content_vec_hubert_onnx_on == True: ort_options = onnxruntime.SessionOptions() ort_options.intra_op_num_threads = 8 self.hubert_onnx = onnxruntime.InferenceSession( - HUBERT_ONNX_MODEL_PATH, + content_vec_hubert_onnx_path, providers=providers ) else: models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task( - [hubert_path], + [content_vec_path], suffix="", ) model = models[0] diff --git a/server/voice_changer/SoVitsSvc40v2/SoVitsSvc40v2.py b/server/voice_changer/SoVitsSvc40v2/SoVitsSvc40v2.py index 0120bce2..0bee0664 100644 --- a/server/voice_changer/SoVitsSvc40v2/SoVitsSvc40v2.py +++ b/server/voice_changer/SoVitsSvc40v2/SoVitsSvc40v2.py @@ -78,16 +78,18 @@ class SoVitsSvc40v2: self.settings.onnxModelFile = props["files"]["onnxModelFilename"] clusterTorchModel = props["files"]["clusterTorchModelFilename"] + content_vec_path = self.params["content_vec_500"] + # content_vec_hubert_onnx_path = self.params["content_vec_500_onnx"] + # content_vec_hubert_onnx_on = self.params["content_vec_500_onnx_on"] + hubert_base_path = self.params["hubert_base"] + # hubert model try: - # if sys.platform.startswith('darwin'): - # vec_path = os.path.join(sys._MEIPASS, "hubert/checkpoint_best_legacy_500.pt") - # else: - # vec_path = "hubert/checkpoint_best_legacy_500.pt" - vec_path = self.params["hubert"] + if os.path.exists(content_vec_path) == False: + content_vec_path = hubert_base_path models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task( - [vec_path], + [content_vec_path], suffix="", ) model = models[0] diff --git a/server/voice_changer/VoiceChanger.py b/server/voice_changer/VoiceChanger.py index 2f5b668b..15e29a36 100755 --- a/server/voice_changer/VoiceChanger.py +++ b/server/voice_changer/VoiceChanger.py @@ -14,7 +14,7 @@ from voice_changer.IORecorder import IORecorder from voice_changer.utils.Timer import Timer from voice_changer.utils.VoiceChangerModel import VoiceChangerModel, AudioInOut import time -from Exceptions import NoModeLoadedException +from Exceptions import NoModeLoadedException, ONNXInputArgumentException providers = ['OpenVINOExecutionProvider', "CUDAExecutionProvider", "DmlExecutionProvider", "CPUExecutionProvider"] @@ -289,6 +289,9 @@ class VoiceChanger(): except NoModeLoadedException as e: print("[Voice Changer] [Exception]", e) return np.zeros(1).astype(np.int16), [0, 0, 0] + except ONNXInputArgumentException as e: + print("[Voice Changer] [Exception]", e) + return np.zeros(1).astype(np.int16), [0, 0, 0] except Exception as e: print("VC PROCESSING!!!! EXCEPTION!!!", e) print(traceback.format_exc())