update

demo/SoftVcServerFastAPI.py

@@ -30,10 +30,10 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-app.mount("/front", StaticFiles(directory="voice-changer/frontend/dist", html=True), name="static")
 
 if MODE == "colab":
     print("ENV: colab")
+    app.mount("/front", StaticFiles(directory="voice-changer/frontend/dist", html=True), name="static")
     
     hubert_model = torch.hub.load("bshall/hubert:main", "hubert_soft").cuda()
     acoustic_model = torch.hub.load("bshall/acoustic-model:main", "hubert_soft").cuda()

demo/SoftVcServerSIO.py

@@ -0,0 +1,116 @@
+import eventlet
+import socketio
+import sys, math, base64
+from datetime import datetime
+import struct
+
+import torch, torchaudio
+import numpy as np
+from scipy.io.wavfile import write, read
+
+
+sys.path.append("/hubert")
+from hubert import hubert_discrete, hubert_soft, kmeans100
+
+sys.path.append("/acoustic-model")
+from acoustic import hubert_discrete, hubert_soft
+
+sys.path.append("/hifigan")
+from hifigan import hifigan
+
+hubert_model = torch.load("/models/bshall_hubert_main.pt").cuda()
+acoustic_model = torch.load("/models/bshall_acoustic-model_main.pt").cuda()
+hifigan_model = torch.load("/models/bshall_hifigan_main.pt").cuda()
+
+
+def applyVol(i, chunk, vols):
+  curVol = vols[i] / 2
+  if curVol < 0.0001:
+    line = torch.zeros(chunk.size())
+  else:
+    line = torch.ones(chunk.size())
+
+  volApplied = torch.mul(line, chunk)  
+  volApplied = volApplied.unsqueeze(0)
+  return volApplied
+
+
+class MyCustomNamespace(socketio.Namespace): 
+    def __init__(self, namespace):
+        super().__init__(namespace)
+
+    def on_connect(self, sid, environ):
+        print('[{}] connet sid : {}'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S') , sid))
+
+    def on_request_message(self, sid, msg): 
+        print("Processing Request...")
+        gpu = int(msg[0])
+        srcId = int(msg[1])
+        dstId = int(msg[2])
+        timestamp = int(msg[3])
+        data = msg[4]
+        # print(srcId, dstId, timestamp)
+        unpackedData = np.array(struct.unpack('<%sh'%(len(data) // struct.calcsize('<h') ), data))
+        write("logs/received_data.wav", 24000, unpackedData.astype(np.int16))
+
+        source, sr = torchaudio.load("logs/received_data.wav") # デフォルトでnormalize=Trueがついており、float32に変換して読んでくれるらしいのでこれを使う。https://pytorch.org/audio/stable/backend.html
+
+        source_16k = torchaudio.functional.resample(source, 24000, 16000)
+        source_16k = source_16k.unsqueeze(0).cuda()
+        # SOFT-VC
+        with torch.inference_mode():
+            units = hubert_model.units(source_16k)
+            mel = acoustic_model.generate(units).transpose(1, 2)
+            target = hifigan_model(mel)
+
+        dest = torchaudio.functional.resample(target, 16000,24000)
+        dest = dest.squeeze().cpu()
+
+        # ソースの音量取得
+        source = source.cpu()
+        specgram = torchaudio.transforms.MelSpectrogram(sample_rate=24000)(source)
+        vol_apply_window_size = math.ceil(len(source[0]) / specgram.size()[2])
+        specgram = specgram.transpose(1,2)
+        vols = [ torch.max(i) for i in specgram[0]]
+        chunks = torch.split(dest, vol_apply_window_size,0)
+
+        chunks = [applyVol(i,c,vols) for i, c in enumerate(chunks)]
+        dest = torch.cat(chunks,1)
+        arr = np.array(dest.squeeze())
+
+        int_size = 2**(16 - 1) - 1
+        arr = (arr * int_size).astype(np.int16)
+        # write("logs/converted_data.wav", 24000, arr)
+        # changedVoiceBase64 = base64.b64encode(arr).decode('utf-8')
+
+        # data = {
+        #     "gpu":gpu,
+        #     "srcId":srcId,
+        #     "dstId":dstId,
+        #     "timestamp":timestamp,
+        #     "changedVoiceBase64":changedVoiceBase64
+        # }
+
+        # audio1 = audio1.astype(np.int16)
+        bin = struct.pack('<%sh'%len(arr), *arr)
+
+        # print("return timestamp", timestamp)        
+        self.emit('response',[timestamp, bin])
+
+    def on_disconnect(self, sid):
+        # print('[{}] disconnect'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
+        pass;
+
+
+if __name__ == '__main__':
+    args = sys.argv
+    PORT = args[1]
+    print(f"start... PORT:{PORT}")    
+    sio = socketio.Server(cors_allowed_origins='*') 
+    sio.register_namespace(MyCustomNamespace('/test')) 
+    app = socketio.WSGIApp(sio,static_files={
+        '': '../frontend/dist',
+        '/': '../frontend/dist/index.html',
+    }) 
+    eventlet.wsgi.server(eventlet.listen(('0.0.0.0',int(PORT))), app) 
+    

demo/setup.sh

@@ -1,27 +1,38 @@
 #!/bin/bash
 
-CONFIG=$1
+TYPE=$1
-MODEL=$2
+CONFIG=$2
-TYPE=$3
+MODEL=$3
 
+echo type: $TYPE
 echo config: $CONFIG
 echo model: $MODEL
-echo type: $TYPE
 
 
 
 cp -r /resources/* .
 
+## Config 設置
 if [[ -e ./setting.json ]]; then
+  echo "カスタムセッティングを使用"
   cp ./setting.json ../frontend/dist/assets/setting.json
+else
+  if [ "${TYPE}" = "SOFT_VC" ] ; then
+    cp ../frontend/dist/assets/setting_softvc.json ../frontend/dist/assets/setting.json
+  elif [ "${TYPE}" = "SOFT_VC_FAST_API" ] ; then
+    cp ../frontend/dist/assets/setting_softvc_colab.json ../frontend/dist/assets/setting.json
+  else
+    cp ../frontend/dist/assets/setting_mmvc.json ../frontend/dist/assets/setting.json  
+  fi
 fi
 
+# 起動
 if [ "${TYPE}" = "SOFT_VC" ] ; then
   echo "SOFT_VCを起動します"
-  python3 SoftVcServerFlask.py 8080
+  python3 SoftVcServerSIO.py 8080
 elif [ "${TYPE}" = "SOFT_VC_FAST_API" ] ; then
   echo "SOFT_VC_FAST_APIを起動します"
-  python3 SoftVcServerFastAPI.py 8080
+  python3 SoftVcServerFastAPI.py 8080 docker
 else
   echo "MMVCを起動します"
   python3 serverSIO.py 8080 $CONFIG $MODEL

frontend/dist/assets/setting.json

@@ -1,7 +1,7 @@
 {
     "app_title": "voice-changer",
     "majar_mode": "docker",
-    "voice_changer_server_url": "./test",
+    "voice_changer_server_url": "/test",
     "sample_rate": 48000,
     "buffer_size": 1024,
     "prefix_chunk_size": 24,

frontend/dist/assets/setting_mmvc.json

@@ -0,0 +1,38 @@
+{
+    "app_title": "voice-changer",
+    "majar_mode": "docker",
+    "voice_changer_server_url": "/test",
+    "sample_rate": 48000,
+    "buffer_size": 1024,
+    "prefix_chunk_size": 24,
+    "chunk_size": 24,
+    "speaker_ids": [100, 107, 101, 102, 103],
+    "speaker_names": ["ずんだもん", "user", "そら", "めたん", "つむぎ"],
+    "src_id": 107,
+    "dst_id": 100,
+    "vf_enable": true,
+    "voice_changer_mode": "realtime",
+    "gpu": 0,
+    "available_gpus": [-1, 0, 1, 2, 3, 4],
+    "avatar": {
+        "enable_avatar": true,
+        "motion_capture_face": true,
+        "motion_capture_upperbody": true,
+        "lip_overwrite_with_voice": true,
+        "avatar_url": "./assets/vrm/zundamon/zundamon.vrm",
+        "backgournd_image_url": "./assets/images/bg_natural_sougen.jpg",
+        "background_color": "#0000dd",
+        "chroma_key": "#0000dd",
+        "avatar_canvas_size": [1280, 720],
+        "screen_canvas_size": [1280, 720]
+    },
+    "advance": {
+        "avatar_draw_skip_rate": 3,
+        "screen_draw_skip_rate": 3,
+        "visualizer_draw_skip_rate": 3,
+        "cross_fade_lower_value": 0.1,
+        "cross_fade_offset_rate": 0.3,
+        "cross_fade_end_rate": 0.6,
+        "cross_fade_type": 2
+    }
+}

frontend/dist/assets/setting_softvc.json

@@ -0,0 +1,38 @@
+{
+    "app_title": "voice-changer[soft-vc]",
+    "majar_mode": "docker",
+    "voice_changer_server_url": "/test",
+    "sample_rate": 48000,
+    "buffer_size": 1024,
+    "prefix_chunk_size": 60,
+    "chunk_size": 60,
+    "speaker_ids": [999, 107],
+    "speaker_names": ["---", "user"],
+    "src_id": 107,
+    "dst_id": 999,
+    "vf_enable": true,
+    "voice_changer_mode": "realtime",
+    "gpu": 0,
+    "available_gpus": [0],
+    "avatar": {
+        "enable_avatar": true,
+        "motion_capture_face": true,
+        "motion_capture_upperbody": true,
+        "lip_overwrite_with_voice": true,
+        "avatar_url": "./assets/vrm/zundamon/zundamon.vrm",
+        "backgournd_image_url": "./assets/images/bg_natural_sougen.jpg",
+        "background_color": "#0000dd",
+        "chroma_key": "#0000dd",
+        "avatar_canvas_size": [1280, 720],
+        "screen_canvas_size": [1280, 720]
+    },
+    "advance": {
+        "avatar_draw_skip_rate": 3,
+        "screen_draw_skip_rate": 3,
+        "visualizer_draw_skip_rate": 3,
+        "cross_fade_lower_value": 0.1,
+        "cross_fade_offset_rate": 0.3,
+        "cross_fade_end_rate": 0.6,
+        "cross_fade_type": 2
+    }
+}

frontend/dist/assets/setting_softvc_colab.json

@@ -0,0 +1,38 @@
+{
+    "app_title": "voice-changer[soft-vc]",
+    "majar_mode": "colab",
+    "voice_changer_server_url": "/test",
+    "sample_rate": 48000,
+    "buffer_size": 1024,
+    "prefix_chunk_size": 60,
+    "chunk_size": 60,
+    "speaker_ids": [999, 107],
+    "speaker_names": ["---", "user"],
+    "src_id": 107,
+    "dst_id": 999,
+    "vf_enable": true,
+    "voice_changer_mode": "realtime",
+    "gpu": 0,
+    "available_gpus": [0],
+    "avatar": {
+        "enable_avatar": true,
+        "motion_capture_face": true,
+        "motion_capture_upperbody": true,
+        "lip_overwrite_with_voice": true,
+        "avatar_url": "./assets/vrm/zundamon/zundamon.vrm",
+        "backgournd_image_url": "./assets/images/bg_natural_sougen.jpg",
+        "background_color": "#0000dd",
+        "chroma_key": "#0000dd",
+        "avatar_canvas_size": [1280, 720],
+        "screen_canvas_size": [1280, 720]
+    },
+    "advance": {
+        "avatar_draw_skip_rate": 3,
+        "screen_draw_skip_rate": 3,
+        "visualizer_draw_skip_rate": 3,
+        "cross_fade_lower_value": 0.1,
+        "cross_fade_offset_rate": 0.3,
+        "cross_fade_end_rate": 0.6,
+        "cross_fade_type": 2
+    }
+}

frontend/dist/index.html

@@ -1,13 +1 @@
-<!DOCTYPE html>
+<!doctype html><html lang="ja" style="width:100%;height:100%;overflow:hidden"><head><meta charset="utf-8"/><title>voice recorder</title><script defer="defer" src="index.js"></script></head><body style="width:100%;height:100%;margin:0"><div id="app" style="width:100%;height:100%"></div><noscript><strong>javascriptを有効にしてください</strong></noscript></body></html>
-<html lang="ja" style="width: 100%; height: 100%; overflow: hidden">
-    <head>
-        <meta charset="utf-8" />
-        <title>voice recorder</title>
-    <script defer src="index.js"></script></head>
-    <body style="width: 100%; height: 100%; margin: 0px">
-        <div id="app" style="width: 100%; height: 100%"></div>
-        <noscript>
-            <strong>javascriptを有効にしてください</strong>
-        </noscript>
-    </body>
-</html>

start2.sh

@@ -2,7 +2,7 @@
 
 # 参考:https://programwiz.org/2022/03/22/how-to-write-shell-script-for-option-parsing/
 
-DOCKER_IMAGE=dannadori/voice-changer:20220918_220447
+DOCKER_IMAGE=dannadori/voice-changer:20220919_043908
 TENSORBOARD_PORT=6006
 VOICE_CHANGER_PORT=8081
 

trainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM dannadori/voice-changer-internal:20220918_215800 as front
+FROM dannadori/voice-changer-internal:20220919_043748 as front
 FROM debian:bullseye-slim as base
 
 ARG DEBIAN_FRONTEND=noninteractive