tuning

client/demo/src/css/App.css

@@ -157,6 +157,11 @@ body {
 .body-item-text {
     color: rgb(30, 30, 30);
 }
+
+.body-item-input {
+    width:90%;
+}
+
 .body-button-container {
     display: flex;
     flex-direction: row;
@@ -165,6 +170,22 @@ body {
         border: solid 1px #333;
         border-radius: 2px;
         padding: 2px;
+        cursor:pointer;
+    }
+    .body-button-active {
+        user-select: none;
+        border: solid 1px #333;
+        border-radius: 2px;
+        padding: 2px;
+        background:#ada;
+    }
+    .body-button-stanby {
+        user-select: none;
+        border: solid 1px #333;
+        border-radius: 2px;
+        padding: 2px;
+        background:#aba;
+        cursor:pointer;
     }
 }
 .body-select-container {

client/demo/src/index.tsx

@@ -1,18 +1,17 @@
 import * as React from "react";
 import { createRoot } from "react-dom/client";
 import "./css/App.css"
-import { useEffect, useMemo, useRef } from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
 import { VoiceChnagerClient } from "@dannadori/voice-changer-client-js"
 import { useMicrophoneOptions } from "./options_microphone";
 const container = document.getElementById("app")!;
 const root = createRoot(container);
 
 const App = () => {
-    const { component: microphoneSettingComponent, options: microphonOptions } = useMicrophoneOptions()
+    const { component: microphoneSettingComponent, options: microphoneOptions, params: microphoneParams, isStarted } = useMicrophoneOptions()
 
-    const voiceChnagerClientRef = useRef<VoiceChnagerClient | null>(null)
+    const voiceChangerClientRef = useRef<VoiceChnagerClient | null>(null)
-
+    const [clientInitialized, setClientInitialized] = useState<boolean>(false)
-    console.log(microphonOptions)
 
     const onClearSettingClicked = async () => {
         //@ts-ignore
@@ -23,32 +22,101 @@ const App = () => {
         location.reload()
     }
 
+
     useEffect(() => {
-        if (microphonOptions.audioInputDeviceId.length == 0) {
+        const initialized = async () => {
-            return
-        }
-        const setAudio = async () => {
             const ctx = new AudioContext()
-
+            voiceChangerClientRef.current = new VoiceChnagerClient(ctx, true, {
-            if (voiceChnagerClientRef.current) {
-
-            }
-            voiceChnagerClientRef.current = new VoiceChnagerClient(ctx, true, {
                 notifySendBufferingTime: (val: number) => { console.log(`buf:${val}`) },
                 notifyResponseTime: (val: number) => { console.log(`res:${val}`) },
-                notifyException: (mes: string) => { console.log(`error:${mes}`) }
+                notifyException: (mes: string) => {
-            })
+                    if (mes.length > 0) {
-            await voiceChnagerClientRef.current.isInitialized()
+                        console.log(`error:${mes}`)
-
+                    }
-            voiceChnagerClientRef.current.setServerUrl("https://192.168.0.3:18888/test", "sio")
+                }
-            voiceChnagerClientRef.current.setup(microphonOptions.audioInputDeviceId, 1024)
+            }, { notifyVolume: (vol: number) => { } })
+            await voiceChangerClientRef.current.isInitialized()
+            setClientInitialized(true)
 
             const audio = document.getElementById("audio-output") as HTMLAudioElement
-            audio.srcObject = voiceChnagerClientRef.current.stream
+            audio.srcObject = voiceChangerClientRef.current.stream
             audio.play()
         }
-        setAudio()
+        initialized()
-    }, [microphonOptions.audioInputDeviceId])
+    }, [])
+
+    useEffect(() => {
+        console.log("START!!!", isStarted)
+        const start = async () => {
+            if (!voiceChangerClientRef.current || !clientInitialized) {
+                console.log("client is not initialized")
+                return
+            }
+            // if (!microphoneOptions.audioInputDeviceId || microphoneOptions.audioInputDeviceId.length == 0) {
+            //     console.log("audioInputDeviceId is not initialized")
+            //     return
+            // }
+            // await voiceChangerClientRef.current.setup(microphoneOptions.audioInputDeviceId!, microphoneOptions.bufferSize)
+            voiceChangerClientRef.current.setServerUrl(microphoneOptions.mmvcServerUrl, microphoneOptions.protocol, true)
+            voiceChangerClientRef.current.start()
+        }
+        const stop = async () => {
+            if (!voiceChangerClientRef.current || !clientInitialized) {
+                console.log("client is not initialized")
+                return
+            }
+            voiceChangerClientRef.current.stop()
+        }
+        if (isStarted) {
+            start()
+        } else {
+            stop()
+        }
+    }, [isStarted])
+
+    // useEffect(() => {
+    //     if (!voiceChangerClientRef.current || !clientInitialized) {
+    //         console.log("client is not initialized")
+    //         return
+    //     }
+    //     voiceChangerClientRef.current.setServerUrl(microphoneOptions.mmvcServerUrl, microphoneOptions.protocol, false)
+    // }, [microphoneOptions.mmvcServerUrl, microphoneOptions.protocol])
+
+    useEffect(() => {
+        const changeInput = async () => {
+            if (!voiceChangerClientRef.current || !clientInitialized) {
+                console.log("client is not initialized")
+                return
+            }
+            await voiceChangerClientRef.current.setup(microphoneOptions.audioInputDeviceId!, microphoneOptions.bufferSize, microphoneOptions.forceVfDisable)
+        }
+        changeInput()
+    }, [microphoneOptions.audioInputDeviceId!, microphoneOptions.bufferSize, microphoneOptions.forceVfDisable])
+
+
+    useEffect(() => {
+        if (!voiceChangerClientRef.current || !clientInitialized) {
+            console.log("client is not initialized")
+            return
+        }
+        voiceChangerClientRef.current.setInputChunkNum(microphoneOptions.inputChunkNum)
+    }, [microphoneOptions.inputChunkNum])
+
+    useEffect(() => {
+        if (!voiceChangerClientRef.current || !clientInitialized) {
+            console.log("client is not initialized")
+            return
+        }
+        voiceChangerClientRef.current.setVoiceChangerMode(microphoneOptions.voiceChangerMode)
+    }, [microphoneOptions.voiceChangerMode])
+
+    useEffect(() => {
+        if (!voiceChangerClientRef.current || !clientInitialized) {
+            console.log("client is not initialized")
+            return
+        }
+        voiceChangerClientRef.current.setRequestParams(microphoneParams)
+    }, [microphoneParams])
 
     const clearRow = useMemo(() => {
         return (

client/demo/src/options_microphone.tsx

@@ -1,42 +1,7 @@
 import * as React from "react";
 import { useEffect, useMemo, useState } from "react";
 import { CHROME_EXTENSION } from "./const";
-import { Speaker, VoiceChangerMode, DefaultSpeakders, SampleRate, BufferSize } from "@dannadori/voice-changer-client-js"
+import { DefaultVoiceChangerRequestParamas, VoiceChangerOptions, VoiceChangerRequestParamas, DefaultVoiceChangerOptions, SampleRate, BufferSize, VoiceChangerMode, Protocol } from "@dannadori/voice-changer-client-js"
-
-export type MicrophoneOptionsState = {
-    audioInputDeviceId: string,
-    mmvcServerUrl: string,
-    sampleRate: number,
-    bufferSize: number,
-    chunkSize: number,
-    speakers: Speaker[],
-    srcId: number,
-    dstId: number,
-    vfEnabled: boolean,
-    voiceChangerMode: VoiceChangerMode,
-    gpu: number,
-
-    crossFadeLowerValue: number,
-    crossFadeOffsetRate: number,
-    crossFadeEndRate: number,
-}
-const InitMicrophoneOptionsState = {
-    audioInputDeviceId: "",
-    mmvcServerUrl: "https://localhost:5543/test",
-    sampleRate: 48000,
-    bufferSize: 1024,
-    chunkSize: 24,
-    speakers: DefaultSpeakders,
-    srcId: 107,
-    dstId: 100,
-    vfEnabled: true,
-    voiceChangerMode: VoiceChangerMode.realtime,
-    gpu: 0,
-
-    crossFadeLowerValue: 0.1,
-    crossFadeOffsetRate: 0.3,
-    crossFadeEndRate: 0.6,
-} as const
 
 
 const reloadDevices = async () => {
@@ -46,13 +11,24 @@ const reloadDevices = async () => {
         console.warn("Enumerate device error::", e)
     }
     const mediaDeviceInfos = await navigator.mediaDevices.enumerateDevices();
-    return mediaDeviceInfos.filter(x => { return x.kind == "audioinput" })
+
+    const audioInputs = mediaDeviceInfos.filter(x => { return x.kind == "audioinput" })
+    audioInputs.push({
+        deviceId: "none",
+        groupId: "none",
+        kind: "audioinput",
+        label: "none",
+        toJSON: () => { }
+    })
+    return audioInputs
 }
 
 
 export type MicrophoneOptionsComponent = {
     component: JSX.Element,
-    options: MicrophoneOptionsState
+    options: VoiceChangerOptions,
+    params: VoiceChangerRequestParamas
+    isStarted: boolean
 }
 
 export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
@@ -61,7 +37,10 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
     const [editSpeakerTargetId, setEditSpeakerTargetId] = useState<number>(0)
     const [editSpeakerTargetName, setEditSpeakerTargetName] = useState<string>("")
 
-    const [options, setOptions] = useState<MicrophoneOptionsState>(InitMicrophoneOptionsState)
+    // const [options, setOptions] = useState<MicrophoneOptionsState>(InitMicrophoneOptionsState)
+    const [params, setParams] = useState<VoiceChangerRequestParamas>(DefaultVoiceChangerRequestParamas)
+    const [options, setOptions] = useState<VoiceChangerOptions>(DefaultVoiceChangerOptions)
+    const [isStarted, setIsStarted] = useState<boolean>(false)
 
     useEffect(() => {
         const initialize = async () => {
@@ -90,6 +69,32 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
     }, [options]) // loadより前に持ってくるとstorage内が初期化されるのでだめかも。（要検証）
 
 
+    const startButtonRow = useMemo(() => {
+        const onStartClicked = () => {
+            setIsStarted(true)
+        }
+        const onStopClicked = () => {
+            setIsStarted(false)
+        }
+        const startClassName = isStarted ? "body-button-active" : "body-button-stanby"
+        const stopClassName = isStarted ? "body-button-stanby" : "body-button-active"
+        console.log("ClassName", startClassName, stopClassName)
+
+        return (
+            <div className="body-row split-3-3-4 left-padding-1">
+                <div className="body-item-title">Start</div>
+                <div className="body-button-container">
+                    <div onClick={onStartClicked} className={startClassName}>start</div>
+                    <div onClick={onStopClicked} className={stopClassName}>stop</div>
+                </div>
+                <div className="body-input-container">
+                </div>
+            </div>
+
+        )
+    }, [isStarted])
+
+
     const setAudioInputDeviceId = async (deviceId: string) => {
         setOptions({ ...options, audioInputDeviceId: deviceId })
     }
@@ -98,6 +103,10 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
         const input = document.getElementById("mmvc-server-url") as HTMLInputElement
         setOptions({ ...options, mmvcServerUrl: input.value })
     }
+    const onProtocolChanged = async (val: Protocol) => {
+        setOptions({ ...options, protocol: val })
+    }
+
     const onSampleRateChanged = async (val: SampleRate) => {
         setOptions({ ...options, sampleRate: val })
     }
@@ -105,13 +114,13 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
         setOptions({ ...options, bufferSize: val })
     }
     const onChunkSizeChanged = async (val: number) => {
-        setOptions({ ...options, chunkSize: val })
+        setOptions({ ...options, inputChunkNum: val })
     }
     const onSrcIdChanged = async (val: number) => {
-        setOptions({ ...options, srcId: val })
+        setParams({ ...params, srcId: val })
     }
     const onDstIdChanged = async (val: number) => {
-        setOptions({ ...options, dstId: val })
+        setParams({ ...params, dstId: val })
     }
     const onSetSpeakerMappingClicked = async () => {
         const targetId = editSpeakerTargetId
@@ -137,22 +146,22 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
     }
 
     const onVfEnabledChange = async (val: boolean) => {
-        setOptions({ ...options, vfEnabled: val })
+        setOptions({ ...options, forceVfDisable: val })
     }
     const onVoiceChangeModeChanged = async (val: VoiceChangerMode) => {
         setOptions({ ...options, voiceChangerMode: val })
     }
     const onGpuChanged = async (val: number) => {
-        setOptions({ ...options, gpu: val })
+        setParams({ ...params, gpu: val })
     }
     const onCrossFadeLowerValueChanged = async (val: number) => {
-        setOptions({ ...options, crossFadeLowerValue: val })
+        setParams({ ...params, crossFadeLowerValue: val })
     }
     const onCrossFadeOffsetRateChanged = async (val: number) => {
-        setOptions({ ...options, crossFadeOffsetRate: val })
+        setParams({ ...params, crossFadeOffsetRate: val })
     }
     const onCrossFadeEndRateChanged = async (val: number) => {
-        setOptions({ ...options, crossFadeEndRate: val })
+        setParams({ ...params, crossFadeEndRate: val })
     }
 
     const settings = useMemo(() => {
@@ -161,26 +170,44 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
                 <div className="body-row left-padding-1">
                     <div className="body-section-title">Virtual Microphone</div>
                 </div>
+                {startButtonRow}
+
+                <div className="body-row split-3-3-4 left-padding-1">
+                    <div className="body-item-title">MMVC Server</div>
+                    <div className="body-input-container">
+                        <input type="text" defaultValue={options.mmvcServerUrl} id="mmvc-server-url" className="body-item-input" />
+                    </div>
+                    <div className="body-button-container">
+                        <div className="body-button" onClick={onSetServerClicked}>set</div>
+                    </div>
+                </div>
+
                 <div className="body-row split-3-7 left-padding-1 highlight">
-                    <div className="body-item-title">Microphone</div>
+                    <div className="body-item-title">Protocol</div>
                     <div className="body-select-container">
-                        <select className="body-select" onChange={(e) => { setAudioInputDeviceId(e.target.value) }}>
+                        <select className="body-select" value={options.protocol} onChange={(e) => {
+                            onProtocolChanged(e.target.value as
+                                Protocol)
+                        }}>
                             {
-                                audioDeviceInfo.map(x => {
+                                Object.values(Protocol).map(x => {
-                                    return <option key={x.deviceId} value={x.deviceId}>{x.label}</option>
+                                    return <option key={x} value={x}>{x}</option>
                                 })
                             }
                         </select>
                     </div>
                 </div>
 
-                <div className="body-row split-3-3-4 left-padding-1">
+                <div className="body-row split-3-7 left-padding-1 highlight">
-                    <div className="body-item-title">MMVC Server</div>
+                    <div className="body-item-title">Microphone</div>
-                    <div className="body-input-container">
+                    <div className="body-select-container">
-                        <input type="text" defaultValue={options.mmvcServerUrl} id="mmvc-server-url" />
+                        <select className="body-select" value={options.audioInputDeviceId || "none"} onChange={(e) => { setAudioInputDeviceId(e.target.value) }}>
-                    </div>
+                            {
-                    <div className="body-button-container">
+                                audioDeviceInfo.map(x => {
-                        <div className="body-button" onClick={onSetServerClicked}>set</div>
+                                    return <option key={x.deviceId} value={x.deviceId}>{x.label}</option>
+                                })
+                            }
+                        </select>
                     </div>
                 </div>
 
@@ -211,16 +238,38 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
                 </div>
 
                 <div className="body-row split-3-7 left-padding-1 highlight">
-                    <div className="body-item-title">Chunk Size</div>
+                    <div className="body-item-title">Chunk Num(128sample/chunk)</div>
                     <div className="body-input-container">
-                        <input type="number" min={1} max={256} step={1} value={options.chunkSize} onChange={(e) => { onChunkSizeChanged(Number(e.target.value)) }} />
+                        <input type="number" min={1} max={256} step={1} value={options.inputChunkNum} onChange={(e) => { onChunkSizeChanged(Number(e.target.value)) }} />
+                    </div>
+                </div>
+
+                <div className="body-row split-3-3-4 left-padding-1 highlight">
+                    <div className="body-item-title">VF Enabled</div>
+                    <div>
+                        <input type="checkbox" checked={options.forceVfDisable} onChange={(e) => onVfEnabledChange(e.target.checked)} />
+                    </div>
+                    <div className="body-button-container">
+                    </div>
+                </div>
+
+                <div className="body-row split-3-7 left-padding-1 highlight">
+                    <div className="body-item-title">Voice Change Mode</div>
+                    <div className="body-select-container">
+                        <select className="body-select" value={options.voiceChangerMode} onChange={(e) => { onVoiceChangeModeChanged(e.target.value as VoiceChangerMode) }}>
+                            {
+                                Object.values(VoiceChangerMode).map(x => {
+                                    return <option key={x} value={x}>{x}</option>
+                                })
+                            }
+                        </select>
                     </div>
                 </div>
 
                 <div className="body-row split-3-7 left-padding-1 highlight">
                     <div className="body-item-title">Source Speaker Id</div>
                     <div className="body-select-container">
-                        <select className="body-select" value={options.srcId} onChange={(e) => { onSrcIdChanged(Number(e.target.value)) }}>
+                        <select className="body-select" value={params.srcId} onChange={(e) => { onSrcIdChanged(Number(e.target.value)) }}>
                             {
                                 options.speakers.map(x => {
                                     return <option key={x.id} value={x.id}>{x.name}({x.id})</option>
@@ -233,7 +282,7 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
                 <div className="body-row split-3-7 left-padding-1 highlight">
                     <div className="body-item-title">Destination Speaker Id</div>
                     <div className="body-select-container">
-                        <select className="body-select" value={options.dstId} onChange={(e) => { onDstIdChanged(Number(e.target.value)) }}>
+                        <select className="body-select" value={params.dstId} onChange={(e) => { onDstIdChanged(Number(e.target.value)) }}>
                             {
                                 options.speakers.map(x => {
                                     return <option key={x.id} value={x.id}>{x.name}({x.id})</option>
@@ -260,32 +309,10 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
                     </div>
                 </div>
 
-                <div className="body-row split-3-3-4 left-padding-1 highlight">
-                    <div className="body-item-title">VF Enabled</div>
-                    <div>
-                        <input type="checkbox" checked={options.vfEnabled} onChange={(e) => onVfEnabledChange(e.target.checked)} />
-                    </div>
-                    <div className="body-button-container">
-                    </div>
-                </div>
-
-                <div className="body-row split-3-7 left-padding-1 highlight">
-                    <div className="body-item-title">Voice Change Mode</div>
-                    <div className="body-select-container">
-                        <select className="body-select" value={options.voiceChangerMode} onChange={(e) => { onVoiceChangeModeChanged(e.target.value as VoiceChangerMode) }}>
-                            {
-                                Object.values(VoiceChangerMode).map(x => {
-                                    return <option key={x} value={x}>{x}</option>
-                                })
-                            }
-                        </select>
-                    </div>
-                </div>
-
                 <div className="body-row split-3-7 left-padding-1 highlight">
                     <div className="body-item-title">GPU</div>
                     <div className="body-input-container">
-                        <input type="number" min={-1} max={5} step={1} value={options.gpu} onChange={(e) => { onGpuChanged(Number(e.target.value)) }} />
+                        <input type="number" min={-1} max={5} step={1} value={params.gpu} onChange={(e) => { onGpuChanged(Number(e.target.value)) }} />
                     </div>
                 </div>
 
@@ -293,30 +320,32 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
                 <div className="body-row split-3-7 left-padding-1 highlight">
                     <div className="body-item-title">Cross Fade Lower Val</div>
                     <div className="body-input-container">
-                        <input type="number" min={0} max={1} step={0.1} value={options.crossFadeLowerValue} onChange={(e) => { onCrossFadeLowerValueChanged(Number(e.target.value)) }} />
+                        <input type="number" min={0} max={1} step={0.1} value={params.crossFadeLowerValue} onChange={(e) => { onCrossFadeLowerValueChanged(Number(e.target.value)) }} />
                     </div>
                 </div>
 
                 <div className="body-row split-3-7 left-padding-1 highlight">
                     <div className="body-item-title">Cross Fade Offset Rate</div>
                     <div className="body-input-container">
-                        <input type="number" min={0} max={1} step={0.1} value={options.crossFadeOffsetRate} onChange={(e) => { onCrossFadeOffsetRateChanged(Number(e.target.value)) }} />
+                        <input type="number" min={0} max={1} step={0.1} value={params.crossFadeOffsetRate} onChange={(e) => { onCrossFadeOffsetRateChanged(Number(e.target.value)) }} />
                     </div>
                 </div>
 
                 <div className="body-row split-3-7 left-padding-1 highlight">
                     <div className="body-item-title">Cross Fade End Rate</div>
                     <div className="body-input-container">
-                        <input type="number" min={0} max={1} step={0.1} value={options.crossFadeEndRate} onChange={(e) => { onCrossFadeEndRateChanged(Number(e.target.value)) }} />
+                        <input type="number" min={0} max={1} step={0.1} value={params.crossFadeEndRate} onChange={(e) => { onCrossFadeEndRateChanged(Number(e.target.value)) }} />
                     </div>
                 </div>
             </>
         )
-    }, [audioDeviceInfo, editSpeakerTargetId, editSpeakerTargetName, options])
+    }, [audioDeviceInfo, editSpeakerTargetId, editSpeakerTargetName, startButtonRow, params, options])
 
     return {
         component: settings,
-        options: options
+        params: params,
+        options: options,
+        isStarted
     }
 }
 

client/lib/src/AudioStreamer.ts

@@ -1,7 +1,7 @@
 import { io, Socket } from "socket.io-client";
 import { DefaultEventsMap } from "@socket.io/component-emitter";
 import { Duplex, DuplexOptions } from "readable-stream";
-import { DefaultVoiceChangerRequestParamas, MajarModeTypes, VoiceChangerMode, VoiceChangerRequestParamas } from "./const";
+import { DefaultVoiceChangerRequestParamas, Protocol, VoiceChangerMode, VoiceChangerRequestParamas, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
 
 export type Callbacks = {
     onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer) => void
@@ -9,17 +9,17 @@ export type Callbacks = {
 export type AudioStreamerListeners = {
     notifySendBufferingTime: (time: number) => void
     notifyResponseTime: (time: number) => void
-    notifyException: (message: string) => void
+    notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
 }
 export class AudioStreamer extends Duplex {
     private callbacks: Callbacks
     private audioStreamerListeners: AudioStreamerListeners
-    private majarMode: MajarModeTypes
+    private protocol: Protocol = "sio"
     private serverUrl = ""
     private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
     private voiceChangerMode: VoiceChangerMode = "realtime"
     private requestParamas: VoiceChangerRequestParamas = DefaultVoiceChangerRequestParamas
-    private chunkNum = 8
+    private inputChunkNum = 10
     private requestChunks: ArrayBuffer[] = []
     private recordChunks: ArrayBuffer[] = []
     private isRecording = false
@@ -27,9 +27,8 @@ export class AudioStreamer extends Duplex {
     // performance monitor
     private bufferStart = 0;
 
-    constructor(majarMode: MajarModeTypes, callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
+    constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
         super(options);
-        this.majarMode = majarMode
         this.callbacks = callbacks
         this.audioStreamerListeners = audioStreamerListeners
     }
@@ -38,17 +37,19 @@ export class AudioStreamer extends Duplex {
         if (this.socket) {
             this.socket.close()
         }
-        if (this.majarMode === "sio") {
+        if (this.protocol === "sio") {
             this.socket = io(this.serverUrl);
+            this.socket.on('connect_error', (err) => {
+                this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
+            })
             this.socket.on('connect', () => console.log(`[SIO] sonnect to ${this.serverUrl}`));
             this.socket.on('response', (response: any[]) => {
                 const cur = Date.now()
                 const responseTime = cur - response[0]
                 const result = response[1] as ArrayBuffer
                 if (result.byteLength < 128 * 2) {
-                    this.audioStreamerListeners.notifyException(`[SIO] recevied data is too short ${result.byteLength}`)
+                    this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
                 } else {
-                    this.audioStreamerListeners.notifyException(``)
                     this.callbacks.onVoiceReceived(this.voiceChangerMode, response[1])
                     this.audioStreamerListeners.notifyResponseTime(responseTime)
                 }
@@ -57,11 +58,13 @@ export class AudioStreamer extends Duplex {
     }
 
     // Option Change
-    setServerUrl = (serverUrl: string, mode: MajarModeTypes) => {
+    setServerUrl = (serverUrl: string, mode: Protocol, openTab: boolean = false) => {
         this.serverUrl = serverUrl
-        this.majarMode = mode
+        this.protocol = mode
-        window.open(serverUrl, '_blank')
+        if (openTab) {
-        console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.majarMode}`)
+            window.open(serverUrl, '_blank')
+        }
+        console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.protocol}`)
 
         this.createSocketIO()// mode check is done in the method.
     }
@@ -70,8 +73,8 @@ export class AudioStreamer extends Duplex {
         this.requestParamas = val
     }
 
-    setChunkNum = (num: number) => {
+    setInputChunkNum = (num: number) => {
-        this.chunkNum = num
+        this.inputChunkNum = num
     }
 
     setVoiceChangerMode = (val: VoiceChangerMode) => {
@@ -115,7 +118,7 @@ export class AudioStreamer extends Duplex {
         }
 
         //// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
-        if (this.requestChunks.length < this.chunkNum) {
+        if (this.requestChunks.length < this.inputChunkNum) {
             return
         }
 
@@ -131,7 +134,7 @@ export class AudioStreamer extends Duplex {
             return prev + cur.byteLength
         }, 0)
 
-        console.log("send buff length", newBuffer.length)
+        // console.log("send buff length", newBuffer.length)
 
         this.sendBuffer(newBuffer)
         this.requestChunks = []
@@ -189,14 +192,14 @@ export class AudioStreamer extends Duplex {
         }
         const timestamp = Date.now()
         // console.log("REQUEST_MESSAGE:", [this.gpu, this.srcId, this.dstId, timestamp, newBuffer.buffer])
-        console.log("SERVER_URL", this.serverUrl, this.majarMode)
+        // console.log("SERVER_URL", this.serverUrl, this.protocol)
         const convertChunkNum = this.voiceChangerMode === "realtime" ? this.requestParamas.convertChunkNum : 0
-        if (this.majarMode === "sio") {
+        if (this.protocol === "sio") {
             if (!this.socket) {
                 console.warn(`sio is not initialized`)
                 return
             }
-            console.log("emit!")
+            // console.log("emit!")
             this.socket.emit('request_message', [
                 this.requestParamas.gpu,
                 this.requestParamas.srcId,
@@ -221,9 +224,8 @@ export class AudioStreamer extends Duplex {
                 newBuffer.buffer)
 
             if (res.byteLength < 128 * 2) {
-                this.audioStreamerListeners.notifyException(`[REST] recevied data is too short ${res.byteLength}`)
+                this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
             } else {
-                this.audioStreamerListeners.notifyException(``)
                 this.callbacks.onVoiceReceived(this.voiceChangerMode, res)
                 this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
             }

client/lib/src/VoiceChangerClient.ts

@@ -1,9 +1,9 @@
-import { VoiceChangerWorkletNode } from "./VoiceChangerWorkletNode";
+import { VoiceChangerWorkletNode, VolumeListener } from "./VoiceChangerWorkletNode";
 // @ts-ignore
 import workerjs from "raw-loader!../worklet/dist/index.js";
 import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js";
 import { createDummyMediaStream } from "./util";
-import { BufferSize, MajarModeTypes, VoiceChangerMode, VoiceChangerRequestParamas } from "./const";
+import { BufferSize, DefaultVoiceChangerOptions, DefaultVoiceChangerRequestParamas, Protocol, VoiceChangerMode, VoiceChangerRequestParamas } from "./const";
 import MicrophoneStream from "microphone-stream";
 import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreamer";
 
@@ -29,10 +29,11 @@ export class VoiceChnagerClient {
     private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode
 
     private promiseForInitialize: Promise<void>
+    private _isVoiceChanging = false
 
     private callbacks: Callbacks = {
         onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer): void => {
-            console.log(voiceChangerMode, data)
+            // console.log(voiceChangerMode, data)
             if (voiceChangerMode === "realtime") {
                 this.vcNode.postReceivedVoice(data)
                 return
@@ -59,19 +60,21 @@ export class VoiceChnagerClient {
         }
     }
 
-    constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners) {
+    constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners, volumeListener: VolumeListener) {
         this.ctx = ctx
         this.vfEnable = vfEnable
         this.promiseForInitialize = new Promise<void>(async (resolve) => {
             const scriptUrl = URL.createObjectURL(new Blob([workerjs], { type: "text/javascript" }));
             await this.ctx.audioWorklet.addModule(scriptUrl)
 
-            this.vcNode = new VoiceChangerWorkletNode(this.ctx); // vc node 
+            this.vcNode = new VoiceChangerWorkletNode(this.ctx, volumeListener); // vc node 
             this.currentMediaStreamAudioDestinationNode = this.ctx.createMediaStreamDestination() // output node
             this.vcNode.connect(this.currentMediaStreamAudioDestinationNode) // vc node -> output node
             // (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる)
-            this.audioStreamer = new AudioStreamer("sio", this.callbacks, audioStreamerListeners, { objectMode: true, })
+            this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, })
-
+            this.audioStreamer.setRequestParams(DefaultVoiceChangerRequestParamas)
+            this.audioStreamer.setInputChunkNum(DefaultVoiceChangerOptions.inputChunkNum)
+            this.audioStreamer.setVoiceChangerMode(DefaultVoiceChangerOptions.voiceChangerMode)
 
             if (this.vfEnable) {
                 this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' })
@@ -113,12 +116,17 @@ export class VoiceChnagerClient {
         }
 
         // create mic stream
+        if (this.micStream) {
+            console.log("DESTROY!!!!!!!!!!!!!!!!!!!")
+            // this.micStream.stop()
+            this.micStream.destroy()
+            this.micStream = null
+        }
         this.micStream = new MicrophoneStream({
             objectMode: true,
             bufferSize: bufferSize,
             context: this.ctx
         })
-
         // connect nodes.
         if (this.currentDevice && forceVfDisable == false) {
             this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream) // input node
@@ -128,6 +136,7 @@ export class VoiceChnagerClient {
             voiceFocusNode.end.connect(this.outputNodeFromVF!)
             this.micStream.setStream(this.outputNodeFromVF!.stream) // vf node -> mic stream
         } else {
+            console.log("VF disabled")
             this.micStream.setStream(this.currentMediaStream) // input device -> mic stream
         }
         this.micStream.pipe(this.audioStreamer!) // mic stream -> audio streamer
@@ -138,22 +147,35 @@ export class VoiceChnagerClient {
         return this.currentMediaStreamAudioDestinationNode.stream
     }
 
-
+    start = () => {
-
+        if (!this.micStream) { return }
+        this.micStream.playRecording()
+        this._isVoiceChanging = true
+    }
+    stop = () => {
+        if (!this.micStream) { return }
+        this.micStream.pauseRecording()
+        this._isVoiceChanging = false
+    }
+    get isVoiceChanging(): boolean {
+        return this._isVoiceChanging
+    }
     // Audio Streamer Settingg
-    setServerUrl = (serverUrl: string, mode: MajarModeTypes) => {
+    setServerUrl = (serverUrl: string, mode: Protocol, openTab: boolean = false) => {
-        this.audioStreamer.setServerUrl(serverUrl, mode)
+        this.audioStreamer.setServerUrl(serverUrl, mode, openTab)
     }
 
     setRequestParams = (val: VoiceChangerRequestParamas) => {
         this.audioStreamer.setRequestParams(val)
     }
 
-    setChunkNum = (num: number) => {
+    setInputChunkNum = (num: number) => {
-        this.audioStreamer.setChunkNum(num)
+        this.audioStreamer.setInputChunkNum(num)
     }
 
     setVoiceChangerMode = (val: VoiceChangerMode) => {
         this.audioStreamer.setVoiceChangerMode(val)
     }
+
+
 }

client/lib/src/VoiceChangerWorkletNode.ts

@@ -1,7 +1,13 @@
+export type VolumeListener = {
+    notifyVolume: (vol: number) => void
+}
+
 export class VoiceChangerWorkletNode extends AudioWorkletNode {
-    constructor(context: AudioContext) {
+    private listener: VolumeListener
+    constructor(context: AudioContext, listener: VolumeListener) {
         super(context, "voice-changer-worklet-processor");
         this.port.onmessage = this.handleMessage.bind(this);
+        this.listener = listener
         console.log(`[worklet_node][voice-changer-worklet-processor] created.`);
     }
 
@@ -12,6 +18,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
     }
 
     handleMessage(event: any) {
-        console.log(`[Node:handleMessage_] `, event.data.volume);
+        // console.log(`[Node:handleMessage_] `, event.data.volume);
+        this.listener.notifyVolume(event.data.volume as number)
     }
 }

client/lib/src/const.ts

@@ -5,7 +5,7 @@
 
 // types
 export type VoiceChangerRequestParamas = {
-    convertChunkNum: number,
+    convertChunkNum: number, // VITSに入力する変換サイズ。(入力データの2倍以上の大きさで指定。それより小さいものが指定された場合は、サーバ側で自動的に入力の2倍のサイズが設定される。)
     srcId: number,
     dstId: number,
     gpu: number,
@@ -15,18 +15,14 @@ export type VoiceChangerRequestParamas = {
     crossFadeEndRate: number,
 }
 
-export type VoiceChangerRequest = VoiceChangerRequestParamas & {
-    data: ArrayBuffer,
-    timestamp: number
-}
-
 export type VoiceChangerOptions = {
     audioInputDeviceId: string | null,
     mediaStream: MediaStream | null,
     mmvcServerUrl: string,
+    protocol: Protocol,
     sampleRate: SampleRate, // 48000Hz
     bufferSize: BufferSize, // 256, 512, 1024, 2048, 4096, 8192, 16384 (for mic stream)
-    chunkNum: number, // n of (256 x n) for send buffer
+    inputChunkNum: number, // n of (256 x n) for send buffer
     speakers: Speaker[],
     forceVfDisable: boolean,
     voiceChangerMode: VoiceChangerMode,
@@ -40,11 +36,11 @@ export type Speaker = {
 
 // Consts
 
-export const MajarModeTypes = {
+export const Protocol = {
     "sio": "sio",
     "rest": "rest",
 } as const
-export type MajarModeTypes = typeof MajarModeTypes[keyof typeof MajarModeTypes]
+export type Protocol = typeof Protocol[keyof typeof Protocol]
 
 export const VoiceChangerMode = {
     "realtime": "realtime",
@@ -58,42 +54,69 @@ export const SampleRate = {
 export type SampleRate = typeof SampleRate[keyof typeof SampleRate]
 
 export const BufferSize = {
+    "256": 256,
+    "512": 512,
     "1024": 1024,
+    "2048": 2048,
+    "4096": 4096,
+    "8192": 8192,
+    "16384": 16384
 } as const
 export type BufferSize = typeof BufferSize[keyof typeof BufferSize]
 
 
 // Defaults
 export const DefaultVoiceChangerRequestParamas: VoiceChangerRequestParamas = {
-    convertChunkNum: 12, //（★１）
+    convertChunkNum: 1, //（★１）
     srcId: 107,
     dstId: 100,
     gpu: 0,
     crossFadeLowerValue: 0.1,
-    crossFadeOffsetRate: 0.3,
+    crossFadeOffsetRate: 0.1,
-    crossFadeEndRate: 0.6
+    crossFadeEndRate: 0.9
 }
 
-export const DefaultSpeakders: Speaker[] = [
+export const DefaultVoiceChangerOptions: VoiceChangerOptions = {
-    {
+    audioInputDeviceId: null,
-        "id": 100,
+    mediaStream: null,
-        "name": "ずんだもん"
+    mmvcServerUrl: "https://192.168.0.3:18888/test",
-    },
+    protocol: "sio",
-    {
+    sampleRate: 48000,
-        "id": 107,
+    bufferSize: 1024,
-        "name": "user"
+    inputChunkNum: 48,
-    },
+    speakers: [
-    {
+        {
-        "id": 101,
+            "id": 100,
-        "name": "そら"
+            "name": "ずんだもん"
-    },
+        },
-    {
+        {
-        "id": 102,
+            "id": 107,
-        "name": "めたん"
+            "name": "user"
-    },
+        },
-    {
+        {
-        "id": 103,
+            "id": 101,
-        "name": "つむぎ"
+            "name": "そら"
-    }
+        },
-]
+        {
+            "id": 102,
+            "name": "めたん"
+        },
+        {
+            "id": 103,
+            "name": "つむぎ"
+        }
+    ],
+    forceVfDisable: false,
+    voiceChangerMode: "realtime"
+}
+
+
+export const VOICE_CHANGER_CLIENT_EXCEPTION = {
+    ERR_SIO_CONNECT_FAILED: "ERR_SIO_CONNECT_FAILED",
+    ERR_SIO_INVALID_RESPONSE: "ERR_SIO_INVALID_RESPONSE",
+    ERR_REST_INVALID_RESPONSE: "ERR_REST_INVALID_RESPONSE"
+
+} as const
+export type VOICE_CHANGER_CLIENT_EXCEPTION = typeof VOICE_CHANGER_CLIENT_EXCEPTION[keyof typeof VOICE_CHANGER_CLIENT_EXCEPTION]
+
 

client/lib/worklet/src/voice-changer-worklet-processor.ts

@@ -19,7 +19,7 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
         // データは(int16)で受信
         const i16Data = new Int16Array(arrayBuffer)
         const f32Data = new Float32Array(i16Data.length)
-        console.log(`[worklet] f32DataLength${f32Data.length} i16DataLength${i16Data.length}`)
+        // console.log(`[worklet] f32DataLength${f32Data.length} i16DataLength${i16Data.length}`)
         i16Data.forEach((x, i) => {
             const float = (x >= 0x8000) ? -(0x10000 - x) / 0x8000 : x / 0x7FFF;
             f32Data[i] = float