diff --git a/client/demo/src/css/App.css b/client/demo/src/css/App.css index 42c7355c..fd5d32e9 100644 --- a/client/demo/src/css/App.css +++ b/client/demo/src/css/App.css @@ -157,6 +157,11 @@ body { .body-item-text { color: rgb(30, 30, 30); } + +.body-item-input { + width:90%; +} + .body-button-container { display: flex; flex-direction: row; @@ -165,6 +170,22 @@ body { border: solid 1px #333; border-radius: 2px; padding: 2px; + cursor:pointer; + } + .body-button-active { + user-select: none; + border: solid 1px #333; + border-radius: 2px; + padding: 2px; + background:#ada; + } + .body-button-stanby { + user-select: none; + border: solid 1px #333; + border-radius: 2px; + padding: 2px; + background:#aba; + cursor:pointer; } } .body-select-container { diff --git a/client/demo/src/index.tsx b/client/demo/src/index.tsx index f87ff66c..721c2940 100644 --- a/client/demo/src/index.tsx +++ b/client/demo/src/index.tsx @@ -1,18 +1,17 @@ import * as React from "react"; import { createRoot } from "react-dom/client"; import "./css/App.css" -import { useEffect, useMemo, useRef } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { VoiceChnagerClient } from "@dannadori/voice-changer-client-js" import { useMicrophoneOptions } from "./options_microphone"; const container = document.getElementById("app")!; const root = createRoot(container); const App = () => { - const { component: microphoneSettingComponent, options: microphonOptions } = useMicrophoneOptions() + const { component: microphoneSettingComponent, options: microphoneOptions, params: microphoneParams, isStarted } = useMicrophoneOptions() - const voiceChnagerClientRef = useRef(null) - - console.log(microphonOptions) + const voiceChangerClientRef = useRef(null) + const [clientInitialized, setClientInitialized] = useState(false) const onClearSettingClicked = async () => { //@ts-ignore @@ -23,32 +22,101 @@ const App = () => { location.reload() } + useEffect(() => { - if (microphonOptions.audioInputDeviceId.length == 0) { - return - } - const setAudio = async () => { + const initialized = async () => { const ctx = new AudioContext() - - if (voiceChnagerClientRef.current) { - - } - voiceChnagerClientRef.current = new VoiceChnagerClient(ctx, true, { + voiceChangerClientRef.current = new VoiceChnagerClient(ctx, true, { notifySendBufferingTime: (val: number) => { console.log(`buf:${val}`) }, notifyResponseTime: (val: number) => { console.log(`res:${val}`) }, - notifyException: (mes: string) => { console.log(`error:${mes}`) } - }) - await voiceChnagerClientRef.current.isInitialized() - - voiceChnagerClientRef.current.setServerUrl("https://192.168.0.3:18888/test", "sio") - voiceChnagerClientRef.current.setup(microphonOptions.audioInputDeviceId, 1024) + notifyException: (mes: string) => { + if (mes.length > 0) { + console.log(`error:${mes}`) + } + } + }, { notifyVolume: (vol: number) => { } }) + await voiceChangerClientRef.current.isInitialized() + setClientInitialized(true) const audio = document.getElementById("audio-output") as HTMLAudioElement - audio.srcObject = voiceChnagerClientRef.current.stream + audio.srcObject = voiceChangerClientRef.current.stream audio.play() } - setAudio() - }, [microphonOptions.audioInputDeviceId]) + initialized() + }, []) + + useEffect(() => { + console.log("START!!!", isStarted) + const start = async () => { + if (!voiceChangerClientRef.current || !clientInitialized) { + console.log("client is not initialized") + return + } + // if (!microphoneOptions.audioInputDeviceId || microphoneOptions.audioInputDeviceId.length == 0) { + // console.log("audioInputDeviceId is not initialized") + // return + // } + // await voiceChangerClientRef.current.setup(microphoneOptions.audioInputDeviceId!, microphoneOptions.bufferSize) + voiceChangerClientRef.current.setServerUrl(microphoneOptions.mmvcServerUrl, microphoneOptions.protocol, true) + voiceChangerClientRef.current.start() + } + const stop = async () => { + if (!voiceChangerClientRef.current || !clientInitialized) { + console.log("client is not initialized") + return + } + voiceChangerClientRef.current.stop() + } + if (isStarted) { + start() + } else { + stop() + } + }, [isStarted]) + + // useEffect(() => { + // if (!voiceChangerClientRef.current || !clientInitialized) { + // console.log("client is not initialized") + // return + // } + // voiceChangerClientRef.current.setServerUrl(microphoneOptions.mmvcServerUrl, microphoneOptions.protocol, false) + // }, [microphoneOptions.mmvcServerUrl, microphoneOptions.protocol]) + + useEffect(() => { + const changeInput = async () => { + if (!voiceChangerClientRef.current || !clientInitialized) { + console.log("client is not initialized") + return + } + await voiceChangerClientRef.current.setup(microphoneOptions.audioInputDeviceId!, microphoneOptions.bufferSize, microphoneOptions.forceVfDisable) + } + changeInput() + }, [microphoneOptions.audioInputDeviceId!, microphoneOptions.bufferSize, microphoneOptions.forceVfDisable]) + + + useEffect(() => { + if (!voiceChangerClientRef.current || !clientInitialized) { + console.log("client is not initialized") + return + } + voiceChangerClientRef.current.setInputChunkNum(microphoneOptions.inputChunkNum) + }, [microphoneOptions.inputChunkNum]) + + useEffect(() => { + if (!voiceChangerClientRef.current || !clientInitialized) { + console.log("client is not initialized") + return + } + voiceChangerClientRef.current.setVoiceChangerMode(microphoneOptions.voiceChangerMode) + }, [microphoneOptions.voiceChangerMode]) + + useEffect(() => { + if (!voiceChangerClientRef.current || !clientInitialized) { + console.log("client is not initialized") + return + } + voiceChangerClientRef.current.setRequestParams(microphoneParams) + }, [microphoneParams]) const clearRow = useMemo(() => { return ( diff --git a/client/demo/src/options_microphone.tsx b/client/demo/src/options_microphone.tsx index ca215970..b4026f43 100644 --- a/client/demo/src/options_microphone.tsx +++ b/client/demo/src/options_microphone.tsx @@ -1,42 +1,7 @@ import * as React from "react"; import { useEffect, useMemo, useState } from "react"; import { CHROME_EXTENSION } from "./const"; -import { Speaker, VoiceChangerMode, DefaultSpeakders, SampleRate, BufferSize } from "@dannadori/voice-changer-client-js" - -export type MicrophoneOptionsState = { - audioInputDeviceId: string, - mmvcServerUrl: string, - sampleRate: number, - bufferSize: number, - chunkSize: number, - speakers: Speaker[], - srcId: number, - dstId: number, - vfEnabled: boolean, - voiceChangerMode: VoiceChangerMode, - gpu: number, - - crossFadeLowerValue: number, - crossFadeOffsetRate: number, - crossFadeEndRate: number, -} -const InitMicrophoneOptionsState = { - audioInputDeviceId: "", - mmvcServerUrl: "https://localhost:5543/test", - sampleRate: 48000, - bufferSize: 1024, - chunkSize: 24, - speakers: DefaultSpeakders, - srcId: 107, - dstId: 100, - vfEnabled: true, - voiceChangerMode: VoiceChangerMode.realtime, - gpu: 0, - - crossFadeLowerValue: 0.1, - crossFadeOffsetRate: 0.3, - crossFadeEndRate: 0.6, -} as const +import { DefaultVoiceChangerRequestParamas, VoiceChangerOptions, VoiceChangerRequestParamas, DefaultVoiceChangerOptions, SampleRate, BufferSize, VoiceChangerMode, Protocol } from "@dannadori/voice-changer-client-js" const reloadDevices = async () => { @@ -46,13 +11,24 @@ const reloadDevices = async () => { console.warn("Enumerate device error::", e) } const mediaDeviceInfos = await navigator.mediaDevices.enumerateDevices(); - return mediaDeviceInfos.filter(x => { return x.kind == "audioinput" }) + + const audioInputs = mediaDeviceInfos.filter(x => { return x.kind == "audioinput" }) + audioInputs.push({ + deviceId: "none", + groupId: "none", + kind: "audioinput", + label: "none", + toJSON: () => { } + }) + return audioInputs } export type MicrophoneOptionsComponent = { component: JSX.Element, - options: MicrophoneOptionsState + options: VoiceChangerOptions, + params: VoiceChangerRequestParamas + isStarted: boolean } export const useMicrophoneOptions = (): MicrophoneOptionsComponent => { @@ -61,7 +37,10 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => { const [editSpeakerTargetId, setEditSpeakerTargetId] = useState(0) const [editSpeakerTargetName, setEditSpeakerTargetName] = useState("") - const [options, setOptions] = useState(InitMicrophoneOptionsState) + // const [options, setOptions] = useState(InitMicrophoneOptionsState) + const [params, setParams] = useState(DefaultVoiceChangerRequestParamas) + const [options, setOptions] = useState(DefaultVoiceChangerOptions) + const [isStarted, setIsStarted] = useState(false) useEffect(() => { const initialize = async () => { @@ -90,6 +69,32 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => { }, [options]) // loadより前に持ってくるとstorage内が初期化されるのでだめかも。（要検証） + const startButtonRow = useMemo(() => { + const onStartClicked = () => { + setIsStarted(true) + } + const onStopClicked = () => { + setIsStarted(false) + } + const startClassName = isStarted ? "body-button-active" : "body-button-stanby" + const stopClassName = isStarted ? "body-button-stanby" : "body-button-active" + console.log("ClassName", startClassName, stopClassName) + + return ( +
+
Start
+
+
start
+
stop
+
+
+
+
+ + ) + }, [isStarted]) + + const setAudioInputDeviceId = async (deviceId: string) => { setOptions({ ...options, audioInputDeviceId: deviceId }) } @@ -98,6 +103,10 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => { const input = document.getElementById("mmvc-server-url") as HTMLInputElement setOptions({ ...options, mmvcServerUrl: input.value }) } + const onProtocolChanged = async (val: Protocol) => { + setOptions({ ...options, protocol: val }) + } + const onSampleRateChanged = async (val: SampleRate) => { setOptions({ ...options, sampleRate: val }) } @@ -105,13 +114,13 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => { setOptions({ ...options, bufferSize: val }) } const onChunkSizeChanged = async (val: number) => { - setOptions({ ...options, chunkSize: val }) + setOptions({ ...options, inputChunkNum: val }) } const onSrcIdChanged = async (val: number) => { - setOptions({ ...options, srcId: val }) + setParams({ ...params, srcId: val }) } const onDstIdChanged = async (val: number) => { - setOptions({ ...options, dstId: val }) + setParams({ ...params, dstId: val }) } const onSetSpeakerMappingClicked = async () => { const targetId = editSpeakerTargetId @@ -137,22 +146,22 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => { } const onVfEnabledChange = async (val: boolean) => { - setOptions({ ...options, vfEnabled: val }) + setOptions({ ...options, forceVfDisable: val }) } const onVoiceChangeModeChanged = async (val: VoiceChangerMode) => { setOptions({ ...options, voiceChangerMode: val }) } const onGpuChanged = async (val: number) => { - setOptions({ ...options, gpu: val }) + setParams({ ...params, gpu: val }) } const onCrossFadeLowerValueChanged = async (val: number) => { - setOptions({ ...options, crossFadeLowerValue: val }) + setParams({ ...params, crossFadeLowerValue: val }) } const onCrossFadeOffsetRateChanged = async (val: number) => { - setOptions({ ...options, crossFadeOffsetRate: val }) + setParams({ ...params, crossFadeOffsetRate: val }) } const onCrossFadeEndRateChanged = async (val: number) => { - setOptions({ ...options, crossFadeEndRate: val }) + setParams({ ...params, crossFadeEndRate: val }) } const settings = useMemo(() => { @@ -161,26 +170,44 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
Virtual Microphone
+ {startButtonRow} + +
+
MMVC Server
+
+ +
+
+
set
+
+
+
-
Microphone
+
Protocol
- { + onProtocolChanged(e.target.value as + Protocol) + }}> { - audioDeviceInfo.map(x => { - return + Object.values(Protocol).map(x => { + return }) }
-
-
MMVC Server
-
- -
-
-
set
+
+
Microphone
+
+
@@ -211,16 +238,38 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
-
Chunk Size
+
Chunk Num(128sample/chunk)
- { onChunkSizeChanged(Number(e.target.value)) }} /> + { onChunkSizeChanged(Number(e.target.value)) }} /> +
+
+ +
+
VF Enabled
+
+ onVfEnabledChange(e.target.checked)} /> +
+
+
+
+ +
+
Voice Change Mode
+
+
Source Speaker Id
- { onSrcIdChanged(Number(e.target.value)) }}> { options.speakers.map(x => { return @@ -233,7 +282,7 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
Destination Speaker Id
- { onDstIdChanged(Number(e.target.value)) }}> { options.speakers.map(x => { return @@ -260,32 +309,10 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
-
-
VF Enabled
-
- onVfEnabledChange(e.target.checked)} /> -
-
-
-
- -
-
Voice Change Mode
-
- -
-
-
GPU
- { onGpuChanged(Number(e.target.value)) }} /> + { onGpuChanged(Number(e.target.value)) }} />
@@ -293,30 +320,32 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
Cross Fade Lower Val
- { onCrossFadeLowerValueChanged(Number(e.target.value)) }} /> + { onCrossFadeLowerValueChanged(Number(e.target.value)) }} />
Cross Fade Offset Rate
- { onCrossFadeOffsetRateChanged(Number(e.target.value)) }} /> + { onCrossFadeOffsetRateChanged(Number(e.target.value)) }} />
Cross Fade End Rate
- { onCrossFadeEndRateChanged(Number(e.target.value)) }} /> + { onCrossFadeEndRateChanged(Number(e.target.value)) }} />
) - }, [audioDeviceInfo, editSpeakerTargetId, editSpeakerTargetName, options]) + }, [audioDeviceInfo, editSpeakerTargetId, editSpeakerTargetName, startButtonRow, params, options]) return { component: settings, - options: options + params: params, + options: options, + isStarted } } diff --git a/client/lib/src/AudioStreamer.ts b/client/lib/src/AudioStreamer.ts index 3918d97a..ced063a5 100644 --- a/client/lib/src/AudioStreamer.ts +++ b/client/lib/src/AudioStreamer.ts @@ -1,7 +1,7 @@ import { io, Socket } from "socket.io-client"; import { DefaultEventsMap } from "@socket.io/component-emitter"; import { Duplex, DuplexOptions } from "readable-stream"; -import { DefaultVoiceChangerRequestParamas, MajarModeTypes, VoiceChangerMode, VoiceChangerRequestParamas } from "./const"; +import { DefaultVoiceChangerRequestParamas, Protocol, VoiceChangerMode, VoiceChangerRequestParamas, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const"; export type Callbacks = { onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer) => void @@ -9,17 +9,17 @@ export type Callbacks = { export type AudioStreamerListeners = { notifySendBufferingTime: (time: number) => void notifyResponseTime: (time: number) => void - notifyException: (message: string) => void + notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void } export class AudioStreamer extends Duplex { private callbacks: Callbacks private audioStreamerListeners: AudioStreamerListeners - private majarMode: MajarModeTypes + private protocol: Protocol = "sio" private serverUrl = "" private socket: Socket | null = null private voiceChangerMode: VoiceChangerMode = "realtime" private requestParamas: VoiceChangerRequestParamas = DefaultVoiceChangerRequestParamas - private chunkNum = 8 + private inputChunkNum = 10 private requestChunks: ArrayBuffer[] = [] private recordChunks: ArrayBuffer[] = [] private isRecording = false @@ -27,9 +27,8 @@ export class AudioStreamer extends Duplex { // performance monitor private bufferStart = 0; - constructor(majarMode: MajarModeTypes, callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) { + constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) { super(options); - this.majarMode = majarMode this.callbacks = callbacks this.audioStreamerListeners = audioStreamerListeners } @@ -38,17 +37,19 @@ export class AudioStreamer extends Duplex { if (this.socket) { this.socket.close() } - if (this.majarMode === "sio") { + if (this.protocol === "sio") { this.socket = io(this.serverUrl); + this.socket.on('connect_error', (err) => { + this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`) + }) this.socket.on('connect', () => console.log(`[SIO] sonnect to ${this.serverUrl}`)); this.socket.on('response', (response: any[]) => { const cur = Date.now() const responseTime = cur - response[0] const result = response[1] as ArrayBuffer if (result.byteLength < 128 * 2) { - this.audioStreamerListeners.notifyException(`[SIO] recevied data is too short ${result.byteLength}`) + this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`) } else { - this.audioStreamerListeners.notifyException(``) this.callbacks.onVoiceReceived(this.voiceChangerMode, response[1]) this.audioStreamerListeners.notifyResponseTime(responseTime) } @@ -57,11 +58,13 @@ export class AudioStreamer extends Duplex { } // Option Change - setServerUrl = (serverUrl: string, mode: MajarModeTypes) => { + setServerUrl = (serverUrl: string, mode: Protocol, openTab: boolean = false) => { this.serverUrl = serverUrl - this.majarMode = mode - window.open(serverUrl, '_blank') - console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.majarMode}`) + this.protocol = mode + if (openTab) { + window.open(serverUrl, '_blank') + } + console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.protocol}`) this.createSocketIO()// mode check is done in the method. } @@ -70,8 +73,8 @@ export class AudioStreamer extends Duplex { this.requestParamas = val } - setChunkNum = (num: number) => { - this.chunkNum = num + setInputChunkNum = (num: number) => { + this.inputChunkNum = num } setVoiceChangerMode = (val: VoiceChangerMode) => { @@ -115,7 +118,7 @@ export class AudioStreamer extends Duplex { } //// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。 - if (this.requestChunks.length < this.chunkNum) { + if (this.requestChunks.length < this.inputChunkNum) { return } @@ -131,7 +134,7 @@ export class AudioStreamer extends Duplex { return prev + cur.byteLength }, 0) - console.log("send buff length", newBuffer.length) + // console.log("send buff length", newBuffer.length) this.sendBuffer(newBuffer) this.requestChunks = [] @@ -189,14 +192,14 @@ export class AudioStreamer extends Duplex { } const timestamp = Date.now() // console.log("REQUEST_MESSAGE:", [this.gpu, this.srcId, this.dstId, timestamp, newBuffer.buffer]) - console.log("SERVER_URL", this.serverUrl, this.majarMode) + // console.log("SERVER_URL", this.serverUrl, this.protocol) const convertChunkNum = this.voiceChangerMode === "realtime" ? this.requestParamas.convertChunkNum : 0 - if (this.majarMode === "sio") { + if (this.protocol === "sio") { if (!this.socket) { console.warn(`sio is not initialized`) return } - console.log("emit!") + // console.log("emit!") this.socket.emit('request_message', [ this.requestParamas.gpu, this.requestParamas.srcId, @@ -221,9 +224,8 @@ export class AudioStreamer extends Duplex { newBuffer.buffer) if (res.byteLength < 128 * 2) { - this.audioStreamerListeners.notifyException(`[REST] recevied data is too short ${res.byteLength}`) + this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`) } else { - this.audioStreamerListeners.notifyException(``) this.callbacks.onVoiceReceived(this.voiceChangerMode, res) this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp) } diff --git a/client/lib/src/VoiceChangerClient.ts b/client/lib/src/VoiceChangerClient.ts index 89df19b0..3b18199b 100644 --- a/client/lib/src/VoiceChangerClient.ts +++ b/client/lib/src/VoiceChangerClient.ts @@ -1,9 +1,9 @@ -import { VoiceChangerWorkletNode } from "./VoiceChangerWorkletNode"; +import { VoiceChangerWorkletNode, VolumeListener } from "./VoiceChangerWorkletNode"; // @ts-ignore import workerjs from "raw-loader!../worklet/dist/index.js"; import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js"; import { createDummyMediaStream } from "./util"; -import { BufferSize, MajarModeTypes, VoiceChangerMode, VoiceChangerRequestParamas } from "./const"; +import { BufferSize, DefaultVoiceChangerOptions, DefaultVoiceChangerRequestParamas, Protocol, VoiceChangerMode, VoiceChangerRequestParamas } from "./const"; import MicrophoneStream from "microphone-stream"; import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreamer"; @@ -29,10 +29,11 @@ export class VoiceChnagerClient { private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode private promiseForInitialize: Promise + private _isVoiceChanging = false private callbacks: Callbacks = { onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer): void => { - console.log(voiceChangerMode, data) + // console.log(voiceChangerMode, data) if (voiceChangerMode === "realtime") { this.vcNode.postReceivedVoice(data) return @@ -59,19 +60,21 @@ export class VoiceChnagerClient { } } - constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners) { + constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners, volumeListener: VolumeListener) { this.ctx = ctx this.vfEnable = vfEnable this.promiseForInitialize = new Promise(async (resolve) => { const scriptUrl = URL.createObjectURL(new Blob([workerjs], { type: "text/javascript" })); await this.ctx.audioWorklet.addModule(scriptUrl) - this.vcNode = new VoiceChangerWorkletNode(this.ctx); // vc node + this.vcNode = new VoiceChangerWorkletNode(this.ctx, volumeListener); // vc node this.currentMediaStreamAudioDestinationNode = this.ctx.createMediaStreamDestination() // output node this.vcNode.connect(this.currentMediaStreamAudioDestinationNode) // vc node -> output node // (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる) - this.audioStreamer = new AudioStreamer("sio", this.callbacks, audioStreamerListeners, { objectMode: true, }) - + this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, }) + this.audioStreamer.setRequestParams(DefaultVoiceChangerRequestParamas) + this.audioStreamer.setInputChunkNum(DefaultVoiceChangerOptions.inputChunkNum) + this.audioStreamer.setVoiceChangerMode(DefaultVoiceChangerOptions.voiceChangerMode) if (this.vfEnable) { this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' }) @@ -113,12 +116,17 @@ export class VoiceChnagerClient { } // create mic stream + if (this.micStream) { + console.log("DESTROY!!!!!!!!!!!!!!!!!!!") + // this.micStream.stop() + this.micStream.destroy() + this.micStream = null + } this.micStream = new MicrophoneStream({ objectMode: true, bufferSize: bufferSize, context: this.ctx }) - // connect nodes. if (this.currentDevice && forceVfDisable == false) { this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream) // input node @@ -128,6 +136,7 @@ export class VoiceChnagerClient { voiceFocusNode.end.connect(this.outputNodeFromVF!) this.micStream.setStream(this.outputNodeFromVF!.stream) // vf node -> mic stream } else { + console.log("VF disabled") this.micStream.setStream(this.currentMediaStream) // input device -> mic stream } this.micStream.pipe(this.audioStreamer!) // mic stream -> audio streamer @@ -138,22 +147,35 @@ export class VoiceChnagerClient { return this.currentMediaStreamAudioDestinationNode.stream } - - + start = () => { + if (!this.micStream) { return } + this.micStream.playRecording() + this._isVoiceChanging = true + } + stop = () => { + if (!this.micStream) { return } + this.micStream.pauseRecording() + this._isVoiceChanging = false + } + get isVoiceChanging(): boolean { + return this._isVoiceChanging + } // Audio Streamer Settingg - setServerUrl = (serverUrl: string, mode: MajarModeTypes) => { - this.audioStreamer.setServerUrl(serverUrl, mode) + setServerUrl = (serverUrl: string, mode: Protocol, openTab: boolean = false) => { + this.audioStreamer.setServerUrl(serverUrl, mode, openTab) } setRequestParams = (val: VoiceChangerRequestParamas) => { this.audioStreamer.setRequestParams(val) } - setChunkNum = (num: number) => { - this.audioStreamer.setChunkNum(num) + setInputChunkNum = (num: number) => { + this.audioStreamer.setInputChunkNum(num) } setVoiceChangerMode = (val: VoiceChangerMode) => { this.audioStreamer.setVoiceChangerMode(val) } + + } \ No newline at end of file diff --git a/client/lib/src/VoiceChangerWorkletNode.ts b/client/lib/src/VoiceChangerWorkletNode.ts index af16f88c..6f40e33d 100644 --- a/client/lib/src/VoiceChangerWorkletNode.ts +++ b/client/lib/src/VoiceChangerWorkletNode.ts @@ -1,7 +1,13 @@ +export type VolumeListener = { + notifyVolume: (vol: number) => void +} + export class VoiceChangerWorkletNode extends AudioWorkletNode { - constructor(context: AudioContext) { + private listener: VolumeListener + constructor(context: AudioContext, listener: VolumeListener) { super(context, "voice-changer-worklet-processor"); this.port.onmessage = this.handleMessage.bind(this); + this.listener = listener console.log(`[worklet_node][voice-changer-worklet-processor] created.`); } @@ -12,6 +18,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode { } handleMessage(event: any) { - console.log(`[Node:handleMessage_] `, event.data.volume); + // console.log(`[Node:handleMessage_] `, event.data.volume); + this.listener.notifyVolume(event.data.volume as number) } } \ No newline at end of file diff --git a/client/lib/src/const.ts b/client/lib/src/const.ts index 01f5f50a..35445da8 100644 --- a/client/lib/src/const.ts +++ b/client/lib/src/const.ts @@ -5,7 +5,7 @@ // types export type VoiceChangerRequestParamas = { - convertChunkNum: number, + convertChunkNum: number, // VITSに入力する変換サイズ。(入力データの2倍以上の大きさで指定。それより小さいものが指定された場合は、サーバ側で自動的に入力の2倍のサイズが設定される。) srcId: number, dstId: number, gpu: number, @@ -15,18 +15,14 @@ export type VoiceChangerRequestParamas = { crossFadeEndRate: number, } -export type VoiceChangerRequest = VoiceChangerRequestParamas & { - data: ArrayBuffer, - timestamp: number -} - export type VoiceChangerOptions = { audioInputDeviceId: string | null, mediaStream: MediaStream | null, mmvcServerUrl: string, + protocol: Protocol, sampleRate: SampleRate, // 48000Hz bufferSize: BufferSize, // 256, 512, 1024, 2048, 4096, 8192, 16384 (for mic stream) - chunkNum: number, // n of (256 x n) for send buffer + inputChunkNum: number, // n of (256 x n) for send buffer speakers: Speaker[], forceVfDisable: boolean, voiceChangerMode: VoiceChangerMode, @@ -40,11 +36,11 @@ export type Speaker = { // Consts -export const MajarModeTypes = { +export const Protocol = { "sio": "sio", "rest": "rest", } as const -export type MajarModeTypes = typeof MajarModeTypes[keyof typeof MajarModeTypes] +export type Protocol = typeof Protocol[keyof typeof Protocol] export const VoiceChangerMode = { "realtime": "realtime", @@ -58,42 +54,69 @@ export const SampleRate = { export type SampleRate = typeof SampleRate[keyof typeof SampleRate] export const BufferSize = { + "256": 256, + "512": 512, "1024": 1024, + "2048": 2048, + "4096": 4096, + "8192": 8192, + "16384": 16384 } as const export type BufferSize = typeof BufferSize[keyof typeof BufferSize] // Defaults export const DefaultVoiceChangerRequestParamas: VoiceChangerRequestParamas = { - convertChunkNum: 12, //（★１） + convertChunkNum: 1, //（★１） srcId: 107, dstId: 100, gpu: 0, crossFadeLowerValue: 0.1, - crossFadeOffsetRate: 0.3, - crossFadeEndRate: 0.6 + crossFadeOffsetRate: 0.1, + crossFadeEndRate: 0.9 } -export const DefaultSpeakders: Speaker[] = [ - { - "id": 100, - "name": "ずんだもん" - }, - { - "id": 107, - "name": "user" - }, - { - "id": 101, - "name": "そら" - }, - { - "id": 102, - "name": "めたん" - }, - { - "id": 103, - "name": "つむぎ" - } -] +export const DefaultVoiceChangerOptions: VoiceChangerOptions = { + audioInputDeviceId: null, + mediaStream: null, + mmvcServerUrl: "https://192.168.0.3:18888/test", + protocol: "sio", + sampleRate: 48000, + bufferSize: 1024, + inputChunkNum: 48, + speakers: [ + { + "id": 100, + "name": "ずんだもん" + }, + { + "id": 107, + "name": "user" + }, + { + "id": 101, + "name": "そら" + }, + { + "id": 102, + "name": "めたん" + }, + { + "id": 103, + "name": "つむぎ" + } + ], + forceVfDisable: false, + voiceChangerMode: "realtime" +} + + +export const VOICE_CHANGER_CLIENT_EXCEPTION = { + ERR_SIO_CONNECT_FAILED: "ERR_SIO_CONNECT_FAILED", + ERR_SIO_INVALID_RESPONSE: "ERR_SIO_INVALID_RESPONSE", + ERR_REST_INVALID_RESPONSE: "ERR_REST_INVALID_RESPONSE" + +} as const +export type VOICE_CHANGER_CLIENT_EXCEPTION = typeof VOICE_CHANGER_CLIENT_EXCEPTION[keyof typeof VOICE_CHANGER_CLIENT_EXCEPTION] + diff --git a/client/lib/worklet/src/voice-changer-worklet-processor.ts b/client/lib/worklet/src/voice-changer-worklet-processor.ts index 6eadcaf8..c7d660d7 100644 --- a/client/lib/worklet/src/voice-changer-worklet-processor.ts +++ b/client/lib/worklet/src/voice-changer-worklet-processor.ts @@ -19,7 +19,7 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor { // データは(int16)で受信 const i16Data = new Int16Array(arrayBuffer) const f32Data = new Float32Array(i16Data.length) - console.log(`[worklet] f32DataLength${f32Data.length} i16DataLength${i16Data.length}`) + // console.log(`[worklet] f32DataLength${f32Data.length} i16DataLength${i16Data.length}`) i16Data.forEach((x, i) => { const float = (x >= 0x8000) ? -(0x10000 - x) / 0x8000 : x / 0x7FFF; f32Data[i] = float