From de0be188ff796093e2383fd3477dd861194c6709 Mon Sep 17 00:00:00 2001 From: wataru Date: Mon, 20 Feb 2023 01:00:28 +0900 Subject: [PATCH] EXP. remove microphone stream --- client/lib/src/VoiceChangerClient.ts | 21 +++- client/lib/src/VoiceChangerWorkletNode.ts | 106 ++++++++++++++++++ .../src/voice-changer-worklet-processor.ts | 17 ++- 3 files changed, 139 insertions(+), 5 deletions(-) diff --git a/client/lib/src/VoiceChangerClient.ts b/client/lib/src/VoiceChangerClient.ts index 5b8c04dd..6a632df1 100644 --- a/client/lib/src/VoiceChangerClient.ts +++ b/client/lib/src/VoiceChangerClient.ts @@ -164,11 +164,24 @@ export class VoiceChangerClient { const voiceFocusNode = await this.currentDevice.createAudioNode(this.ctx); // vf node this.inputGainNode.connect(voiceFocusNode.start) // input node -> vf node voiceFocusNode.end.connect(this.outputNodeFromVF!) - this.micStream.setStream(this.outputNodeFromVF!.stream) // vf node -> mic stream + // this.micStream.setStream(this.outputNodeFromVF!.stream) // vf node -> mic stream } else { - const inputDestinationNodeForMicStream = this.ctx.createMediaStreamDestination() - this.inputGainNode.connect(inputDestinationNodeForMicStream) - this.micStream.setStream(inputDestinationNodeForMicStream.stream) // input device -> mic stream + // const inputDestinationNodeForMicStream = this.ctx.createMediaStreamDestination() + // this.inputGainNode.connect(inputDestinationNodeForMicStream) + console.log("input___ media stream", this.currentMediaStream) + this.currentMediaStream.getTracks().forEach(x => { + console.log("input___ media stream set", x.getSettings()) + console.log("input___ media stream con", x.getConstraints()) + console.log("input___ media stream cap", x.getCapabilities()) + }) + console.log("input___ media node", this.currentMediaStreamAudioSourceNode) + console.log("input___ gain node", this.inputGainNode.channelCount, this.inputGainNode) + this.inputGainNode.connect(this.vcNode) + + + + + // this.micStream.setStream(inputDestinationNodeForMicStream.stream) // input device -> mic stream } this.micStream.pipe(this.audioStreamer) // mic stream -> audio streamer if (!this._isVoiceChanging) { diff --git a/client/lib/src/VoiceChangerWorkletNode.ts b/client/lib/src/VoiceChangerWorkletNode.ts index 52e16245..25828341 100644 --- a/client/lib/src/VoiceChangerWorkletNode.ts +++ b/client/lib/src/VoiceChangerWorkletNode.ts @@ -1,5 +1,7 @@ import { VoiceChangerWorkletProcessorRequest } from "./@types/voice-changer-worklet-processor"; import { WorkletSetting } from "./const"; +import { io, Socket } from "socket.io-client"; +import { DefaultEventsMap } from "@socket.io/component-emitter"; export type VoiceChangerWorkletListener = { notifyVolume: (vol: number) => void @@ -8,10 +10,13 @@ export type VoiceChangerWorkletListener = { export class VoiceChangerWorkletNode extends AudioWorkletNode { private listener: VoiceChangerWorkletListener + private requestChunks: ArrayBuffer[] = [] + private socket: Socket | null = null constructor(context: AudioContext, listener: VoiceChangerWorkletListener) { super(context, "voice-changer-worklet-processor"); this.port.onmessage = this.handleMessage.bind(this); this.listener = listener + this.createSocketIO() console.log(`[worklet_node][voice-changer-worklet-processor] created.`); } @@ -26,17 +31,118 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode { this.port.postMessage(req) } + + private createSocketIO = () => { + if (this.socket) { + this.socket.close() + } + // if (this.setting.protocol === "sio") { + // this.socket = io(this.setting.serverUrl + "/test"); + this.socket = io("/test"); + this.socket.on('connect_error', (err) => { + console.log("connect exception !!!!!") + // this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`) + }) + this.socket.on('connect', () => { + // console.log(`[SIO] sonnect to ${this.setting.serverUrl}`) + console.log(`[SIO] ${this.socket?.id}`) + }); + this.socket.on('response', (response: any[]) => { + const cur = Date.now() + const responseTime = cur - response[0] + const result = response[1] as ArrayBuffer + if (result.byteLength < 128 * 2) { + console.log("tooshort!!") + // this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`) + } else { + console.log("response!!!") + this.postReceivedVoice(response[1]) + // this.callbacks.onVoiceReceived(response[1]) + // this.audioStreamerListeners.notifyResponseTime(responseTime) + } + }); + // } + } + handleMessage(event: any) { // console.log(`[Node:handleMessage_] `, event.data.volume); if (event.data.responseType === "volume") { this.listener.notifyVolume(event.data.volume as number) } else if (event.data.responseType === "recordData") { this.listener.notifyOutputRecordData(event.data.recordData as Float32Array[]) + } else if (event.data.responseType === "inputData") { + const inputData = event.data.inputData as Float32Array + // console.log("receive input data", inputData) + + const arrayBuffer = new ArrayBuffer(inputData.length * 2) + const dataView = new DataView(arrayBuffer); + for (let i = 0; i < inputData.length; i++) { + let s = Math.max(-1, Math.min(1, inputData[i])); + s = s < 0 ? s * 0x8000 : s * 0x7FFF + dataView.setInt16(i * 2, s, true); + } + + this.requestChunks.push(arrayBuffer) + + //// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。 + if (this.requestChunks.length < 32) { + return + } + console.log("sending...") + + // リクエスト用の入れ物を作成 + const windowByteLength = this.requestChunks.reduce((prev, cur) => { + return prev + cur.byteLength + }, 0) + const newBuffer = new Uint8Array(windowByteLength); + + // リクエストのデータをセット + this.requestChunks.reduce((prev, cur) => { + newBuffer.set(new Uint8Array(cur), prev) + return prev + cur.byteLength + }, 0) + + + this.sendBuffer(newBuffer) + console.log("sended...") + this.requestChunks = [] + + + } else { console.warn(`[worklet_node][voice-changer-worklet-processor] unknown response ${event.data.responseType}`, event.data) } } + + + private sendBuffer = async (newBuffer: Uint8Array) => { + const timestamp = Date.now() + // if (this.setting.protocol === "sio") { + if (!this.socket) { + console.warn(`sio is not initialized`) + return + } + // console.log("emit!") + this.socket.emit('request_message', [ + timestamp, + newBuffer.buffer]); + // } else { + // const res = await postVoice( + // this.setting.serverUrl + "/test", + // timestamp, + // newBuffer.buffer) + + // if (res.byteLength < 128 * 2) { + // this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`) + // } else { + // this.callbacks.onVoiceReceived(res) + // this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp) + // } + // } + } + + configure = (setting: WorkletSetting) => { const req: VoiceChangerWorkletProcessorRequest = { requestType: "config", diff --git a/client/lib/worklet/src/voice-changer-worklet-processor.ts b/client/lib/worklet/src/voice-changer-worklet-processor.ts index 5c216582..8ca32e42 100644 --- a/client/lib/worklet/src/voice-changer-worklet-processor.ts +++ b/client/lib/worklet/src/voice-changer-worklet-processor.ts @@ -9,7 +9,8 @@ export type RequestType = typeof RequestType[keyof typeof RequestType] export const ResponseType = { "volume": "volume", - "recordData": "recordData" + "recordData": "recordData", + "inputData": "inputData" } as const export type ResponseType = typeof ResponseType[keyof typeof ResponseType] @@ -27,6 +28,7 @@ export type VoiceChangerWorkletProcessorResponse = { responseType: ResponseType, volume?: number, recordData?: Float32Array[] + inputData?: Float32Array } class VoiceChangerWorkletProcessor extends AudioWorkletProcessor { @@ -132,12 +134,24 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor { } + pushData = (inputData: Float32Array) => { + const volumeResponse: VoiceChangerWorkletProcessorResponse = { + responseType: ResponseType.inputData, + inputData: inputData + } + this.port.postMessage(volumeResponse); + } + process(_inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record) { if (!this.initialized) { console.warn("[worklet] worklet_process not ready"); return true; } + if (_inputs.length > 0 && _inputs[0].length > 0) { + this.pushData(_inputs[0][0]) + } + if (this.playBuffer.length === 0) { // console.log("[worklet] no play buffer") return true @@ -174,6 +188,7 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor { } this.port.postMessage(volumeResponse); outputs[0][0].set(voice) + outputs[0][1].set(voice) } return true;