diff --git a/client/demo/src/components/demo/components2/102-3_DeviceArea.tsx b/client/demo/src/components/demo/components2/102-3_DeviceArea.tsx index 327c1ccc..113072a1 100644 --- a/client/demo/src/components/demo/components2/102-3_DeviceArea.tsx +++ b/client/demo/src/components/demo/components2/102-3_DeviceArea.tsx @@ -389,8 +389,12 @@ export const DeviceArea = (_props: DeviceAreaProps) => { // Server Audio を使う場合はElementから音は出さない。 audio.volume = 0; } else if (audioOutputForGUI == "none") { - // @ts-ignore - audio.setSinkId(""); + try { + // @ts-ignore + audio.setSinkId(""); + } catch (e) { + console.error("catch:" + e); + } if (x == AUDIO_ELEMENT_FOR_TEST_CONVERTED_ECHOBACK) { audio.volume = 0; } else { @@ -404,8 +408,12 @@ export const DeviceArea = (_props: DeviceAreaProps) => { return x.deviceId == audioOutputForGUI; }); if (found) { - // @ts-ignore // 例外キャッチできないので事前にIDチェックが必要らしい。！？ - audio.setSinkId(audioOutputForGUI); + try { + // @ts-ignore // 例外キャッチできないので事前にIDチェックが必要らしい。！？ + audio.setSinkId(audioOutputForGUI); + } catch (e) { + console.error("catch:" + e); + } } else { console.warn("No audio output device. use default"); } @@ -620,9 +628,13 @@ export const DeviceArea = (_props: DeviceAreaProps) => { // Server Audio を使う場合はElementから音は出さない。 audio.volume = 0; } else if (audioMonitorForGUI == "none") { - // @ts-ignore - audio.setSinkId(""); - audio.volume = 0; + try { + // @ts-ignore + audio.setSinkId(""); + audio.volume = 0; + } catch (e) { + console.error("catch:" + e); + } } else { const audioOutputs = mediaDeviceInfos.filter((x) => { return x.kind == "audiooutput"; @@ -631,9 +643,13 @@ export const DeviceArea = (_props: DeviceAreaProps) => { return x.deviceId == audioMonitorForGUI; }); if (found) { - // @ts-ignore // 例外キャッチできないので事前にIDチェックが必要らしい。！？ - audio.setSinkId(audioMonitorForGUI); - audio.volume = 1; + try { + // @ts-ignore // 例外キャッチできないので事前にIDチェックが必要らしい。！？ + audio.setSinkId(audioMonitorForGUI); + audio.volume = 1; + } catch (e) { + console.error("catch:" + e); + } } else { console.warn("No audio output device. use default"); } diff --git a/client/demo/src/components/demo/components2/102-4_RecorderArea.tsx b/client/demo/src/components/demo/components2/102-4_RecorderArea.tsx index c6812555..b02d88d1 100644 --- a/client/demo/src/components/demo/components2/102-4_RecorderArea.tsx +++ b/client/demo/src/components/demo/components2/102-4_RecorderArea.tsx @@ -1,44 +1,50 @@ -import React, { useMemo, useState } from "react" -import { useAppState } from "../../../001_provider/001_AppStateProvider" -import { useGuiState } from "../001_GuiStateProvider" -import { AUDIO_ELEMENT_FOR_SAMPLING_INPUT, AUDIO_ELEMENT_FOR_SAMPLING_OUTPUT } from "../../../const" +import React, { useMemo, useState } from "react"; +import { useAppState } from "../../../001_provider/001_AppStateProvider"; +import { useGuiState } from "../001_GuiStateProvider"; +import { AUDIO_ELEMENT_FOR_SAMPLING_INPUT, AUDIO_ELEMENT_FOR_SAMPLING_OUTPUT } from "../../../const"; -export type RecorderAreaProps = { -} +export type RecorderAreaProps = {}; export const RecorderArea = (_props: RecorderAreaProps) => { - const { serverSetting } = useAppState() - const { audioOutputForAnalyzer, setAudioOutputForAnalyzer, outputAudioDeviceInfo } = useGuiState() - - const [serverIORecording, setServerIORecording] = useState(false) + const { serverSetting } = useAppState(); + const { audioOutputForAnalyzer, setAudioOutputForAnalyzer, outputAudioDeviceInfo } = useGuiState(); + const [serverIORecording, setServerIORecording] = useState(false); const serverIORecorderRow = useMemo(() => { const onServerIORecordStartClicked = async () => { - setServerIORecording(true) - await serverSetting.updateServerSettings({ ...serverSetting.serverSetting, recordIO: 1 }) - } + setServerIORecording(true); + await serverSetting.updateServerSettings({ ...serverSetting.serverSetting, recordIO: 1 }); + }; const onServerIORecordStopClicked = async () => { - setServerIORecording(false) - await serverSetting.updateServerSettings({ ...serverSetting.serverSetting, recordIO: 0 }) + setServerIORecording(false); + await serverSetting.updateServerSettings({ ...serverSetting.serverSetting, recordIO: 0 }); // set wav (input) - const wavInput = document.getElementById(AUDIO_ELEMENT_FOR_SAMPLING_INPUT) as HTMLAudioElement - wavInput.src = "/tmp/in.wav?" + new Date().getTime() - wavInput.controls = true - // @ts-ignore - wavInput.setSinkId(audioOutputForAnalyzer) + const wavInput = document.getElementById(AUDIO_ELEMENT_FOR_SAMPLING_INPUT) as HTMLAudioElement; + wavInput.src = "/tmp/in.wav?" + new Date().getTime(); + wavInput.controls = true; + try { + // @ts-ignore + wavInput.setSinkId(audioOutputForAnalyzer); + } catch (e) { + console.log(e); + } // set wav (output) - const wavOutput = document.getElementById(AUDIO_ELEMENT_FOR_SAMPLING_OUTPUT) as HTMLAudioElement - wavOutput.src = "/tmp/out.wav?" + new Date().getTime() - wavOutput.controls = true - // @ts-ignore - wavOutput.setSinkId(audioOutputForAnalyzer) - } + const wavOutput = document.getElementById(AUDIO_ELEMENT_FOR_SAMPLING_OUTPUT) as HTMLAudioElement; + wavOutput.src = "/tmp/out.wav?" + new Date().getTime(); + wavOutput.controls = true; + try { + // @ts-ignore + wavOutput.setSinkId(audioOutputForAnalyzer); + } catch (e) { + console.log(e); + } + }; - const startClassName = serverIORecording ? "config-sub-area-button-active" : "config-sub-area-button" - const stopClassName = serverIORecording ? "config-sub-area-button" : "config-sub-area-button-active" + const startClassName = serverIORecording ? "config-sub-area-button-active" : "config-sub-area-button"; + const stopClassName = serverIORecording ? "config-sub-area-button" : "config-sub-area-button-active"; return ( <>
@@ -49,34 +55,51 @@ export const RecorderArea = (_props: RecorderAreaProps) => {
SIO rec.
-
start
-
stop
+
+ start +
+
+ stop +
-
output
- { + setAudioOutputForAnalyzer(e.target.value); + const wavInput = document.getElementById(AUDIO_ELEMENT_FOR_SAMPLING_INPUT) as HTMLAudioElement; + const wavOutput = document.getElementById(AUDIO_ELEMENT_FOR_SAMPLING_OUTPUT) as HTMLAudioElement; + try { + //@ts-ignore + wavInput.setSinkId(e.target.value); + //@ts-ignore + wavOutput.setSinkId(e.target.value); + } catch (e) { + console.log(e); + } + }} + > + {outputAudioDeviceInfo + .map((x) => { if (x.deviceId == "none") { - return null + return null; } - return - }).filter(x => { return x != null }) - } + return ( + + ); + }) + .filter((x) => { + return x != null; + })}
@@ -102,17 +125,9 @@ export const RecorderArea = (_props: RecorderAreaProps) => {
- - ) - - }, [serverIORecording, audioOutputForAnalyzer, outputAudioDeviceInfo, serverSetting.updateServerSettings]) - - return ( -
- {serverIORecorderRow} -
- ) -} - + ); + }, [serverIORecording, audioOutputForAnalyzer, outputAudioDeviceInfo, serverSetting.updateServerSettings]); + return
{serverIORecorderRow}
; +}; diff --git a/client/lib/worklet/src/voice-changer-worklet-processor.ts b/client/lib/worklet/src/voice-changer-worklet-processor.ts index 23138adc..4a199db3 100644 --- a/client/lib/worklet/src/voice-changer-worklet-processor.ts +++ b/client/lib/worklet/src/voice-changer-worklet-processor.ts @@ -1,191 +1,207 @@ export const RequestType = { - voice: "voice", - config: "config", - start: "start", - stop: "stop", - trancateBuffer: "trancateBuffer", + voice: "voice", + config: "config", + start: "start", + stop: "stop", + trancateBuffer: "trancateBuffer", } as const; export type RequestType = (typeof RequestType)[keyof typeof RequestType]; export const ResponseType = { - volume: "volume", - inputData: "inputData", - start_ok: "start_ok", - stop_ok: "stop_ok", + volume: "volume", + inputData: "inputData", + start_ok: "start_ok", + stop_ok: "stop_ok", } as const; export type ResponseType = (typeof ResponseType)[keyof typeof ResponseType]; export type VoiceChangerWorkletProcessorRequest = { - requestType: RequestType; - voice: Float32Array; - numTrancateTreshold: number; - volTrancateThreshold: number; - volTrancateLength: number; + requestType: RequestType; + voice: Float32Array; + numTrancateTreshold: number; + volTrancateThreshold: number; + volTrancateLength: number; }; export type VoiceChangerWorkletProcessorResponse = { - responseType: ResponseType; - volume?: number; - recordData?: Float32Array[]; - inputData?: Float32Array; + responseType: ResponseType; + volume?: number; + recordData?: Float32Array[]; + inputData?: Float32Array; }; class VoiceChangerWorkletProcessor extends AudioWorkletProcessor { - private BLOCK_SIZE = 128; - private initialized = false; - private volume = 0; - // private numTrancateTreshold = 100; - // private volTrancateThreshold = 0.0005 - // private volTrancateLength = 32 - // private volTrancateCount = 0 + private BLOCK_SIZE = 128; + private initialized = false; + private volume = 0; + // private numTrancateTreshold = 100; + // private volTrancateThreshold = 0.0005 + // private volTrancateLength = 32 + // private volTrancateCount = 0 - private isRecording = false; + private isRecording = false; - playBuffer: Float32Array[] = []; - unpushedF32Data: Float32Array = new Float32Array(0); - /** - * @constructor - */ - constructor() { - super(); - this.initialized = true; - this.port.onmessage = this.handleMessage.bind(this); + playBuffer: Float32Array[] = []; + unpushedF32Data: Float32Array = new Float32Array(0); + /** + * @constructor + */ + constructor() { + super(); + console.log("[AudioWorkletProcessor] created."); + this.initialized = true; + this.port.onmessage = this.handleMessage.bind(this); + } + + calcVol = (data: Float32Array, prevVol: number) => { + const sum = data.reduce((prev, cur) => { + return prev + cur * cur; + }, 0); + const rms = Math.sqrt(sum / data.length); + return Math.max(rms, prevVol * 0.95); + }; + + trancateBuffer = () => { + console.log("[worklet] Buffer truncated"); + while (this.playBuffer.length > 2) { + this.playBuffer.shift(); + } + }; + handleMessage(event: any) { + const request = event.data as VoiceChangerWorkletProcessorRequest; + if (request.requestType === "config") { + // this.numTrancateTreshold = request.numTrancateTreshold; + // this.volTrancateLength = request.volTrancateLength + // this.volTrancateThreshold = request.volTrancateThreshold + console.log("[worklet] worklet configured", request); + return; + } else if (request.requestType === "start") { + if (this.isRecording) { + console.warn("[worklet] recoring is already started"); + return; + } + this.isRecording = true; + const startResponse: VoiceChangerWorkletProcessorResponse = { + responseType: "start_ok", + }; + this.port.postMessage(startResponse); + return; + } else if (request.requestType === "stop") { + if (!this.isRecording) { + console.warn("[worklet] recoring is not started"); + return; + } + this.isRecording = false; + const stopResponse: VoiceChangerWorkletProcessorResponse = { + responseType: "stop_ok", + }; + this.port.postMessage(stopResponse); + return; + } else if (request.requestType === "trancateBuffer") { + this.trancateBuffer(); + return; } - calcVol = (data: Float32Array, prevVol: number) => { - const sum = data.reduce((prev, cur) => { - return prev + cur * cur; - }, 0); - const rms = Math.sqrt(sum / data.length); - return Math.max(rms, prevVol * 0.95); - }; - - trancateBuffer = () => { - console.log("[worklet] Buffer truncated"); - while (this.playBuffer.length > 2) { - this.playBuffer.shift(); - } - }; - handleMessage(event: any) { - const request = event.data as VoiceChangerWorkletProcessorRequest; - if (request.requestType === "config") { - // this.numTrancateTreshold = request.numTrancateTreshold; - // this.volTrancateLength = request.volTrancateLength - // this.volTrancateThreshold = request.volTrancateThreshold - console.log("[worklet] worklet configured", request); - return; - } else if (request.requestType === "start") { - if (this.isRecording) { - console.warn("[worklet] recoring is already started"); - return; - } - this.isRecording = true; - const startResponse: VoiceChangerWorkletProcessorResponse = { - responseType: "start_ok", - }; - this.port.postMessage(startResponse); - return; - } else if (request.requestType === "stop") { - if (!this.isRecording) { - console.warn("[worklet] recoring is not started"); - return; - } - this.isRecording = false; - const stopResponse: VoiceChangerWorkletProcessorResponse = { - responseType: "stop_ok", - }; - this.port.postMessage(stopResponse); - return; - } else if (request.requestType === "trancateBuffer") { - this.trancateBuffer(); - return; - } - - const f32Data = request.voice; - // if (this.playBuffer.length > this.numTrancateTreshold) { - // console.log(`[worklet] Truncate ${this.playBuffer.length} > ${this.numTrancateTreshold}`); - // this.trancateBuffer(); - // } - if (this.playBuffer.length > f32Data.length / this.BLOCK_SIZE) { - console.log(`[worklet] Truncate ${this.playBuffer.length} > ${f32Data.length / this.BLOCK_SIZE}`); - this.trancateBuffer(); - } - - const concatedF32Data = new Float32Array(this.unpushedF32Data.length + f32Data.length); - concatedF32Data.set(this.unpushedF32Data); - concatedF32Data.set(f32Data, this.unpushedF32Data.length); - - const chunkNum = Math.floor(concatedF32Data.length / this.BLOCK_SIZE); - for (let i = 0; i < chunkNum; i++) { - const block = concatedF32Data.slice(i * this.BLOCK_SIZE, (i + 1) * this.BLOCK_SIZE); - this.playBuffer.push(block); - } - this.unpushedF32Data = concatedF32Data.slice(chunkNum * this.BLOCK_SIZE); + const f32Data = request.voice; + // if (this.playBuffer.length > this.numTrancateTreshold) { + // console.log(`[worklet] Truncate ${this.playBuffer.length} > ${this.numTrancateTreshold}`); + // this.trancateBuffer(); + // } + if (this.playBuffer.length > (f32Data.length / this.BLOCK_SIZE) * 1.5) { + console.log( + `[worklet] Truncate ${this.playBuffer.length} > ${ + f32Data.length / this.BLOCK_SIZE + }` + ); + this.trancateBuffer(); } - pushData = (inputData: Float32Array) => { - const volumeResponse: VoiceChangerWorkletProcessorResponse = { - responseType: ResponseType.inputData, - inputData: inputData, - }; - this.port.postMessage(volumeResponse); - }; + const concatedF32Data = new Float32Array( + this.unpushedF32Data.length + f32Data.length + ); + concatedF32Data.set(this.unpushedF32Data); + concatedF32Data.set(f32Data, this.unpushedF32Data.length); - process(_inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record) { - if (!this.initialized) { - console.warn("[worklet] worklet_process not ready"); - return true; - } - - if (this.isRecording) { - if (_inputs.length > 0 && _inputs[0].length > 0) { - this.pushData(_inputs[0][0]); - } - } - - if (this.playBuffer.length === 0) { - // console.log("[worklet] no play buffer"); - return true; - } - // console.log("[worklet] play buffer"); - //// 一定期間無音状態が続いている場合はスキップ。 - // let voice: Float32Array | undefined - // while (true) { - // voice = this.playBuffer.shift() - // if (!voice) { - // break - // } - // this.volume = this.calcVol(voice, this.volume) - // if (this.volume < this.volTrancateThreshold) { - // this.volTrancateCount += 1 - // } else { - // this.volTrancateCount = 0 - // } - - // // V.1.5.0よりsilent skipで音飛びするようになったので無効化 - // if (this.volTrancateCount < this.volTrancateLength || this.volTrancateLength < 0) { - // break - // } else { - // break - // // console.log("silent...skip") - // } - // } - let voice = this.playBuffer.shift(); - - if (voice) { - this.volume = this.calcVol(voice, this.volume); - const volumeResponse: VoiceChangerWorkletProcessorResponse = { - responseType: ResponseType.volume, - volume: this.volume, - }; - this.port.postMessage(volumeResponse); - outputs[0][0].set(voice); - if (outputs[0].length == 2) { - outputs[0][1].set(voice); - } - } - - return true; + const chunkNum = Math.floor(concatedF32Data.length / this.BLOCK_SIZE); + for (let i = 0; i < chunkNum; i++) { + const block = concatedF32Data.slice( + i * this.BLOCK_SIZE, + (i + 1) * this.BLOCK_SIZE + ); + this.playBuffer.push(block); } + this.unpushedF32Data = concatedF32Data.slice(chunkNum * this.BLOCK_SIZE); + } + + pushData = (inputData: Float32Array) => { + const volumeResponse: VoiceChangerWorkletProcessorResponse = { + responseType: ResponseType.inputData, + inputData: inputData, + }; + this.port.postMessage(volumeResponse); + }; + + process( + _inputs: Float32Array[][], + outputs: Float32Array[][], + _parameters: Record + ) { + if (!this.initialized) { + console.warn("[worklet] worklet_process not ready"); + return true; + } + + if (this.isRecording) { + if (_inputs.length > 0 && _inputs[0].length > 0) { + this.pushData(_inputs[0][0]); + } + } + + if (this.playBuffer.length === 0) { + // console.log("[worklet] no play buffer"); + return true; + } + // console.log("[worklet] play buffer"); + //// 一定期間無音状態が続いている場合はスキップ。 + // let voice: Float32Array | undefined + // while (true) { + // voice = this.playBuffer.shift() + // if (!voice) { + // break + // } + // this.volume = this.calcVol(voice, this.volume) + // if (this.volume < this.volTrancateThreshold) { + // this.volTrancateCount += 1 + // } else { + // this.volTrancateCount = 0 + // } + + // // V.1.5.0よりsilent skipで音飛びするようになったので無効化 + // if (this.volTrancateCount < this.volTrancateLength || this.volTrancateLength < 0) { + // break + // } else { + // break + // // console.log("silent...skip") + // } + // } + let voice = this.playBuffer.shift(); + if (voice) { + this.volume = this.calcVol(voice, this.volume); + const volumeResponse: VoiceChangerWorkletProcessorResponse = { + responseType: ResponseType.volume, + volume: this.volume, + }; + this.port.postMessage(volumeResponse); + outputs[0][0].set(voice); + if (outputs[0].length == 2) { + outputs[0][1].set(voice); + } + } + + return true; + } } -registerProcessor("voice-changer-worklet-processor", VoiceChangerWorkletProcessor); +registerProcessor( + "voice-changer-worklet-processor", + VoiceChangerWorkletProcessor +);