mirror of
https://github.com/w-okada/voice-changer.git
synced 2025-01-23 13:35:12 +03:00
WIP: refactoring...
This commit is contained in:
wataru
parent
e1d2660a59
commit
12e717d1b7
58
client/demo/dist/index.js
vendored
58
client/demo/dist/index.js
vendored
File diff suppressed because one or more lines are too long
@ -1,4 +1,4 @@
|
||||
import { OnnxExecutionProvider, Framework, fileSelector } from "@dannadori/voice-changer-client-js"
|
||||
import { OnnxExecutionProvider, Framework, fileSelector, Correspondence } from "@dannadori/voice-changer-client-js"
|
||||
import React, { useState } from "react"
|
||||
import { useMemo } from "react"
|
||||
import { useAppState } from "./001_provider/001_AppStateProvider";
|
||||
@ -84,10 +84,27 @@ export const useModelSettingArea = (): ServerSettingState => {
|
||||
}
|
||||
const onCorrespondenceFileLoadClicked = async () => {
|
||||
const file = await fileSelector("")
|
||||
appState.clientSetting.setCorrespondences(file)
|
||||
|
||||
const correspondenceText = await file.text()
|
||||
const cors = correspondenceText.split("\n").map(line => {
|
||||
const items = line.split("|")
|
||||
if (items.length != 3) {
|
||||
console.warn("Invalid Correspondence Line:", line)
|
||||
return null
|
||||
} else {
|
||||
const cor: Correspondence = {
|
||||
sid: Number(items[0]),
|
||||
correspondence: Number(items[1]),
|
||||
dirname: items[2]
|
||||
}
|
||||
return cor
|
||||
}
|
||||
}).filter(x => { return x != null }) as Correspondence[]
|
||||
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, correspondences: cors })
|
||||
|
||||
}
|
||||
const onCorrespondenceFileClearClicked = () => {
|
||||
appState.clientSetting.setCorrespondences(null)
|
||||
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, correspondences: [] })
|
||||
}
|
||||
|
||||
const onModelUploadClicked = async () => {
|
||||
@ -101,7 +118,7 @@ export const useModelSettingArea = (): ServerSettingState => {
|
||||
const configFilenameText = appState.serverSetting.fileUploadSetting.configFile?.filename || appState.serverSetting.fileUploadSetting.configFile?.file?.name || ""
|
||||
const onnxModelFilenameText = appState.serverSetting.fileUploadSetting.onnxModel?.filename || appState.serverSetting.fileUploadSetting.onnxModel?.file?.name || ""
|
||||
const pyTorchFilenameText = appState.serverSetting.fileUploadSetting.pyTorchModel?.filename || appState.serverSetting.fileUploadSetting.pyTorchModel?.file?.name || ""
|
||||
const correspondenceFileText = appState.clientSetting.setting.correspondences ? JSON.stringify(appState.clientSetting.setting.correspondences.map(x => { return x.dirname })) : ""
|
||||
const correspondenceFileText = appState.serverSetting.serverSetting.correspondences ? JSON.stringify(appState.serverSetting.serverSetting.correspondences.map(x => { return x.dirname })) : ""
|
||||
|
||||
return (
|
||||
<>
|
||||
@ -185,7 +202,9 @@ export const useModelSettingArea = (): ServerSettingState => {
|
||||
appState.serverSetting.loadModel,
|
||||
appState.serverSetting.isUploading,
|
||||
appState.serverSetting.uploadProgress,
|
||||
appState.clientSetting.setting.correspondences,
|
||||
appState.serverSetting.serverSetting.correspondences,
|
||||
appState.serverSetting.updateServerSettings,
|
||||
appState.serverSetting.setFileUploadSetting,
|
||||
showPyTorch])
|
||||
|
||||
const frameworkRow = useMemo(() => {
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { fileSelectorAsDataURL, ServerAudioDevice, useIndexedDB } from "@dannadori/voice-changer-client-js"
|
||||
import { fileSelectorAsDataURL, useIndexedDB } from "@dannadori/voice-changer-client-js"
|
||||
import React, { useEffect, useMemo, useRef, useState } from "react"
|
||||
import { AUDIO_ELEMENT_FOR_PLAY_RESULT, AUDIO_ELEMENT_FOR_TEST_CONVERTED, AUDIO_ELEMENT_FOR_TEST_CONVERTED_ECHOBACK, AUDIO_ELEMENT_FOR_TEST_ORIGINAL, INDEXEDDB_KEY_AUDIO_OUTPUT } from "./const"
|
||||
import { useAppState } from "./001_provider/001_AppStateProvider";
|
||||
@ -60,7 +60,6 @@ export const useDeviceSetting = (): DeviceSettingState => {
|
||||
|
||||
const [inputAudioDeviceInfo, setInputAudioDeviceInfo] = useState<MediaDeviceInfo[]>([])
|
||||
const [outputAudioDeviceInfo, setOutputAudioDeviceInfo] = useState<MediaDeviceInfo[]>([])
|
||||
const [serverInputAudioDeviceInfo, setServerInputAudioDeviceInfo] = useState<ServerAudioDevice[]>([])
|
||||
|
||||
const [audioInputForGUI, setAudioInputForGUI] = useState<string>("none")
|
||||
const [audioOutputForGUI, setAudioOutputForGUI] = useState<string>("none")
|
||||
@ -79,30 +78,30 @@ export const useDeviceSetting = (): DeviceSettingState => {
|
||||
const audioInfo = await reloadDevices()
|
||||
setInputAudioDeviceInfo(audioInfo[0])
|
||||
setOutputAudioDeviceInfo(audioInfo[1])
|
||||
if (useServerMicrophone) {
|
||||
try {
|
||||
const serverDevices = await appState.serverSetting.getServerDevices()
|
||||
setServerInputAudioDeviceInfo(serverDevices.audio_input_devices)
|
||||
} catch (e) {
|
||||
console.warn(e)
|
||||
}
|
||||
}
|
||||
// if (useServerMicrophone) {
|
||||
// try {
|
||||
// const serverDevices = await appState.serverSetting.getServerDevices()
|
||||
// setServerInputAudioDeviceInfo(serverDevices.audio_input_devices)
|
||||
// } catch (e) {
|
||||
// console.warn(e)
|
||||
// }
|
||||
// }
|
||||
}
|
||||
initialize()
|
||||
}, [useServerMicrophone])
|
||||
|
||||
// キャッシュの設定は反映（たぶん、設定操作の時も起動していしまう。が問題は起こらないはず）
|
||||
useEffect(() => {
|
||||
if (typeof appState.clientSetting.setting.audioInput == "string") {
|
||||
if (typeof appState.clientSetting.clientSetting.audioInput == "string") {
|
||||
if (inputAudioDeviceInfo.find(x => {
|
||||
// console.log("COMPARE:", x.deviceId, appState.clientSetting.setting.audioInput)
|
||||
return x.deviceId == appState.clientSetting.setting.audioInput
|
||||
return x.deviceId == appState.clientSetting.clientSetting.audioInput
|
||||
})) {
|
||||
setAudioInputForGUI(appState.clientSetting.setting.audioInput)
|
||||
setAudioInputForGUI(appState.clientSetting.clientSetting.audioInput)
|
||||
|
||||
}
|
||||
}
|
||||
}, [inputAudioDeviceInfo, appState.clientSetting.setting.audioInput])
|
||||
}, [inputAudioDeviceInfo, appState.clientSetting.clientSetting.audioInput])
|
||||
|
||||
const audioInputRow = useMemo(() => {
|
||||
if (useServerMicrophone) {
|
||||
@ -126,40 +125,18 @@ export const useDeviceSetting = (): DeviceSettingState => {
|
||||
)
|
||||
}, [inputAudioDeviceInfo, audioInputForGUI, useServerMicrophone])
|
||||
|
||||
const audioInputServerRow = useMemo(() => {
|
||||
if (!useServerMicrophone) {
|
||||
return <></>
|
||||
}
|
||||
return (
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">AudioInput(Server)</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={audioInputForGUI} onChange={(e) => {
|
||||
setAudioInputForGUI(e.target.value)
|
||||
}}>
|
||||
{
|
||||
serverInputAudioDeviceInfo.map(x => {
|
||||
return <option key={x.name} value={x.index}>{x.name}</option>
|
||||
})
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [serverInputAudioDeviceInfo, audioInputForGUI, useServerMicrophone])
|
||||
|
||||
useEffect(() => {
|
||||
if (audioInputForGUI == "file") {
|
||||
// file selector (audioMediaInputRow)
|
||||
} else {
|
||||
if (!useServerMicrophone) {
|
||||
appState.clientSetting.setAudioInput(audioInputForGUI)
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, audioInput: audioInputForGUI })
|
||||
} else {
|
||||
console.log("server mic")
|
||||
appState.clientSetting.setAudioInput(null)
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, audioInput: null })
|
||||
}
|
||||
}
|
||||
}, [appState.audioContext, audioInputForGUI, appState.clientSetting.setAudioInput])
|
||||
}, [appState.audioContext, audioInputForGUI, appState.clientSetting.updateClientSetting])
|
||||
|
||||
const audioMediaInputRow = useMemo(() => {
|
||||
if (audioInputForGUI != "file") {
|
||||
@ -184,7 +161,7 @@ export const useDeviceSetting = (): DeviceSettingState => {
|
||||
|
||||
const dst = appState.audioContext.createMediaStreamDestination()
|
||||
audioSrcNode.current.connect(dst)
|
||||
appState.clientSetting.setAudioInput(dst.stream)
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, audioInput: dst.stream })
|
||||
|
||||
const audio_echo = document.getElementById(AUDIO_ELEMENT_FOR_TEST_CONVERTED_ECHOBACK) as HTMLAudioElement
|
||||
audio_echo.srcObject = dst.stream
|
||||
@ -222,7 +199,7 @@ export const useDeviceSetting = (): DeviceSettingState => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [audioInputForGUI, appState.clientSetting.setAudioInput, fileInputEchoback])
|
||||
}, [audioInputForGUI, appState.clientSetting.updateClientSetting, fileInputEchoback])
|
||||
|
||||
|
||||
|
||||
@ -345,7 +322,6 @@ export const useDeviceSetting = (): DeviceSettingState => {
|
||||
|
||||
<div className="partition-content">
|
||||
{audioInputRow}
|
||||
{audioInputServerRow}
|
||||
{audioMediaInputRow}
|
||||
{audioOutputRow}
|
||||
{audioOutputRecordingRow}
|
||||
@ -353,7 +329,7 @@ export const useDeviceSetting = (): DeviceSettingState => {
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}, [audioInputRow, audioInputServerRow, audioMediaInputRow, audioOutputRow, audioOutputRecordingRow, useServerMicrophone])
|
||||
}, [audioInputRow, audioMediaInputRow, audioOutputRow, audioOutputRecordingRow, useServerMicrophone])
|
||||
|
||||
|
||||
// 出力の録音データ(from worklet)がストアされたら実行
|
||||
|
||||
@ -53,18 +53,18 @@ export const useQualityControl = (): QualityControlState => {
|
||||
<div className="body-row split-3-2-2-2-1 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1 ">Noise Suppression</div>
|
||||
<div>
|
||||
<input type="checkbox" checked={appState.clientSetting.setting.echoCancel} onChange={(e) => {
|
||||
appState.clientSetting.setEchoCancel(e.target.checked)
|
||||
<input type="checkbox" checked={appState.clientSetting.clientSetting.echoCancel} onChange={(e) => {
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, echoCancel: e.target.checked })
|
||||
}} /> echo cancel
|
||||
</div>
|
||||
<div>
|
||||
<input type="checkbox" checked={appState.clientSetting.setting.noiseSuppression} onChange={(e) => {
|
||||
appState.clientSetting.setNoiseSuppression(e.target.checked)
|
||||
<input type="checkbox" checked={appState.clientSetting.clientSetting.noiseSuppression} onChange={(e) => {
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, noiseSuppression: e.target.checked })
|
||||
}} /> suppression1
|
||||
</div>
|
||||
<div>
|
||||
<input type="checkbox" checked={appState.clientSetting.setting.noiseSuppression2} onChange={(e) => {
|
||||
appState.clientSetting.setNoiseSuppression2(e.target.checked)
|
||||
<input type="checkbox" checked={appState.clientSetting.clientSetting.noiseSuppression2} onChange={(e) => {
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, noiseSuppression2: e.target.checked })
|
||||
}} /> suppression2
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
@ -72,9 +72,10 @@ export const useQualityControl = (): QualityControlState => {
|
||||
</div>
|
||||
)
|
||||
}, [
|
||||
appState.clientSetting.setting.echoCancel, appState.clientSetting.setEchoCancel,
|
||||
appState.clientSetting.setting.noiseSuppression, appState.clientSetting.setNoiseSuppression,
|
||||
appState.clientSetting.setting.noiseSuppression2, appState.clientSetting.setNoiseSuppression2,
|
||||
appState.clientSetting.clientSetting.echoCancel,
|
||||
appState.clientSetting.clientSetting.noiseSuppression,
|
||||
appState.clientSetting.clientSetting.noiseSuppression2,
|
||||
appState.clientSetting.updateClientSetting
|
||||
])
|
||||
|
||||
const gainControlRow = useMemo(() => {
|
||||
@ -83,25 +84,26 @@ export const useQualityControl = (): QualityControlState => {
|
||||
<div className="body-item-title left-padding-1 ">Gain Control</div>
|
||||
<div>
|
||||
<span className="body-item-input-slider-label">in</span>
|
||||
<input type="range" className="body-item-input-slider" min="0.0" max="1.0" step="0.1" value={appState.clientSetting.setting.inputGain} onChange={(e) => {
|
||||
appState.clientSetting.setInputGain(Number(e.target.value))
|
||||
<input type="range" className="body-item-input-slider" min="0.0" max="1.0" step="0.1" value={appState.clientSetting.clientSetting.inputGain} onChange={(e) => {
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, inputGain: Number(e.target.value) })
|
||||
}}></input>
|
||||
<span className="body-item-input-slider-val">{appState.clientSetting.setting.inputGain}</span>
|
||||
<span className="body-item-input-slider-val">{appState.clientSetting.clientSetting.inputGain}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span className="body-item-input-slider-label">out</span>
|
||||
<input type="range" className="body-item-input-slider" min="0.0" max="1.0" step="0.1" value={appState.clientSetting.setting.outputGain} onChange={(e) => {
|
||||
appState.clientSetting.setOutputGain(Number(e.target.value))
|
||||
<input type="range" className="body-item-input-slider" min="0.0" max="1.0" step="0.1" value={appState.clientSetting.clientSetting.outputGain} onChange={(e) => {
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, outputGain: Number(e.target.value) })
|
||||
}}></input>
|
||||
<span className="body-item-input-slider-val">{appState.clientSetting.setting.outputGain}</span>
|
||||
<span className="body-item-input-slider-val">{appState.clientSetting.clientSetting.outputGain}</span>
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [
|
||||
appState.clientSetting.setting.inputGain, appState.clientSetting.setting.inputGain,
|
||||
appState.clientSetting.setting.outputGain, appState.clientSetting.setOutputGain,
|
||||
appState.clientSetting.clientSetting.inputGain,
|
||||
appState.clientSetting.clientSetting.outputGain,
|
||||
appState.clientSetting.updateClientSetting
|
||||
])
|
||||
|
||||
const f0DetectorRow = useMemo(() => {
|
||||
|
||||
@ -33,10 +33,10 @@ export const useSpeakerSetting = () => {
|
||||
|
||||
|
||||
const calcDefaultF0Factor = (srcId: number, dstId: number) => {
|
||||
const src = appState.clientSetting.setting.correspondences?.find(x => {
|
||||
const src = appState.serverSetting.serverSetting.correspondences?.find(x => {
|
||||
return x.sid == srcId
|
||||
})
|
||||
const dst = appState.clientSetting.setting.correspondences?.find(x => {
|
||||
const dst = appState.serverSetting.serverSetting.correspondences?.find(x => {
|
||||
return x.sid == dstId
|
||||
})
|
||||
const recommendedF0Factor = dst && src ? dst.correspondence / src.correspondence : 0
|
||||
@ -46,7 +46,7 @@ export const useSpeakerSetting = () => {
|
||||
console.log()
|
||||
|
||||
const srcIdRow = useMemo(() => {
|
||||
const selected = appState.clientSetting.setting.correspondences?.find(x => {
|
||||
const selected = appState.serverSetting.serverSetting.correspondences?.find(x => {
|
||||
return x.sid == appState.serverSetting.serverSetting.srcId
|
||||
})
|
||||
return (
|
||||
@ -58,7 +58,7 @@ export const useSpeakerSetting = () => {
|
||||
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, srcId: Number(e.target.value), f0Factor: recF0 })
|
||||
}}>
|
||||
{
|
||||
appState.clientSetting.setting.correspondences?.map(x => {
|
||||
appState.serverSetting.serverSetting.correspondences?.map(x => {
|
||||
return <option key={x.sid} value={x.sid}>{x.dirname}({x.sid})</option>
|
||||
})
|
||||
|
||||
@ -71,10 +71,10 @@ export const useSpeakerSetting = () => {
|
||||
<div className="body-item-text"></div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.speakers, appState.serverSetting.serverSetting.srcId, appState.serverSetting.serverSetting.dstId, appState.clientSetting.setting.correspondences, appState.serverSetting.updateServerSettings])
|
||||
}, [appState.serverSetting.serverSetting.srcId, appState.serverSetting.serverSetting.dstId, appState.serverSetting.serverSetting.correspondences, appState.serverSetting.updateServerSettings])
|
||||
|
||||
const dstIdRow = useMemo(() => {
|
||||
const selected = appState.clientSetting.setting.correspondences?.find(x => {
|
||||
const selected = appState.serverSetting.serverSetting.correspondences?.find(x => {
|
||||
return x.sid == appState.serverSetting.serverSetting.dstId
|
||||
})
|
||||
return (
|
||||
@ -90,7 +90,7 @@ export const useSpeakerSetting = () => {
|
||||
// appState.clientSetting.setting.speakers.map(x => {
|
||||
// return <option key={x.id} value={x.id}>{x.name}({x.id})</option>
|
||||
// })
|
||||
appState.clientSetting.setting.correspondences?.map(x => {
|
||||
appState.serverSetting.serverSetting.correspondences?.map(x => {
|
||||
return <option key={x.sid} value={x.sid}>{x.dirname}({x.sid})</option>
|
||||
})
|
||||
}
|
||||
@ -102,60 +102,60 @@ export const useSpeakerSetting = () => {
|
||||
<div className="body-item-text"></div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.speakers, appState.serverSetting.serverSetting.srcId, appState.serverSetting.serverSetting.dstId, appState.clientSetting.setting.correspondences, appState.serverSetting.updateServerSettings])
|
||||
}, [appState.serverSetting.serverSetting.srcId, appState.serverSetting.serverSetting.dstId, appState.serverSetting.serverSetting.correspondences, appState.serverSetting.updateServerSettings])
|
||||
|
||||
const editSpeakerIdMappingRow = useMemo(() => {
|
||||
const onSetSpeakerMappingClicked = async () => {
|
||||
const targetId = editSpeakerTargetId
|
||||
const targetName = editSpeakerTargetName
|
||||
const targetSpeaker = appState.clientSetting.setting.speakers.find(x => { return x.id == targetId })
|
||||
if (targetSpeaker) {
|
||||
if (targetName.length == 0) { // Delete
|
||||
const newSpeakers = appState.clientSetting.setting.speakers.filter(x => { return x.id != targetId })
|
||||
appState.clientSetting.setSpeakers(newSpeakers)
|
||||
} else { // Update
|
||||
targetSpeaker.name = targetName
|
||||
appState.clientSetting.setSpeakers([...appState.clientSetting.setting.speakers])
|
||||
}
|
||||
} else {
|
||||
if (targetName.length == 0) { // Noop
|
||||
} else {// add
|
||||
appState.clientSetting.setting.speakers.push({
|
||||
id: targetId,
|
||||
name: targetName
|
||||
})
|
||||
appState.clientSetting.setSpeakers([...appState.clientSetting.setting.speakers])
|
||||
}
|
||||
}
|
||||
}
|
||||
return (
|
||||
<div className="body-row split-3-1-2-4 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">Edit Speaker Mapping</div>
|
||||
<div className="body-input-container">
|
||||
<input type="number" min={1} max={256} step={1} value={editSpeakerTargetId} onChange={(e) => {
|
||||
const id = Number(e.target.value)
|
||||
setEditSpeakerTargetId(id)
|
||||
setEditSpeakerTargetName(appState.clientSetting.setting.speakers.find(x => { return x.id == id })?.name || "")
|
||||
}} />
|
||||
</div>
|
||||
<div className="body-input-container">
|
||||
<input type="text" value={editSpeakerTargetName} onChange={(e) => {
|
||||
setEditSpeakerTargetName(e.target.value)
|
||||
}} />
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
<div className="body-button" onClick={onSetSpeakerMappingClicked}>set</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.speakers, editSpeakerTargetId, editSpeakerTargetName])
|
||||
// const editSpeakerIdMappingRow = useMemo(() => {
|
||||
// const onSetSpeakerMappingClicked = async () => {
|
||||
// const targetId = editSpeakerTargetId
|
||||
// const targetName = editSpeakerTargetName
|
||||
// const targetSpeaker = appState.clientSetting.setting.speakers.find(x => { return x.id == targetId })
|
||||
// if (targetSpeaker) {
|
||||
// if (targetName.length == 0) { // Delete
|
||||
// const newSpeakers = appState.clientSetting.setting.speakers.filter(x => { return x.id != targetId })
|
||||
// appState.clientSetting.setSpeakers(newSpeakers)
|
||||
// } else { // Update
|
||||
// targetSpeaker.name = targetName
|
||||
// appState.clientSetting.setSpeakers([...appState.clientSetting.setting.speakers])
|
||||
// }
|
||||
// } else {
|
||||
// if (targetName.length == 0) { // Noop
|
||||
// } else {// add
|
||||
// appState.clientSetting.setting.speakers.push({
|
||||
// id: targetId,
|
||||
// name: targetName
|
||||
// })
|
||||
// appState.clientSetting.setSpeakers([...appState.clientSetting.setting.speakers])
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// return (
|
||||
// <div className="body-row split-3-1-2-4 left-padding-1 guided">
|
||||
// <div className="body-item-title left-padding-1">Edit Speaker Mapping</div>
|
||||
// <div className="body-input-container">
|
||||
// <input type="number" min={1} max={256} step={1} value={editSpeakerTargetId} onChange={(e) => {
|
||||
// const id = Number(e.target.value)
|
||||
// setEditSpeakerTargetId(id)
|
||||
// setEditSpeakerTargetName(appState.clientSetting.setting.speakers.find(x => { return x.id == id })?.name || "")
|
||||
// }} />
|
||||
// </div>
|
||||
// <div className="body-input-container">
|
||||
// <input type="text" value={editSpeakerTargetName} onChange={(e) => {
|
||||
// setEditSpeakerTargetName(e.target.value)
|
||||
// }} />
|
||||
// </div>
|
||||
// <div className="body-button-container">
|
||||
// <div className="body-button" onClick={onSetSpeakerMappingClicked}>set</div>
|
||||
// </div>
|
||||
// </div>
|
||||
// )
|
||||
// }, [appState.clientSetting.setting.speakers, editSpeakerTargetId, editSpeakerTargetName])
|
||||
|
||||
|
||||
const f0FactorRow = useMemo(() => {
|
||||
const src = appState.clientSetting.setting.correspondences?.find(x => {
|
||||
const src = appState.serverSetting.serverSetting.correspondences?.find(x => {
|
||||
return x.sid == appState.serverSetting.serverSetting.srcId
|
||||
})
|
||||
const dst = appState.clientSetting.setting.correspondences?.find(x => {
|
||||
const dst = appState.serverSetting.serverSetting.correspondences?.find(x => {
|
||||
return x.sid == appState.serverSetting.serverSetting.dstId
|
||||
})
|
||||
|
||||
@ -174,7 +174,7 @@ export const useSpeakerSetting = () => {
|
||||
<div className="body-item-text">recommend: {recommendedF0Factor.toFixed(1)}</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.serverSetting.serverSetting.f0Factor, appState.serverSetting.serverSetting.srcId, appState.serverSetting.serverSetting.dstId, appState.clientSetting.setting.correspondences, appState.serverSetting.updateServerSettings])
|
||||
}, [appState.serverSetting.serverSetting.f0Factor, appState.serverSetting.serverSetting.srcId, appState.serverSetting.serverSetting.dstId, appState.serverSetting.serverSetting.correspondences, appState.serverSetting.updateServerSettings])
|
||||
|
||||
const speakerSetting = useMemo(() => {
|
||||
return (
|
||||
@ -198,7 +198,7 @@ export const useSpeakerSetting = () => {
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}, [srcIdRow, dstIdRow, editSpeakerIdMappingRow, f0FactorRow])
|
||||
}, [srcIdRow, dstIdRow, f0FactorRow])
|
||||
|
||||
return {
|
||||
speakerSetting,
|
||||
|
||||
@ -25,18 +25,18 @@ export const useConvertSetting = (): ConvertSettingState => {
|
||||
<div className="body-row split-3-2-1-4 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">Input Chunk Num(128sample/chunk)</div>
|
||||
<div className="body-input-container">
|
||||
<input type="number" min={1} max={256} step={1} value={appState.clientSetting.setting.inputChunkNum} onChange={(e) => {
|
||||
appState.clientSetting.setInputChunkNum(Number(e.target.value))
|
||||
<input type="number" min={1} max={256} step={1} value={appState.streamerSetting.audioStreamerSetting.inputChunkNum} onChange={(e) => {
|
||||
appState.streamerSetting.setSetting({ ...appState.streamerSetting.audioStreamerSetting, inputChunkNum: Number(e.target.value) })
|
||||
}} />
|
||||
</div>
|
||||
<div className="body-item-text">
|
||||
<div>buff: {(appState.clientSetting.setting.inputChunkNum * 128 * 1000 / 24000).toFixed(1)}ms</div>
|
||||
<div>buff: {(appState.streamerSetting.audioStreamerSetting.inputChunkNum * 128 * 1000 / 24000).toFixed(1)}ms</div>
|
||||
</div>
|
||||
<div className="body-item-text"></div>
|
||||
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.inputChunkNum, appState.clientSetting.setInputChunkNum])
|
||||
}, [appState.streamerSetting.audioStreamerSetting.inputChunkNum, appState.streamerSetting.setSetting])
|
||||
|
||||
const gpuRow = useMemo(() => {
|
||||
return (
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { BufferSize, CrossFadeOverlapSize, DownSamplingMode, InputSampleRate, Protocol, SampleRate, VoiceChangerMode } from "@dannadori/voice-changer-client-js"
|
||||
import { BufferSize, CrossFadeOverlapSize, DownSamplingMode, InputSampleRate, Protocol, SampleRate } from "@dannadori/voice-changer-client-js"
|
||||
import React, { useMemo } from "react"
|
||||
import { useAppState } from "./001_provider/001_AppStateProvider";
|
||||
import { AnimationTypes, HeaderButton, HeaderButtonProps } from "./components/101_HeaderButton";
|
||||
@ -30,24 +30,24 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
<div className="body-row split-3-3-4 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">MMVC Server</div>
|
||||
<div className="body-input-container">
|
||||
<input type="text" defaultValue={appState.clientSetting.setting.mmvcServerUrl} id="mmvc-server-url" className="body-item-input" />
|
||||
<input type="text" defaultValue={appState.streamerSetting.audioStreamerSetting.serverUrl} id="mmvc-server-url" className="body-item-input" />
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
<div className="body-button" onClick={onSetServerClicked}>set</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.mmvcServerUrl, appState.clientSetting.setServerUrl])
|
||||
}, [appState.streamerSetting.audioStreamerSetting.serverUrl, appState.clientSetting.setServerUrl])
|
||||
|
||||
const protocolRow = useMemo(() => {
|
||||
const onProtocolChanged = async (val: Protocol) => {
|
||||
appState.clientSetting.setProtocol(val)
|
||||
appState.streamerSetting.setSetting({ ...appState.streamerSetting.audioStreamerSetting, protocol: val })
|
||||
}
|
||||
return (
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">Protocol</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={appState.clientSetting.setting.protocol} onChange={(e) => {
|
||||
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.protocol} onChange={(e) => {
|
||||
onProtocolChanged(e.target.value as
|
||||
Protocol)
|
||||
}}>
|
||||
@ -60,7 +60,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.protocol, appState.clientSetting.setProtocol])
|
||||
}, [appState.streamerSetting.audioStreamerSetting.protocol, appState.streamerSetting.setSetting])
|
||||
|
||||
|
||||
const sampleRateRow = useMemo(() => {
|
||||
@ -68,8 +68,8 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">Sample Rate</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={appState.clientSetting.setting.sampleRate} onChange={(e) => {
|
||||
appState.clientSetting.setSampleRate(Number(e.target.value) as SampleRate)
|
||||
<select className="body-select" value={appState.clientSetting.clientSetting.sampleRate} onChange={(e) => {
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, sampleRate: Number(e.target.value) as SampleRate })
|
||||
}}>
|
||||
{
|
||||
Object.values(SampleRate).map(x => {
|
||||
@ -80,15 +80,15 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.sampleRate, appState.clientSetting.setSampleRate])
|
||||
}, [appState.clientSetting.clientSetting.sampleRate, appState.clientSetting.updateClientSetting])
|
||||
|
||||
const sendingSampleRateRow = useMemo(() => {
|
||||
return (
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">Sending Sample Rate</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={appState.clientSetting.setting.sendingSampleRate} onChange={(e) => {
|
||||
appState.clientSetting.setSendingSampleRate(Number(e.target.value) as InputSampleRate)
|
||||
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.sendingSampleRate} onChange={(e) => {
|
||||
appState.streamerSetting.setSetting({ ...appState.streamerSetting.audioStreamerSetting, sendingSampleRate: Number(e.target.value) as InputSampleRate })
|
||||
appState.serverSetting.updateServerSettings({ ...appState.serverSetting.serverSetting, inputSampleRate: Number(e.target.value) as InputSampleRate })
|
||||
}}>
|
||||
{
|
||||
@ -100,7 +100,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.sendingSampleRate, appState.clientSetting.setSendingSampleRate, appState.serverSetting.updateServerSettings])
|
||||
}, [appState.streamerSetting.audioStreamerSetting.sendingSampleRate, appState.streamerSetting.setSetting, appState.serverSetting.updateServerSettings])
|
||||
|
||||
const bufferSizeRow = useMemo(() => {
|
||||
return (
|
||||
@ -108,8 +108,8 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1">Buffer Size</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={appState.clientSetting.setting.bufferSize} onChange={(e) => {
|
||||
appState.clientSetting.setBufferSize(Number(e.target.value) as BufferSize)
|
||||
<select className="body-select" value={appState.clientSetting.clientSetting.bufferSize} onChange={(e) => {
|
||||
appState.clientSetting.updateClientSetting({ ...appState.clientSetting.clientSetting, bufferSize: Number(e.target.value) as BufferSize })
|
||||
}}>
|
||||
{
|
||||
Object.values(BufferSize).map(x => {
|
||||
@ -120,7 +120,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.bufferSize, appState.clientSetting.setBufferSize])
|
||||
}, [appState.clientSetting.clientSetting.bufferSize, appState.clientSetting.updateClientSetting])
|
||||
|
||||
|
||||
const crossFadeOverlapSizeRow = useMemo(() => {
|
||||
@ -169,33 +169,13 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
}, [appState.serverSetting.serverSetting.crossFadeEndRate, appState.serverSetting.updateServerSettings])
|
||||
|
||||
|
||||
const voiceChangeModeRow = useMemo(() => {
|
||||
return (
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1 ">Voice Change Mode</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={appState.clientSetting.setting.voiceChangerMode} onChange={(e) => {
|
||||
appState.clientSetting.setVoiceChangerMode(e.target.value as VoiceChangerMode)
|
||||
}}>
|
||||
{
|
||||
Object.values(VoiceChangerMode).map(x => {
|
||||
return <option key={x} value={x}>{x}</option>
|
||||
})
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.voiceChangerMode, appState.clientSetting.setVoiceChangerMode])
|
||||
|
||||
|
||||
const downSamplingModeRow = useMemo(() => {
|
||||
return (
|
||||
<div className="body-row split-3-7 left-padding-1 guided">
|
||||
<div className="body-item-title left-padding-1 ">DownSamplingMode</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={appState.clientSetting.setting.downSamplingMode} onChange={(e) => {
|
||||
appState.clientSetting.setDownSamplingMode(e.target.value as DownSamplingMode)
|
||||
<select className="body-select" value={appState.streamerSetting.audioStreamerSetting.downSamplingMode} onChange={(e) => {
|
||||
appState.streamerSetting.setSetting({ ...appState.streamerSetting.audioStreamerSetting, downSamplingMode: e.target.value as DownSamplingMode })
|
||||
}}>
|
||||
{
|
||||
Object.values(DownSamplingMode).map(x => {
|
||||
@ -206,7 +186,7 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}, [appState.clientSetting.setting.downSamplingMode, appState.clientSetting.setDownSamplingMode])
|
||||
}, [appState.streamerSetting.audioStreamerSetting.downSamplingMode, appState.streamerSetting.setSetting])
|
||||
|
||||
|
||||
const workletSettingRow = useMemo(() => {
|
||||
@ -268,15 +248,13 @@ export const useAdvancedSetting = (): AdvancedSettingState => {
|
||||
{crossFadeOffsetRateRow}
|
||||
{crossFadeEndRateRow}
|
||||
<div className="body-row divider"></div>
|
||||
{voiceChangeModeRow}
|
||||
<div className="body-row divider"></div>
|
||||
{workletSettingRow}
|
||||
<div className="body-row divider"></div>
|
||||
{downSamplingModeRow}
|
||||
|
||||
</>
|
||||
)
|
||||
}, [mmvcServerUrlRow, protocolRow, sampleRateRow, sendingSampleRateRow, bufferSizeRow, crossFadeOverlapSizeRow, crossFadeOffsetRateRow, crossFadeEndRateRow, voiceChangeModeRow, workletSettingRow, downSamplingModeRow])
|
||||
}, [mmvcServerUrlRow, protocolRow, sampleRateRow, sendingSampleRateRow, bufferSizeRow, crossFadeOverlapSizeRow, crossFadeOffsetRateRow, crossFadeEndRateRow, workletSettingRow, downSamplingModeRow])
|
||||
|
||||
|
||||
const advancedSetting = useMemo(() => {
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
import { io, Socket } from "socket.io-client";
|
||||
import { DefaultEventsMap } from "@socket.io/component-emitter";
|
||||
import { Duplex, DuplexOptions } from "readable-stream";
|
||||
import { DefaultVoiceChangerClientSetting, DownSamplingMode, Protocol, SendingSampleRate, VoiceChangerMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
|
||||
import { AudioStreamerSetting, DefaultAudioStreamerSetting, DownSamplingMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
|
||||
|
||||
|
||||
export type Callbacks = {
|
||||
onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer) => void
|
||||
onVoiceReceived: (data: ArrayBuffer) => void
|
||||
}
|
||||
export type AudioStreamerListeners = {
|
||||
notifySendBufferingTime: (time: number) => void
|
||||
@ -13,34 +13,17 @@ export type AudioStreamerListeners = {
|
||||
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
|
||||
}
|
||||
|
||||
export type AudioStreamerSettings = {
|
||||
serverUrl: string;
|
||||
protocol: Protocol;
|
||||
inputChunkNum: number;
|
||||
voiceChangerMode: VoiceChangerMode;
|
||||
}
|
||||
|
||||
|
||||
export class AudioStreamer extends Duplex {
|
||||
private setting: AudioStreamerSetting = DefaultAudioStreamerSetting
|
||||
|
||||
private callbacks: Callbacks
|
||||
private audioStreamerListeners: AudioStreamerListeners
|
||||
private protocol: Protocol = "sio"
|
||||
private serverUrl = ""
|
||||
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
|
||||
private voiceChangerMode: VoiceChangerMode = "realtime"
|
||||
private inputChunkNum = 128
|
||||
private requestChunks: ArrayBuffer[] = []
|
||||
private recordChunks: ArrayBuffer[] = []
|
||||
private isRecording = false
|
||||
|
||||
// performance monitor
|
||||
private bufferStart = 0;
|
||||
|
||||
// Flags
|
||||
// private downSamplingMode: DownSamplingMode = DownSamplingMode.decimate
|
||||
private downSamplingMode: DownSamplingMode = DownSamplingMode.average
|
||||
private sendingSampleRate: number = DefaultVoiceChangerClientSetting.sendingSampleRate
|
||||
|
||||
constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
|
||||
super(options);
|
||||
this.callbacks = callbacks
|
||||
@ -51,13 +34,13 @@ export class AudioStreamer extends Duplex {
|
||||
if (this.socket) {
|
||||
this.socket.close()
|
||||
}
|
||||
if (this.protocol === "sio") {
|
||||
this.socket = io(this.serverUrl + "/test");
|
||||
if (this.setting.protocol === "sio") {
|
||||
this.socket = io(this.setting.serverUrl + "/test");
|
||||
this.socket.on('connect_error', (err) => {
|
||||
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
|
||||
})
|
||||
this.socket.on('connect', () => {
|
||||
console.log(`[SIO] sonnect to ${this.serverUrl}`)
|
||||
console.log(`[SIO] sonnect to ${this.setting.serverUrl}`)
|
||||
console.log(`[SIO] ${this.socket?.id}`)
|
||||
});
|
||||
this.socket.on('response', (response: any[]) => {
|
||||
@ -67,66 +50,40 @@ export class AudioStreamer extends Duplex {
|
||||
if (result.byteLength < 128 * 2) {
|
||||
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
|
||||
} else {
|
||||
this.callbacks.onVoiceReceived(this.voiceChangerMode, response[1])
|
||||
this.callbacks.onVoiceReceived(response[1])
|
||||
this.audioStreamerListeners.notifyResponseTime(responseTime)
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Option Change
|
||||
setServerUrl = (serverUrl: string) => {
|
||||
this.serverUrl = serverUrl
|
||||
console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.protocol}`)
|
||||
this.createSocketIO()// mode check is done in the method.
|
||||
}
|
||||
setProtocol = (mode: Protocol) => {
|
||||
this.protocol = mode
|
||||
console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.protocol}`)
|
||||
this.createSocketIO()// mode check is done in the method.
|
||||
}
|
||||
|
||||
setInputChunkNum = (num: number) => {
|
||||
this.inputChunkNum = num
|
||||
}
|
||||
|
||||
setVoiceChangerMode = (val: VoiceChangerMode) => {
|
||||
this.voiceChangerMode = val
|
||||
}
|
||||
|
||||
// set Flags
|
||||
setDownSamplingMode = (val: DownSamplingMode) => {
|
||||
this.downSamplingMode = val
|
||||
}
|
||||
setSendingSampleRate = (val: SendingSampleRate) => {
|
||||
this.sendingSampleRate = val
|
||||
}
|
||||
|
||||
getSettings = (): AudioStreamerSettings => {
|
||||
return {
|
||||
serverUrl: this.serverUrl,
|
||||
protocol: this.protocol,
|
||||
inputChunkNum: this.inputChunkNum,
|
||||
voiceChangerMode: this.voiceChangerMode
|
||||
updateSetting = (setting: AudioStreamerSetting) => {
|
||||
console.log(`[AudioStreamer] Updating AudioStreamer Setting,`, this.setting, setting)
|
||||
let recreateSocketIoRequired = false
|
||||
if (this.setting.serverUrl != setting.serverUrl || this.setting.protocol != setting.protocol) {
|
||||
recreateSocketIoRequired = true
|
||||
}
|
||||
this.setting = setting
|
||||
if (recreateSocketIoRequired) {
|
||||
this.createSocketIO()
|
||||
}
|
||||
}
|
||||
|
||||
getSettings = (): AudioStreamerSetting => {
|
||||
return this.setting
|
||||
}
|
||||
|
||||
getSocketId = () => {
|
||||
return this.socket?.id
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Main Process
|
||||
//// Pipe from mic stream
|
||||
_write = (chunk: AudioBuffer, _encoding: any, callback: any) => {
|
||||
const buffer = chunk.getChannelData(0);
|
||||
// console.log("SAMPLERATE:", chunk.sampleRate, chunk.numberOfChannels, chunk.length, buffer)
|
||||
if (this.voiceChangerMode === "realtime") {
|
||||
this._write_realtime(buffer)
|
||||
} else {
|
||||
this._write_record(buffer)
|
||||
}
|
||||
this._write_realtime(buffer)
|
||||
callback();
|
||||
}
|
||||
|
||||
@ -163,9 +120,9 @@ export class AudioStreamer extends Duplex {
|
||||
private _write_realtime = async (buffer: Float32Array) => {
|
||||
|
||||
let downsampledBuffer: Float32Array | null = null
|
||||
if (this.sendingSampleRate == 48000) {
|
||||
if (this.setting.sendingSampleRate == 48000) {
|
||||
downsampledBuffer = buffer
|
||||
} else if (this.downSamplingMode == DownSamplingMode.decimate) {
|
||||
} else if (this.setting.downSamplingMode == DownSamplingMode.decimate) {
|
||||
//////// (Kind 1) 間引き //////////
|
||||
// bufferSize個のデータ（48Khz）が入ってくる。
|
||||
//// 48000Hz で入ってくるので間引いて24000Hzに変換する。
|
||||
@ -178,7 +135,7 @@ export class AudioStreamer extends Duplex {
|
||||
} else {
|
||||
//////// (Kind 2) 平均 //////////
|
||||
// downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, 24000)
|
||||
downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, this.sendingSampleRate)
|
||||
downsampledBuffer = this._averageDownsampleBuffer(buffer, 48000, this.setting.sendingSampleRate)
|
||||
}
|
||||
|
||||
// Float to signed16
|
||||
@ -194,7 +151,7 @@ export class AudioStreamer extends Duplex {
|
||||
// 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理
|
||||
// const chunkByteSize = 256 // (const.ts ★1)
|
||||
// const chunkByteSize = 256 * 2 // (const.ts ★1)
|
||||
const chunkByteSize = (256 * 2) * (this.sendingSampleRate / 48000) // (const.ts ★1)
|
||||
const chunkByteSize = (256 * 2) * (this.setting.sendingSampleRate / 48000) // (const.ts ★1)
|
||||
for (let i = 0; i < arrayBuffer.byteLength / chunkByteSize; i++) {
|
||||
const ab = arrayBuffer.slice(i * chunkByteSize, (i + 1) * chunkByteSize)
|
||||
this.requestChunks.push(ab)
|
||||
@ -202,7 +159,7 @@ export class AudioStreamer extends Duplex {
|
||||
|
||||
|
||||
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
|
||||
if (this.requestChunks.length < this.inputChunkNum) {
|
||||
if (this.requestChunks.length < this.setting.inputChunkNum) {
|
||||
return
|
||||
}
|
||||
|
||||
@ -227,51 +184,9 @@ export class AudioStreamer extends Duplex {
|
||||
this.bufferStart = Date.now()
|
||||
}
|
||||
|
||||
|
||||
private _write_record = (buffer: Float32Array) => {
|
||||
if (!this.isRecording) { return }
|
||||
// buffer(for48Khz)x16bit * chunksize / 2(for24Khz)
|
||||
const sendBuffer = new ArrayBuffer(buffer.length * 2 / 2);
|
||||
const sendDataView = new DataView(sendBuffer);
|
||||
for (var i = 0; i < buffer.length; i++) {
|
||||
if (i % 2 == 0) {
|
||||
let s = Math.max(-1, Math.min(1, buffer[i]));
|
||||
s = s < 0 ? s * 0x8000 : s * 0x7FFF
|
||||
sendDataView.setInt16(i, s, true);
|
||||
// if (i % 3000 === 0) {
|
||||
// console.log("buffer_converting", s, buffer[i])
|
||||
// }
|
||||
}
|
||||
}
|
||||
this.recordChunks.push(sendBuffer)
|
||||
}
|
||||
|
||||
// Near Realtime用のトリガ
|
||||
sendRecordedData = () => {
|
||||
const length = this.recordChunks.reduce((prev, cur) => {
|
||||
return prev + cur.byteLength
|
||||
}, 0)
|
||||
const newBuffer = new Uint8Array(length);
|
||||
this.recordChunks.reduce((prev, cur) => {
|
||||
newBuffer.set(new Uint8Array(cur), prev)
|
||||
return prev + cur.byteLength
|
||||
}, 0)
|
||||
|
||||
this.sendBuffer(newBuffer)
|
||||
}
|
||||
|
||||
startRecord = () => {
|
||||
this.recordChunks = []
|
||||
this.isRecording = true
|
||||
}
|
||||
|
||||
stopRecord = () => {
|
||||
this.isRecording = false
|
||||
}
|
||||
|
||||
private sendBuffer = async (newBuffer: Uint8Array) => {
|
||||
const timestamp = Date.now()
|
||||
if (this.protocol === "sio") {
|
||||
if (this.setting.protocol === "sio") {
|
||||
if (!this.socket) {
|
||||
console.warn(`sio is not initialized`)
|
||||
return
|
||||
@ -282,14 +197,14 @@ export class AudioStreamer extends Duplex {
|
||||
newBuffer.buffer]);
|
||||
} else {
|
||||
const res = await postVoice(
|
||||
this.serverUrl + "/test",
|
||||
this.setting.serverUrl + "/test",
|
||||
timestamp,
|
||||
newBuffer.buffer)
|
||||
|
||||
if (res.byteLength < 128 * 2) {
|
||||
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
|
||||
} else {
|
||||
this.callbacks.onVoiceReceived(this.voiceChangerMode, res)
|
||||
this.callbacks.onVoiceReceived(res)
|
||||
this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@ import { VoiceChangerWorkletNode, VoiceChangerWorkletListener } from "./VoiceCha
|
||||
import workerjs from "raw-loader!../worklet/dist/index.js";
|
||||
import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js";
|
||||
import { createDummyMediaStream, validateUrl } from "./util";
|
||||
import { BufferSize, DefaultVoiceChangerClientSetting, DownSamplingMode, Protocol, SendingSampleRate, ServerSettingKey, VoiceChangerMode, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletSetting } from "./const";
|
||||
import { AudioStreamerSetting, DefaultVoiceChangerClientSetting, ServerSettingKey, VoiceChangerClientSetting, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletSetting } from "./const";
|
||||
import MicrophoneStream from "microphone-stream";
|
||||
import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreamer";
|
||||
import { ServerConfigurator } from "./ServerConfigurator";
|
||||
@ -31,41 +31,19 @@ export class VoiceChangerClient {
|
||||
private vcNode!: VoiceChangerWorkletNode
|
||||
private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode
|
||||
|
||||
private inputGain = 1.0
|
||||
|
||||
private promiseForInitialize: Promise<void>
|
||||
private _isVoiceChanging = false
|
||||
|
||||
private setting: VoiceChangerClientSetting = DefaultVoiceChangerClientSetting
|
||||
|
||||
private sslCertified: string[] = []
|
||||
|
||||
private sem = new BlockingQueue<number>();
|
||||
|
||||
private callbacks: Callbacks = {
|
||||
onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer): void => {
|
||||
// console.log(voiceChangerMode, data)
|
||||
if (voiceChangerMode === "realtime") {
|
||||
this.vcNode.postReceivedVoice(data)
|
||||
return
|
||||
}
|
||||
|
||||
// For Near Realtime Mode
|
||||
console.log("near realtime mode")
|
||||
|
||||
const i16Data = new Int16Array(data)
|
||||
const f32Data = new Float32Array(i16Data.length)
|
||||
// https://stackoverflow.com/questions/35234551/javascript-converting-from-int16-to-float32
|
||||
i16Data.forEach((x, i) => {
|
||||
const float = (x >= 0x8000) ? -(0x10000 - x) / 0x8000 : x / 0x7FFF;
|
||||
f32Data[i] = float
|
||||
|
||||
})
|
||||
|
||||
const source = this.ctx.createBufferSource();
|
||||
const buffer = this.ctx.createBuffer(1, f32Data.length, 24000);
|
||||
buffer.getChannelData(0).set(f32Data);
|
||||
source.buffer = buffer;
|
||||
source.start();
|
||||
source.connect(this.currentMediaStreamAudioDestinationNode)
|
||||
onVoiceReceived: (data: ArrayBuffer): void => {
|
||||
this.vcNode.postReceivedVoice(data)
|
||||
}
|
||||
}
|
||||
|
||||
@ -81,12 +59,11 @@ export class VoiceChangerClient {
|
||||
this.vcNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node
|
||||
this.currentMediaStreamAudioDestinationNode = this.ctx.createMediaStreamDestination() // output node
|
||||
this.outputGainNode = this.ctx.createGain()
|
||||
this.outputGainNode.gain.value = this.setting.outputGain
|
||||
this.vcNode.connect(this.outputGainNode) // vc node -> output node
|
||||
this.outputGainNode.connect(this.currentMediaStreamAudioDestinationNode)
|
||||
// (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる)
|
||||
this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, })
|
||||
this.audioStreamer.setInputChunkNum(DefaultVoiceChangerClientSetting.inputChunkNum)
|
||||
this.audioStreamer.setVoiceChangerMode(DefaultVoiceChangerClientSetting.voiceChangerMode)
|
||||
|
||||
if (this.vfEnable) {
|
||||
this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' })
|
||||
@ -106,7 +83,6 @@ export class VoiceChangerClient {
|
||||
this.sem.enqueue(num + 1);
|
||||
};
|
||||
|
||||
|
||||
isInitialized = async () => {
|
||||
if (this.promiseForInitialize) {
|
||||
await this.promiseForInitialize
|
||||
@ -114,15 +90,15 @@ export class VoiceChangerClient {
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// オペレーション
|
||||
/////////////////////////////////////////////////////
|
||||
/// Operations ///
|
||||
setup = async (input: string | MediaStream | null, bufferSize: BufferSize, echoCancel: boolean = true, noiseSuppression: boolean = true, noiseSuppression2: boolean = false) => {
|
||||
// setup = async (input: string | MediaStream | null, bufferSize: BufferSize, echoCancel: boolean = true, noiseSuppression: boolean = true, noiseSuppression2: boolean = false) => {
|
||||
setup = async () => {
|
||||
const lockNum = await this.lock()
|
||||
|
||||
console.log(`Input Setup=> echo: ${echoCancel}, noise1: ${noiseSuppression}, noise2: ${noiseSuppression2}`)
|
||||
console.log(`Input Setup=> echo: ${this.setting.echoCancel}, noise1: ${this.setting.noiseSuppression}, noise2: ${this.setting.noiseSuppression2}`)
|
||||
// condition check
|
||||
if (!this.vcNode) {
|
||||
console.warn("vc node is not initialized.")
|
||||
@ -137,7 +113,7 @@ export class VoiceChangerClient {
|
||||
}
|
||||
|
||||
//// Input デバイスがnullの時はmicStreamを止めてリターン
|
||||
if (!input) {
|
||||
if (!this.setting.audioInput) {
|
||||
console.log(`Input Setup=> client mic is disabled.`)
|
||||
if (this.micStream) {
|
||||
this.micStream.pauseRecording()
|
||||
@ -146,16 +122,16 @@ export class VoiceChangerClient {
|
||||
return
|
||||
}
|
||||
|
||||
if (typeof input == "string") {
|
||||
if (typeof this.setting.audioInput == "string") {
|
||||
this.currentMediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
deviceId: input,
|
||||
deviceId: this.setting.audioInput,
|
||||
channelCount: 1,
|
||||
sampleRate: 48000,
|
||||
sampleRate: this.setting.sampleRate,
|
||||
sampleSize: 16,
|
||||
autoGainControl: false,
|
||||
echoCancellation: echoCancel,
|
||||
noiseSuppression: noiseSuppression
|
||||
echoCancellation: this.setting.echoCancel,
|
||||
noiseSuppression: this.setting.noiseSuppression
|
||||
}
|
||||
})
|
||||
// this.currentMediaStream.getAudioTracks().forEach((x) => {
|
||||
@ -164,7 +140,7 @@ export class VoiceChangerClient {
|
||||
// console.log("MIC Setting(setting)", x.getSettings())
|
||||
// })
|
||||
} else {
|
||||
this.currentMediaStream = input
|
||||
this.currentMediaStream = this.setting.audioInput
|
||||
}
|
||||
|
||||
// create mic stream
|
||||
@ -175,15 +151,15 @@ export class VoiceChangerClient {
|
||||
}
|
||||
this.micStream = new MicrophoneStream({
|
||||
objectMode: true,
|
||||
bufferSize: bufferSize,
|
||||
bufferSize: this.setting.bufferSize,
|
||||
context: this.ctx
|
||||
})
|
||||
// connect nodes.
|
||||
this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream)
|
||||
this.inputGainNode = this.ctx.createGain()
|
||||
this.inputGainNode.gain.value = this.inputGain
|
||||
this.inputGainNode.gain.value = this.setting.inputGain
|
||||
this.currentMediaStreamAudioSourceNode.connect(this.inputGainNode)
|
||||
if (this.currentDevice && noiseSuppression2) {
|
||||
if (this.currentDevice && this.setting.noiseSuppression2) {
|
||||
this.currentDevice.chooseNewInnerDevice(this.currentMediaStream)
|
||||
const voiceFocusNode = await this.currentDevice.createAudioNode(this.ctx); // vf node
|
||||
this.inputGainNode.connect(voiceFocusNode.start) // input node -> vf node
|
||||
@ -242,12 +218,32 @@ export class VoiceChangerClient {
|
||||
}
|
||||
}
|
||||
}
|
||||
this.audioStreamer.setServerUrl(url)
|
||||
this.audioStreamer.updateSetting({ ...this.audioStreamer.getSettings(), serverUrl: url })
|
||||
this.configurator.setServerUrl(url)
|
||||
}
|
||||
|
||||
updateClientSetting = (setting: VoiceChangerClientSetting) => {
|
||||
console.log(`[VoiceChangerClient] Updating Client Setting,`, this.setting, setting)
|
||||
let reconstructInputRequired = false
|
||||
if (
|
||||
this.setting.audioInput != setting.audioInput ||
|
||||
this.setting.bufferSize != setting.bufferSize ||
|
||||
this.setting.echoCancel != setting.echoCancel ||
|
||||
this.setting.noiseSuppression != setting.noiseSuppression ||
|
||||
this.setting.noiseSuppression2 != setting.noiseSuppression2 ||
|
||||
this.setting.sampleRate != setting.sampleRate
|
||||
) {
|
||||
reconstructInputRequired = true
|
||||
}
|
||||
|
||||
this.setting = setting
|
||||
if (reconstructInputRequired) {
|
||||
this.setup()
|
||||
}
|
||||
}
|
||||
|
||||
setInputGain = (val: number) => {
|
||||
this.inputGain = val
|
||||
this.setting.inputGain = val
|
||||
if (!this.inputGainNode) {
|
||||
return
|
||||
}
|
||||
@ -291,23 +287,8 @@ export class VoiceChangerClient {
|
||||
|
||||
|
||||
//## Audio Streamer ##//
|
||||
setProtocol = (mode: Protocol) => {
|
||||
this.audioStreamer.setProtocol(mode)
|
||||
}
|
||||
|
||||
setInputChunkNum = (num: number) => {
|
||||
this.audioStreamer.setInputChunkNum(num)
|
||||
}
|
||||
|
||||
setVoiceChangerMode = (val: VoiceChangerMode) => {
|
||||
this.audioStreamer.setVoiceChangerMode(val)
|
||||
}
|
||||
//// Audio Streamer Flag
|
||||
setDownSamplingMode = (val: DownSamplingMode) => {
|
||||
this.audioStreamer.setDownSamplingMode(val)
|
||||
}
|
||||
setSendingSampleRate = (val: SendingSampleRate) => {
|
||||
this.audioStreamer.setSendingSampleRate(val)
|
||||
updateAudioStreamerSetting = (setting: AudioStreamerSetting) => {
|
||||
this.audioStreamer.updateSetting(setting)
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -41,8 +41,6 @@ export const F0Detector = {
|
||||
} as const
|
||||
export type F0Detector = typeof F0Detector[keyof typeof F0Detector]
|
||||
|
||||
|
||||
|
||||
export const ServerSettingKey = {
|
||||
"srcId": "srcId",
|
||||
"dstId": "dstId",
|
||||
@ -64,6 +62,16 @@ export const ServerSettingKey = {
|
||||
export type ServerSettingKey = typeof ServerSettingKey[keyof typeof ServerSettingKey]
|
||||
|
||||
|
||||
export type Speaker = {
|
||||
"id": number,
|
||||
"name": string,
|
||||
}
|
||||
export type Correspondence = {
|
||||
"sid": number,
|
||||
"correspondence": number,
|
||||
"dirname": string
|
||||
}
|
||||
|
||||
export type VoiceChangerServerSetting = {
|
||||
srcId: number,
|
||||
dstId: number,
|
||||
@ -81,6 +89,7 @@ export type VoiceChangerServerSetting = {
|
||||
recordIO: number // 0:off, 1:on
|
||||
|
||||
inputSampleRate: InputSampleRate
|
||||
|
||||
}
|
||||
|
||||
export type ServerInfo = VoiceChangerServerSetting & {
|
||||
@ -89,6 +98,9 @@ export type ServerInfo = VoiceChangerServerSetting & {
|
||||
pyTorchModelFile: string,
|
||||
onnxModelFile: string,
|
||||
onnxExecutionProviders: OnnxExecutionProvider[]
|
||||
|
||||
speakers: Speaker[],
|
||||
correspondences: Correspondence[],
|
||||
}
|
||||
|
||||
export const DefaultServerSetting: ServerInfo = {
|
||||
@ -113,122 +125,9 @@ export const DefaultServerSetting: ServerInfo = {
|
||||
configFile: "",
|
||||
pyTorchModelFile: "",
|
||||
onnxModelFile: "",
|
||||
onnxExecutionProviders: []
|
||||
}
|
||||
onnxExecutionProviders: [],
|
||||
|
||||
|
||||
///////////////////////
|
||||
// Workletセッティング
|
||||
///////////////////////
|
||||
|
||||
///////////////////////
|
||||
// Clientセッティング
|
||||
///////////////////////
|
||||
|
||||
|
||||
export type VoiceChangerClientSetting = {
|
||||
audioInput: string | MediaStream | null,
|
||||
mmvcServerUrl: string,
|
||||
protocol: Protocol,
|
||||
sampleRate: SampleRate, // 48000Hz
|
||||
sendingSampleRate: SendingSampleRate,
|
||||
bufferSize: BufferSize, // 256, 512, 1024, 2048, 4096, 8192, 16384 (for mic stream)
|
||||
inputChunkNum: number, // n of (256 x n) for send buffer
|
||||
speakers: Speaker[],
|
||||
correspondences: Correspondence[],
|
||||
echoCancel: boolean,
|
||||
noiseSuppression: boolean,
|
||||
noiseSuppression2: boolean,
|
||||
voiceChangerMode: VoiceChangerMode,
|
||||
downSamplingMode: DownSamplingMode,
|
||||
|
||||
inputGain: number
|
||||
outputGain: number
|
||||
}
|
||||
|
||||
export type WorkletSetting = {
|
||||
numTrancateTreshold: number,
|
||||
volTrancateThreshold: number,
|
||||
volTrancateLength: number
|
||||
}
|
||||
|
||||
export type Speaker = {
|
||||
"id": number,
|
||||
"name": string,
|
||||
}
|
||||
export type Correspondence = {
|
||||
"sid": number,
|
||||
"correspondence": number,
|
||||
"dirname": string
|
||||
}
|
||||
|
||||
|
||||
export type ServerAudioDevice = {
|
||||
kind: string,
|
||||
index: number,
|
||||
name: string,
|
||||
hostAPI: string
|
||||
}
|
||||
|
||||
export type ServerAudioDevices = {
|
||||
audio_input_devices: ServerAudioDevice[]
|
||||
audio_output_devices: ServerAudioDevice[]
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Consts
|
||||
export const Protocol = {
|
||||
"sio": "sio",
|
||||
"rest": "rest",
|
||||
} as const
|
||||
export type Protocol = typeof Protocol[keyof typeof Protocol]
|
||||
|
||||
export const VoiceChangerMode = {
|
||||
"realtime": "realtime",
|
||||
"near-realtime": "near-realtime",
|
||||
} as const
|
||||
export type VoiceChangerMode = typeof VoiceChangerMode[keyof typeof VoiceChangerMode]
|
||||
|
||||
export const DownSamplingMode = {
|
||||
"decimate": "decimate",
|
||||
"average": "average"
|
||||
} as const
|
||||
export type DownSamplingMode = typeof DownSamplingMode[keyof typeof DownSamplingMode]
|
||||
|
||||
export const SampleRate = {
|
||||
"48000": 48000,
|
||||
} as const
|
||||
export type SampleRate = typeof SampleRate[keyof typeof SampleRate]
|
||||
|
||||
export const SendingSampleRate = {
|
||||
"48000": 48000,
|
||||
"24000": 24000
|
||||
} as const
|
||||
export type SendingSampleRate = typeof SendingSampleRate[keyof typeof SendingSampleRate]
|
||||
|
||||
export const BufferSize = {
|
||||
"256": 256,
|
||||
"512": 512,
|
||||
"1024": 1024,
|
||||
"2048": 2048,
|
||||
"4096": 4096,
|
||||
"8192": 8192,
|
||||
"16384": 16384
|
||||
} as const
|
||||
export type BufferSize = typeof BufferSize[keyof typeof BufferSize]
|
||||
|
||||
// Defaults
|
||||
|
||||
|
||||
export const DefaultVoiceChangerClientSetting: VoiceChangerClientSetting = {
|
||||
audioInput: null,
|
||||
mmvcServerUrl: "",
|
||||
protocol: "sio",
|
||||
sampleRate: 48000,
|
||||
sendingSampleRate: 48000,
|
||||
bufferSize: 1024,
|
||||
inputChunkNum: 48,
|
||||
//
|
||||
speakers: [
|
||||
{
|
||||
"id": 0,
|
||||
@ -252,21 +151,108 @@ export const DefaultVoiceChangerClientSetting: VoiceChangerClientSetting = {
|
||||
}
|
||||
],
|
||||
correspondences: [],
|
||||
echoCancel: true,
|
||||
noiseSuppression: true,
|
||||
noiseSuppression2: false,
|
||||
voiceChangerMode: "realtime",
|
||||
downSamplingMode: "average",
|
||||
inputGain: 1.0,
|
||||
outputGain: 1.0
|
||||
}
|
||||
|
||||
|
||||
///////////////////////
|
||||
// Workletセッティング
|
||||
///////////////////////
|
||||
|
||||
export type WorkletSetting = {
|
||||
numTrancateTreshold: number,
|
||||
volTrancateThreshold: number,
|
||||
volTrancateLength: number
|
||||
}
|
||||
export const DefaultWorkletSetting: WorkletSetting = {
|
||||
numTrancateTreshold: 188,
|
||||
volTrancateThreshold: 0.0005,
|
||||
volTrancateLength: 32
|
||||
}
|
||||
///////////////////////
|
||||
// Audio Streamerセッティング
|
||||
///////////////////////
|
||||
export const Protocol = {
|
||||
"sio": "sio",
|
||||
"rest": "rest",
|
||||
} as const
|
||||
export type Protocol = typeof Protocol[keyof typeof Protocol]
|
||||
|
||||
export const SendingSampleRate = {
|
||||
"48000": 48000,
|
||||
"24000": 24000
|
||||
} as const
|
||||
export type SendingSampleRate = typeof SendingSampleRate[keyof typeof SendingSampleRate]
|
||||
|
||||
export const DownSamplingMode = {
|
||||
"decimate": "decimate",
|
||||
"average": "average"
|
||||
} as const
|
||||
export type DownSamplingMode = typeof DownSamplingMode[keyof typeof DownSamplingMode]
|
||||
|
||||
|
||||
export type AudioStreamerSetting = {
|
||||
serverUrl: string,
|
||||
protocol: Protocol,
|
||||
sendingSampleRate: SendingSampleRate,
|
||||
inputChunkNum: number,
|
||||
downSamplingMode: DownSamplingMode,
|
||||
}
|
||||
export const DefaultAudioStreamerSetting: AudioStreamerSetting = {
|
||||
serverUrl: "",
|
||||
protocol: "sio",
|
||||
sendingSampleRate: 48000,
|
||||
inputChunkNum: 48,
|
||||
downSamplingMode: "average"
|
||||
}
|
||||
|
||||
///////////////////////
|
||||
// クライアントセッティング
|
||||
///////////////////////
|
||||
export const SampleRate = {
|
||||
"48000": 48000,
|
||||
} as const
|
||||
export type SampleRate = typeof SampleRate[keyof typeof SampleRate]
|
||||
|
||||
export const BufferSize = {
|
||||
"256": 256,
|
||||
"512": 512,
|
||||
"1024": 1024,
|
||||
"2048": 2048,
|
||||
"4096": 4096,
|
||||
"8192": 8192,
|
||||
"16384": 16384
|
||||
} as const
|
||||
export type BufferSize = typeof BufferSize[keyof typeof BufferSize]
|
||||
|
||||
export type VoiceChangerClientSetting = {
|
||||
audioInput: string | MediaStream | null,
|
||||
sampleRate: SampleRate, // 48000Hz
|
||||
bufferSize: BufferSize, // 256, 512, 1024, 2048, 4096, 8192, 16384 (for mic stream)
|
||||
echoCancel: boolean,
|
||||
noiseSuppression: boolean,
|
||||
noiseSuppression2: boolean
|
||||
|
||||
|
||||
inputGain: number
|
||||
outputGain: number
|
||||
}
|
||||
|
||||
export const DefaultVoiceChangerClientSetting: VoiceChangerClientSetting = {
|
||||
audioInput: null,
|
||||
sampleRate: 48000,
|
||||
bufferSize: 1024,
|
||||
|
||||
echoCancel: true,
|
||||
noiseSuppression: true,
|
||||
noiseSuppression2: false,
|
||||
inputGain: 1.0,
|
||||
outputGain: 1.0
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////
|
||||
// Exceptions
|
||||
////////////////////////////////////
|
||||
export const VOICE_CHANGER_CLIENT_EXCEPTION = {
|
||||
ERR_SIO_CONNECT_FAILED: "ERR_SIO_CONNECT_FAILED",
|
||||
ERR_SIO_INVALID_RESPONSE: "ERR_SIO_INVALID_RESPONSE",
|
||||
@ -284,6 +270,7 @@ export const INDEXEDDB_DB_APP_NAME = "INDEXEDDB_KEY_VOICE_CHANGER"
|
||||
export const INDEXEDDB_DB_NAME = "INDEXEDDB_KEY_VOICE_CHANGER_DB"
|
||||
export const INDEXEDDB_KEY_CLIENT = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_CLIENT"
|
||||
export const INDEXEDDB_KEY_SERVER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_SERVER"
|
||||
export const INDEXEDDB_KEY_STREAMER = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_STREAMER"
|
||||
export const INDEXEDDB_KEY_MODEL_DATA = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_MODEL_DATA"
|
||||
export const INDEXEDDB_KEY_WORKLET = "INDEXEDDB_KEY_VOICE_CHANGER_LIB_WORKLET"
|
||||
|
||||
|
||||
77
client/lib/src/hooks/useAudioStreamerSetting.ts
Normal file
77
client/lib/src/hooks/useAudioStreamerSetting.ts
Normal file
@ -0,0 +1,77 @@
|
||||
import { useState, useMemo, useEffect } from "react"
|
||||
|
||||
import { INDEXEDDB_KEY_CLIENT, INDEXEDDB_KEY_STREAMER, AudioStreamerSetting, DefaultAudioStreamerSetting } from "../const"
|
||||
import { VoiceChangerClient } from "../VoiceChangerClient"
|
||||
import { useIndexedDB } from "./useIndexedDB"
|
||||
|
||||
export type UseAudioStreamerSettingProps = {
|
||||
voiceChangerClient: VoiceChangerClient | null
|
||||
}
|
||||
|
||||
export type AudioStreamerSettingState = {
|
||||
audioStreamerSetting: AudioStreamerSetting;
|
||||
clearSetting: () => Promise<void>
|
||||
setSetting: (setting: AudioStreamerSetting) => void
|
||||
|
||||
}
|
||||
|
||||
export const useAudioStreamerSetting = (props: UseAudioStreamerSettingProps): AudioStreamerSettingState => {
|
||||
const [audioStreamerSetting, _setAudioStreamerSetting] = useState<AudioStreamerSetting>(DefaultAudioStreamerSetting)
|
||||
const { setItem, getItem, removeItem } = useIndexedDB()
|
||||
|
||||
// 初期化 その１ DBから取得
|
||||
useEffect(() => {
|
||||
const loadCache = async () => {
|
||||
const setting = await getItem(INDEXEDDB_KEY_STREAMER) as AudioStreamerSetting
|
||||
if (setting) {
|
||||
_setAudioStreamerSetting(setting)
|
||||
}
|
||||
}
|
||||
loadCache()
|
||||
}, [])
|
||||
|
||||
// 初期化 その２ クライアントに設定
|
||||
useEffect(() => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setServerUrl(audioStreamerSetting.serverUrl)
|
||||
props.voiceChangerClient.updateAudioStreamerSetting(audioStreamerSetting)
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
|
||||
|
||||
const clearSetting = async () => {
|
||||
await removeItem(INDEXEDDB_KEY_STREAMER)
|
||||
}
|
||||
|
||||
//////////////
|
||||
// 設定
|
||||
/////////////
|
||||
|
||||
|
||||
// const setServerUrl = useMemo(() => {
|
||||
// return (url: string) => {
|
||||
// if (!props.voiceChangerClient) return
|
||||
// props.voiceChangerClient.setServerUrl(url, true)
|
||||
// settingRef.current.mmvcServerUrl = url
|
||||
// setSetting({ ...settingRef.current })
|
||||
// }
|
||||
// }, [props.voiceChangerClient])
|
||||
|
||||
const setSetting = useMemo(() => {
|
||||
return (setting: AudioStreamerSetting) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
_setAudioStreamerSetting(setting)
|
||||
setItem(INDEXEDDB_KEY_CLIENT, setting)
|
||||
props.voiceChangerClient.updateAudioStreamerSetting(setting)
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
|
||||
console.log("AUDIO STREAMER SETTING", audioStreamerSetting)
|
||||
return {
|
||||
audioStreamerSetting,
|
||||
clearSetting,
|
||||
setSetting,
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,5 +1,6 @@
|
||||
import { useEffect, useMemo, useRef, useState } from "react"
|
||||
import { VoiceChangerClient } from "../VoiceChangerClient"
|
||||
import { AudioStreamerSettingState, useAudioStreamerSetting } from "./useAudioStreamerSetting"
|
||||
import { ClientSettingState, useClientSetting } from "./useClientSetting"
|
||||
import { ServerSettingState, useServerSetting } from "./useServerSetting"
|
||||
import { useWorkletSetting, WorkletSettingState } from "./useWorkletSetting"
|
||||
@ -13,6 +14,7 @@ export type ClientState = {
|
||||
// 各種設定I/Fへの参照
|
||||
workletSetting: WorkletSettingState
|
||||
clientSetting: ClientSettingState
|
||||
streamerSetting: AudioStreamerSettingState
|
||||
serverSetting: ServerSettingState
|
||||
|
||||
// モニタリングデータ
|
||||
@ -45,6 +47,7 @@ export const useClient = (props: UseClientProps): ClientState => {
|
||||
|
||||
// (1-2) 各種設定I/F
|
||||
const clientSetting = useClientSetting({ voiceChangerClient, audioContext: props.audioContext })
|
||||
const streamerSetting = useAudioStreamerSetting({ voiceChangerClient })
|
||||
const workletSetting = useWorkletSetting({ voiceChangerClient })
|
||||
const serverSetting = useServerSetting({ voiceChangerClient })
|
||||
|
||||
@ -118,10 +121,12 @@ export const useClient = (props: UseClientProps): ClientState => {
|
||||
await workletSetting.clearSetting()
|
||||
await serverSetting.clearSetting()
|
||||
}
|
||||
console.log("AUDIO STREAMER SETTING USE CLIENT", clientSetting, streamerSetting)
|
||||
|
||||
return {
|
||||
// 各種設定I/Fへの参照
|
||||
clientSetting,
|
||||
streamerSetting,
|
||||
workletSetting,
|
||||
serverSetting,
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { useState, useMemo, useRef, useEffect } from "react"
|
||||
import { useState, useMemo, useEffect } from "react"
|
||||
|
||||
import { VoiceChangerClientSetting, Protocol, BufferSize, VoiceChangerMode, SampleRate, Speaker, DefaultVoiceChangerClientSetting, INDEXEDDB_KEY_CLIENT, Correspondence, DownSamplingMode, SendingSampleRate } from "../const"
|
||||
import { VoiceChangerClientSetting, DefaultVoiceChangerClientSetting, INDEXEDDB_KEY_CLIENT } from "../const"
|
||||
import { VoiceChangerClient } from "../VoiceChangerClient"
|
||||
import { useIndexedDB } from "./useIndexedDB"
|
||||
|
||||
@ -10,79 +10,48 @@ export type UseClientSettingProps = {
|
||||
}
|
||||
|
||||
export type ClientSettingState = {
|
||||
setting: VoiceChangerClientSetting;
|
||||
clientSetting: VoiceChangerClientSetting;
|
||||
clearSetting: () => Promise<void>
|
||||
setServerUrl: (url: string) => void;
|
||||
setProtocol: (proto: Protocol) => void;
|
||||
setAudioInput: (audioInput: string | MediaStream | null) => Promise<void>
|
||||
setBufferSize: (bufferSize: BufferSize) => Promise<void>
|
||||
setEchoCancel: (voiceFocus: boolean) => Promise<void>
|
||||
setNoiseSuppression: (voiceFocus: boolean) => Promise<void>
|
||||
setNoiseSuppression2: (voiceFocus: boolean) => Promise<void>
|
||||
setInputChunkNum: (num: number) => void;
|
||||
setVoiceChangerMode: (mode: VoiceChangerMode) => void
|
||||
setDownSamplingMode: (mode: DownSamplingMode) => void
|
||||
setSendingSampleRate: (val: SendingSampleRate) => void
|
||||
setSampleRate: (num: SampleRate) => void
|
||||
setSpeakers: (speakers: Speaker[]) => void
|
||||
setCorrespondences: (file: File | null) => Promise<void>
|
||||
setInputGain: (val: number) => void
|
||||
setOutputGain: (val: number) => void
|
||||
updateClientSetting: (clientSetting: VoiceChangerClientSetting) => void
|
||||
|
||||
start: () => Promise<void>
|
||||
stop: () => Promise<void>
|
||||
reloadClientSetting: () => Promise<void>
|
||||
}
|
||||
|
||||
export const useClientSetting = (props: UseClientSettingProps): ClientSettingState => {
|
||||
const settingRef = useRef<VoiceChangerClientSetting>(DefaultVoiceChangerClientSetting)
|
||||
const [setting, _setSetting] = useState<VoiceChangerClientSetting>(settingRef.current)
|
||||
const [clientSetting, setClientSetting] = useState<VoiceChangerClientSetting>(DefaultVoiceChangerClientSetting)
|
||||
const { setItem, getItem, removeItem } = useIndexedDB()
|
||||
|
||||
// 初期化 その１ DBから取得
|
||||
useEffect(() => {
|
||||
const loadCache = async () => {
|
||||
const setting = await getItem(INDEXEDDB_KEY_CLIENT)
|
||||
if (!setting) {
|
||||
// デフォルト設定
|
||||
console.log("No Chache",)
|
||||
const params = new URLSearchParams(location.search);
|
||||
const colab = params.get("colab")
|
||||
if (colab == "true") {
|
||||
settingRef.current.protocol = "rest"
|
||||
settingRef.current.inputChunkNum = 64
|
||||
} else {
|
||||
settingRef.current.protocol = "sio"
|
||||
settingRef.current.inputChunkNum = 32
|
||||
}
|
||||
} else {
|
||||
settingRef.current = setting as VoiceChangerClientSetting
|
||||
const setting = await getItem(INDEXEDDB_KEY_CLIENT) as VoiceChangerClientSetting
|
||||
console.log("[ClientSetting] Load Setting from db", setting)
|
||||
if (setting.audioInput == "null") {
|
||||
setting.audioInput = null
|
||||
}
|
||||
if (setting) {
|
||||
setClientSetting({ ...setting })
|
||||
}
|
||||
_setSetting({ ...settingRef.current })
|
||||
}
|
||||
|
||||
loadCache()
|
||||
}, [])
|
||||
// 初期化 その２ クライアントに設定
|
||||
useEffect(() => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setServerUrl(settingRef.current.mmvcServerUrl)
|
||||
props.voiceChangerClient.setInputChunkNum(settingRef.current.inputChunkNum)
|
||||
props.voiceChangerClient.setProtocol(settingRef.current.protocol)
|
||||
props.voiceChangerClient.setVoiceChangerMode(settingRef.current.voiceChangerMode)
|
||||
props.voiceChangerClient.setInputGain(settingRef.current.inputGain)
|
||||
|
||||
// Input, bufferSize, VoiceFocus Disableは_setInputで設定
|
||||
_setInput()
|
||||
props.voiceChangerClient.updateClientSetting(clientSetting)
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
|
||||
const setSetting = async (setting: VoiceChangerClientSetting) => {
|
||||
const storeSetting = async (setting: VoiceChangerClientSetting) => {
|
||||
const storeData = { ...setting }
|
||||
if (typeof storeData.audioInput != "string") {
|
||||
storeData.audioInput = null
|
||||
}
|
||||
setItem(INDEXEDDB_KEY_CLIENT, storeData)
|
||||
_setSetting(setting)
|
||||
setClientSetting(setting)
|
||||
}
|
||||
|
||||
const clearSetting = async () => {
|
||||
@ -92,177 +61,21 @@ export const useClientSetting = (props: UseClientSettingProps): ClientSettingSta
|
||||
//////////////
|
||||
// 設定
|
||||
/////////////
|
||||
const updateClientSetting = useMemo(() => {
|
||||
return (clientSetting: VoiceChangerClientSetting) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
// storeSetting(clientSetting)
|
||||
// props.voiceChangerClient.updateClientSetting(clientSetting)
|
||||
}
|
||||
}, [props.voiceChangerClient, clientSetting])
|
||||
|
||||
const setServerUrl = useMemo(() => {
|
||||
return (url: string) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setServerUrl(url, true)
|
||||
settingRef.current.mmvcServerUrl = url
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setProtocol = useMemo(() => {
|
||||
return (proto: Protocol) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setProtocol(proto)
|
||||
settingRef.current.protocol = proto
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const _setInput = async () => {
|
||||
if (!props.voiceChangerClient) return
|
||||
if (!settingRef.current.audioInput || settingRef.current.audioInput == "none") {
|
||||
await props.voiceChangerClient.setup(null, settingRef.current.bufferSize, settingRef.current.echoCancel, settingRef.current.noiseSuppression, settingRef.current.noiseSuppression2)
|
||||
} else {
|
||||
// console.log("[useClient] setup!(2)", settingRef.current.audioInput)
|
||||
await props.voiceChangerClient.setup(settingRef.current.audioInput, settingRef.current.bufferSize, settingRef.current.echoCancel, settingRef.current.noiseSuppression, settingRef.current.noiseSuppression2)
|
||||
}
|
||||
}
|
||||
|
||||
const setAudioInput = useMemo(() => {
|
||||
return async (audioInput: string | MediaStream | null) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
settingRef.current.audioInput = audioInput
|
||||
await _setInput()
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setBufferSize = useMemo(() => {
|
||||
return async (bufferSize: BufferSize) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
settingRef.current.bufferSize = bufferSize
|
||||
await _setInput()
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setEchoCancel = useMemo(() => {
|
||||
return async (val: boolean) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
settingRef.current.echoCancel = val
|
||||
await _setInput()
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setNoiseSuppression = useMemo(() => {
|
||||
return async (val: boolean) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
settingRef.current.noiseSuppression = val
|
||||
await _setInput()
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setNoiseSuppression2 = useMemo(() => {
|
||||
return async (val: boolean) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
settingRef.current.noiseSuppression2 = val
|
||||
await _setInput()
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setInputChunkNum = useMemo(() => {
|
||||
return (num: number) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setInputChunkNum(num)
|
||||
settingRef.current.inputChunkNum = num
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setVoiceChangerMode = useMemo(() => {
|
||||
return (mode: VoiceChangerMode) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setVoiceChangerMode(mode)
|
||||
settingRef.current.voiceChangerMode = mode
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setDownSamplingMode = useMemo(() => {
|
||||
return (mode: DownSamplingMode) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setDownSamplingMode(mode)
|
||||
settingRef.current.downSamplingMode = mode
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setSendingSampleRate = useMemo(() => {
|
||||
return (val: SendingSampleRate) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setSendingSampleRate(val)
|
||||
settingRef.current.sendingSampleRate = val
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
|
||||
|
||||
const setSampleRate = useMemo(() => {
|
||||
return (num: SampleRate) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
//props.voiceChangerClient.setSampleRate(num) // Not Implemented
|
||||
settingRef.current.sampleRate = num
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setSpeakers = useMemo(() => {
|
||||
return (speakers: Speaker[]) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
settingRef.current.speakers = speakers
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setCorrespondences = useMemo(() => {
|
||||
return async (file: File | null) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
if (!file) {
|
||||
settingRef.current.correspondences = []
|
||||
} else {
|
||||
const correspondenceText = await file.text()
|
||||
const cors = correspondenceText.split("\n").map(line => {
|
||||
const items = line.split("|")
|
||||
if (items.length != 3) {
|
||||
console.warn("Invalid Correspondence Line:", line)
|
||||
return null
|
||||
} else {
|
||||
const cor: Correspondence = {
|
||||
sid: Number(items[0]),
|
||||
correspondence: Number(items[1]),
|
||||
dirname: items[2]
|
||||
}
|
||||
return cor
|
||||
}
|
||||
}).filter(x => { return x != null }) as Correspondence[]
|
||||
settingRef.current.correspondences = cors
|
||||
}
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
const setInputGain = useMemo(() => {
|
||||
return (val: number) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setInputGain(val)
|
||||
settingRef.current.inputGain = val
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
const setOutputGain = useMemo(() => {
|
||||
return (val: number) => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setOutputGain(val)
|
||||
settingRef.current.outputGain = val
|
||||
setSetting({ ...settingRef.current })
|
||||
}
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
//////////////
|
||||
// 操作
|
||||
@ -271,10 +84,10 @@ export const useClientSetting = (props: UseClientSettingProps): ClientSettingSta
|
||||
const start = useMemo(() => {
|
||||
return async () => {
|
||||
if (!props.voiceChangerClient) return
|
||||
props.voiceChangerClient.setServerUrl(setting.mmvcServerUrl, true)
|
||||
// props.voiceChangerClient.setServerUrl(setting.mmvcServerUrl, true)
|
||||
props.voiceChangerClient.start()
|
||||
}
|
||||
}, [setting.mmvcServerUrl, props.voiceChangerClient])
|
||||
}, [props.voiceChangerClient])
|
||||
// (2) stop
|
||||
const stop = useMemo(() => {
|
||||
return async () => {
|
||||
@ -290,24 +103,10 @@ export const useClientSetting = (props: UseClientSettingProps): ClientSettingSta
|
||||
}, [props.voiceChangerClient])
|
||||
|
||||
return {
|
||||
setting,
|
||||
clientSetting,
|
||||
clearSetting,
|
||||
setServerUrl,
|
||||
setProtocol,
|
||||
setAudioInput,
|
||||
setBufferSize,
|
||||
setEchoCancel,
|
||||
setNoiseSuppression,
|
||||
setNoiseSuppression2,
|
||||
setInputChunkNum,
|
||||
setVoiceChangerMode,
|
||||
setDownSamplingMode,
|
||||
setSendingSampleRate,
|
||||
setSampleRate,
|
||||
setSpeakers,
|
||||
setCorrespondences,
|
||||
setInputGain,
|
||||
setOutputGain,
|
||||
updateClientSetting,
|
||||
|
||||
start,
|
||||
stop,
|
||||
|
||||
@ -4,17 +4,16 @@
|
||||
"declaration": true,
|
||||
"outDir": "./dist",
|
||||
|
||||
|
||||
/* ファイル名の大文字小文字を区別 */
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
|
||||
/* 型チェック関係のオプション */
|
||||
"strict": true,
|
||||
"noImplicitAny": true,
|
||||
"strictNullChecks": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noImplicitReturns": true,
|
||||
// /* 型チェック関係のオプション */
|
||||
// "strict": true,
|
||||
// "noImplicitAny": true,
|
||||
// "strictNullChecks": true,
|
||||
// "noUnusedLocals": true,
|
||||
// "noUnusedParameters": true,
|
||||
// "noImplicitReturns": true,
|
||||
|
||||
/* Module解決方法 */
|
||||
"moduleResolution": "node",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user