mirror of
https://github.com/w-okada/voice-changer.git
synced 2025-02-02 16:23:58 +03:00
tuning
This commit is contained in:
wataru
parent
e74752f548
commit
1363b1d07f
@ -157,6 +157,11 @@ body {
|
||||
.body-item-text {
|
||||
color: rgb(30, 30, 30);
|
||||
}
|
||||
|
||||
.body-item-input {
|
||||
width:90%;
|
||||
}
|
||||
|
||||
.body-button-container {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
@ -165,6 +170,22 @@ body {
|
||||
border: solid 1px #333;
|
||||
border-radius: 2px;
|
||||
padding: 2px;
|
||||
cursor:pointer;
|
||||
}
|
||||
.body-button-active {
|
||||
user-select: none;
|
||||
border: solid 1px #333;
|
||||
border-radius: 2px;
|
||||
padding: 2px;
|
||||
background:#ada;
|
||||
}
|
||||
.body-button-stanby {
|
||||
user-select: none;
|
||||
border: solid 1px #333;
|
||||
border-radius: 2px;
|
||||
padding: 2px;
|
||||
background:#aba;
|
||||
cursor:pointer;
|
||||
}
|
||||
}
|
||||
.body-select-container {
|
||||
|
||||
@ -1,18 +1,17 @@
|
||||
import * as React from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import "./css/App.css"
|
||||
import { useEffect, useMemo, useRef } from "react";
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { VoiceChnagerClient } from "@dannadori/voice-changer-client-js"
|
||||
import { useMicrophoneOptions } from "./options_microphone";
|
||||
const container = document.getElementById("app")!;
|
||||
const root = createRoot(container);
|
||||
|
||||
const App = () => {
|
||||
const { component: microphoneSettingComponent, options: microphonOptions } = useMicrophoneOptions()
|
||||
const { component: microphoneSettingComponent, options: microphoneOptions, params: microphoneParams, isStarted } = useMicrophoneOptions()
|
||||
|
||||
const voiceChnagerClientRef = useRef<VoiceChnagerClient | null>(null)
|
||||
|
||||
console.log(microphonOptions)
|
||||
const voiceChangerClientRef = useRef<VoiceChnagerClient | null>(null)
|
||||
const [clientInitialized, setClientInitialized] = useState<boolean>(false)
|
||||
|
||||
const onClearSettingClicked = async () => {
|
||||
//@ts-ignore
|
||||
@ -23,32 +22,101 @@ const App = () => {
|
||||
location.reload()
|
||||
}
|
||||
|
||||
|
||||
useEffect(() => {
|
||||
if (microphonOptions.audioInputDeviceId.length == 0) {
|
||||
return
|
||||
}
|
||||
const setAudio = async () => {
|
||||
const initialized = async () => {
|
||||
const ctx = new AudioContext()
|
||||
|
||||
if (voiceChnagerClientRef.current) {
|
||||
|
||||
}
|
||||
voiceChnagerClientRef.current = new VoiceChnagerClient(ctx, true, {
|
||||
voiceChangerClientRef.current = new VoiceChnagerClient(ctx, true, {
|
||||
notifySendBufferingTime: (val: number) => { console.log(`buf:${val}`) },
|
||||
notifyResponseTime: (val: number) => { console.log(`res:${val}`) },
|
||||
notifyException: (mes: string) => { console.log(`error:${mes}`) }
|
||||
})
|
||||
await voiceChnagerClientRef.current.isInitialized()
|
||||
|
||||
voiceChnagerClientRef.current.setServerUrl("https://192.168.0.3:18888/test", "sio")
|
||||
voiceChnagerClientRef.current.setup(microphonOptions.audioInputDeviceId, 1024)
|
||||
notifyException: (mes: string) => {
|
||||
if (mes.length > 0) {
|
||||
console.log(`error:${mes}`)
|
||||
}
|
||||
}
|
||||
}, { notifyVolume: (vol: number) => { } })
|
||||
await voiceChangerClientRef.current.isInitialized()
|
||||
setClientInitialized(true)
|
||||
|
||||
const audio = document.getElementById("audio-output") as HTMLAudioElement
|
||||
audio.srcObject = voiceChnagerClientRef.current.stream
|
||||
audio.srcObject = voiceChangerClientRef.current.stream
|
||||
audio.play()
|
||||
}
|
||||
setAudio()
|
||||
}, [microphonOptions.audioInputDeviceId])
|
||||
initialized()
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
console.log("START!!!", isStarted)
|
||||
const start = async () => {
|
||||
if (!voiceChangerClientRef.current || !clientInitialized) {
|
||||
console.log("client is not initialized")
|
||||
return
|
||||
}
|
||||
// if (!microphoneOptions.audioInputDeviceId || microphoneOptions.audioInputDeviceId.length == 0) {
|
||||
// console.log("audioInputDeviceId is not initialized")
|
||||
// return
|
||||
// }
|
||||
// await voiceChangerClientRef.current.setup(microphoneOptions.audioInputDeviceId!, microphoneOptions.bufferSize)
|
||||
voiceChangerClientRef.current.setServerUrl(microphoneOptions.mmvcServerUrl, microphoneOptions.protocol, true)
|
||||
voiceChangerClientRef.current.start()
|
||||
}
|
||||
const stop = async () => {
|
||||
if (!voiceChangerClientRef.current || !clientInitialized) {
|
||||
console.log("client is not initialized")
|
||||
return
|
||||
}
|
||||
voiceChangerClientRef.current.stop()
|
||||
}
|
||||
if (isStarted) {
|
||||
start()
|
||||
} else {
|
||||
stop()
|
||||
}
|
||||
}, [isStarted])
|
||||
|
||||
// useEffect(() => {
|
||||
// if (!voiceChangerClientRef.current || !clientInitialized) {
|
||||
// console.log("client is not initialized")
|
||||
// return
|
||||
// }
|
||||
// voiceChangerClientRef.current.setServerUrl(microphoneOptions.mmvcServerUrl, microphoneOptions.protocol, false)
|
||||
// }, [microphoneOptions.mmvcServerUrl, microphoneOptions.protocol])
|
||||
|
||||
useEffect(() => {
|
||||
const changeInput = async () => {
|
||||
if (!voiceChangerClientRef.current || !clientInitialized) {
|
||||
console.log("client is not initialized")
|
||||
return
|
||||
}
|
||||
await voiceChangerClientRef.current.setup(microphoneOptions.audioInputDeviceId!, microphoneOptions.bufferSize, microphoneOptions.forceVfDisable)
|
||||
}
|
||||
changeInput()
|
||||
}, [microphoneOptions.audioInputDeviceId!, microphoneOptions.bufferSize, microphoneOptions.forceVfDisable])
|
||||
|
||||
|
||||
useEffect(() => {
|
||||
if (!voiceChangerClientRef.current || !clientInitialized) {
|
||||
console.log("client is not initialized")
|
||||
return
|
||||
}
|
||||
voiceChangerClientRef.current.setInputChunkNum(microphoneOptions.inputChunkNum)
|
||||
}, [microphoneOptions.inputChunkNum])
|
||||
|
||||
useEffect(() => {
|
||||
if (!voiceChangerClientRef.current || !clientInitialized) {
|
||||
console.log("client is not initialized")
|
||||
return
|
||||
}
|
||||
voiceChangerClientRef.current.setVoiceChangerMode(microphoneOptions.voiceChangerMode)
|
||||
}, [microphoneOptions.voiceChangerMode])
|
||||
|
||||
useEffect(() => {
|
||||
if (!voiceChangerClientRef.current || !clientInitialized) {
|
||||
console.log("client is not initialized")
|
||||
return
|
||||
}
|
||||
voiceChangerClientRef.current.setRequestParams(microphoneParams)
|
||||
}, [microphoneParams])
|
||||
|
||||
const clearRow = useMemo(() => {
|
||||
return (
|
||||
|
||||
@ -1,42 +1,7 @@
|
||||
import * as React from "react";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { CHROME_EXTENSION } from "./const";
|
||||
import { Speaker, VoiceChangerMode, DefaultSpeakders, SampleRate, BufferSize } from "@dannadori/voice-changer-client-js"
|
||||
|
||||
export type MicrophoneOptionsState = {
|
||||
audioInputDeviceId: string,
|
||||
mmvcServerUrl: string,
|
||||
sampleRate: number,
|
||||
bufferSize: number,
|
||||
chunkSize: number,
|
||||
speakers: Speaker[],
|
||||
srcId: number,
|
||||
dstId: number,
|
||||
vfEnabled: boolean,
|
||||
voiceChangerMode: VoiceChangerMode,
|
||||
gpu: number,
|
||||
|
||||
crossFadeLowerValue: number,
|
||||
crossFadeOffsetRate: number,
|
||||
crossFadeEndRate: number,
|
||||
}
|
||||
const InitMicrophoneOptionsState = {
|
||||
audioInputDeviceId: "",
|
||||
mmvcServerUrl: "https://localhost:5543/test",
|
||||
sampleRate: 48000,
|
||||
bufferSize: 1024,
|
||||
chunkSize: 24,
|
||||
speakers: DefaultSpeakders,
|
||||
srcId: 107,
|
||||
dstId: 100,
|
||||
vfEnabled: true,
|
||||
voiceChangerMode: VoiceChangerMode.realtime,
|
||||
gpu: 0,
|
||||
|
||||
crossFadeLowerValue: 0.1,
|
||||
crossFadeOffsetRate: 0.3,
|
||||
crossFadeEndRate: 0.6,
|
||||
} as const
|
||||
import { DefaultVoiceChangerRequestParamas, VoiceChangerOptions, VoiceChangerRequestParamas, DefaultVoiceChangerOptions, SampleRate, BufferSize, VoiceChangerMode, Protocol } from "@dannadori/voice-changer-client-js"
|
||||
|
||||
|
||||
const reloadDevices = async () => {
|
||||
@ -46,13 +11,24 @@ const reloadDevices = async () => {
|
||||
console.warn("Enumerate device error::", e)
|
||||
}
|
||||
const mediaDeviceInfos = await navigator.mediaDevices.enumerateDevices();
|
||||
return mediaDeviceInfos.filter(x => { return x.kind == "audioinput" })
|
||||
|
||||
const audioInputs = mediaDeviceInfos.filter(x => { return x.kind == "audioinput" })
|
||||
audioInputs.push({
|
||||
deviceId: "none",
|
||||
groupId: "none",
|
||||
kind: "audioinput",
|
||||
label: "none",
|
||||
toJSON: () => { }
|
||||
})
|
||||
return audioInputs
|
||||
}
|
||||
|
||||
|
||||
export type MicrophoneOptionsComponent = {
|
||||
component: JSX.Element,
|
||||
options: MicrophoneOptionsState
|
||||
options: VoiceChangerOptions,
|
||||
params: VoiceChangerRequestParamas
|
||||
isStarted: boolean
|
||||
}
|
||||
|
||||
export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
@ -61,7 +37,10 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
const [editSpeakerTargetId, setEditSpeakerTargetId] = useState<number>(0)
|
||||
const [editSpeakerTargetName, setEditSpeakerTargetName] = useState<string>("")
|
||||
|
||||
const [options, setOptions] = useState<MicrophoneOptionsState>(InitMicrophoneOptionsState)
|
||||
// const [options, setOptions] = useState<MicrophoneOptionsState>(InitMicrophoneOptionsState)
|
||||
const [params, setParams] = useState<VoiceChangerRequestParamas>(DefaultVoiceChangerRequestParamas)
|
||||
const [options, setOptions] = useState<VoiceChangerOptions>(DefaultVoiceChangerOptions)
|
||||
const [isStarted, setIsStarted] = useState<boolean>(false)
|
||||
|
||||
useEffect(() => {
|
||||
const initialize = async () => {
|
||||
@ -90,6 +69,32 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
}, [options]) // loadより前に持ってくるとstorage内が初期化されるのでだめかも。（要検証）
|
||||
|
||||
|
||||
const startButtonRow = useMemo(() => {
|
||||
const onStartClicked = () => {
|
||||
setIsStarted(true)
|
||||
}
|
||||
const onStopClicked = () => {
|
||||
setIsStarted(false)
|
||||
}
|
||||
const startClassName = isStarted ? "body-button-active" : "body-button-stanby"
|
||||
const stopClassName = isStarted ? "body-button-stanby" : "body-button-active"
|
||||
console.log("ClassName", startClassName, stopClassName)
|
||||
|
||||
return (
|
||||
<div className="body-row split-3-3-4 left-padding-1">
|
||||
<div className="body-item-title">Start</div>
|
||||
<div className="body-button-container">
|
||||
<div onClick={onStartClicked} className={startClassName}>start</div>
|
||||
<div onClick={onStopClicked} className={stopClassName}>stop</div>
|
||||
</div>
|
||||
<div className="body-input-container">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
)
|
||||
}, [isStarted])
|
||||
|
||||
|
||||
const setAudioInputDeviceId = async (deviceId: string) => {
|
||||
setOptions({ ...options, audioInputDeviceId: deviceId })
|
||||
}
|
||||
@ -98,6 +103,10 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
const input = document.getElementById("mmvc-server-url") as HTMLInputElement
|
||||
setOptions({ ...options, mmvcServerUrl: input.value })
|
||||
}
|
||||
const onProtocolChanged = async (val: Protocol) => {
|
||||
setOptions({ ...options, protocol: val })
|
||||
}
|
||||
|
||||
const onSampleRateChanged = async (val: SampleRate) => {
|
||||
setOptions({ ...options, sampleRate: val })
|
||||
}
|
||||
@ -105,13 +114,13 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
setOptions({ ...options, bufferSize: val })
|
||||
}
|
||||
const onChunkSizeChanged = async (val: number) => {
|
||||
setOptions({ ...options, chunkSize: val })
|
||||
setOptions({ ...options, inputChunkNum: val })
|
||||
}
|
||||
const onSrcIdChanged = async (val: number) => {
|
||||
setOptions({ ...options, srcId: val })
|
||||
setParams({ ...params, srcId: val })
|
||||
}
|
||||
const onDstIdChanged = async (val: number) => {
|
||||
setOptions({ ...options, dstId: val })
|
||||
setParams({ ...params, dstId: val })
|
||||
}
|
||||
const onSetSpeakerMappingClicked = async () => {
|
||||
const targetId = editSpeakerTargetId
|
||||
@ -137,22 +146,22 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
}
|
||||
|
||||
const onVfEnabledChange = async (val: boolean) => {
|
||||
setOptions({ ...options, vfEnabled: val })
|
||||
setOptions({ ...options, forceVfDisable: val })
|
||||
}
|
||||
const onVoiceChangeModeChanged = async (val: VoiceChangerMode) => {
|
||||
setOptions({ ...options, voiceChangerMode: val })
|
||||
}
|
||||
const onGpuChanged = async (val: number) => {
|
||||
setOptions({ ...options, gpu: val })
|
||||
setParams({ ...params, gpu: val })
|
||||
}
|
||||
const onCrossFadeLowerValueChanged = async (val: number) => {
|
||||
setOptions({ ...options, crossFadeLowerValue: val })
|
||||
setParams({ ...params, crossFadeLowerValue: val })
|
||||
}
|
||||
const onCrossFadeOffsetRateChanged = async (val: number) => {
|
||||
setOptions({ ...options, crossFadeOffsetRate: val })
|
||||
setParams({ ...params, crossFadeOffsetRate: val })
|
||||
}
|
||||
const onCrossFadeEndRateChanged = async (val: number) => {
|
||||
setOptions({ ...options, crossFadeEndRate: val })
|
||||
setParams({ ...params, crossFadeEndRate: val })
|
||||
}
|
||||
|
||||
const settings = useMemo(() => {
|
||||
@ -161,26 +170,44 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
<div className="body-row left-padding-1">
|
||||
<div className="body-section-title">Virtual Microphone</div>
|
||||
</div>
|
||||
{startButtonRow}
|
||||
|
||||
<div className="body-row split-3-3-4 left-padding-1">
|
||||
<div className="body-item-title">MMVC Server</div>
|
||||
<div className="body-input-container">
|
||||
<input type="text" defaultValue={options.mmvcServerUrl} id="mmvc-server-url" className="body-item-input" />
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
<div className="body-button" onClick={onSetServerClicked}>set</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Microphone</div>
|
||||
<div className="body-item-title">Protocol</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" onChange={(e) => { setAudioInputDeviceId(e.target.value) }}>
|
||||
<select className="body-select" value={options.protocol} onChange={(e) => {
|
||||
onProtocolChanged(e.target.value as
|
||||
Protocol)
|
||||
}}>
|
||||
{
|
||||
audioDeviceInfo.map(x => {
|
||||
return <option key={x.deviceId} value={x.deviceId}>{x.label}</option>
|
||||
Object.values(Protocol).map(x => {
|
||||
return <option key={x} value={x}>{x}</option>
|
||||
})
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-3-4 left-padding-1">
|
||||
<div className="body-item-title">MMVC Server</div>
|
||||
<div className="body-input-container">
|
||||
<input type="text" defaultValue={options.mmvcServerUrl} id="mmvc-server-url" />
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
<div className="body-button" onClick={onSetServerClicked}>set</div>
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Microphone</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={options.audioInputDeviceId || "none"} onChange={(e) => { setAudioInputDeviceId(e.target.value) }}>
|
||||
{
|
||||
audioDeviceInfo.map(x => {
|
||||
return <option key={x.deviceId} value={x.deviceId}>{x.label}</option>
|
||||
})
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -211,16 +238,38 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Chunk Size</div>
|
||||
<div className="body-item-title">Chunk Num(128sample/chunk)</div>
|
||||
<div className="body-input-container">
|
||||
<input type="number" min={1} max={256} step={1} value={options.chunkSize} onChange={(e) => { onChunkSizeChanged(Number(e.target.value)) }} />
|
||||
<input type="number" min={1} max={256} step={1} value={options.inputChunkNum} onChange={(e) => { onChunkSizeChanged(Number(e.target.value)) }} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-3-4 left-padding-1 highlight">
|
||||
<div className="body-item-title">VF Enabled</div>
|
||||
<div>
|
||||
<input type="checkbox" checked={options.forceVfDisable} onChange={(e) => onVfEnabledChange(e.target.checked)} />
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Voice Change Mode</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={options.voiceChangerMode} onChange={(e) => { onVoiceChangeModeChanged(e.target.value as VoiceChangerMode) }}>
|
||||
{
|
||||
Object.values(VoiceChangerMode).map(x => {
|
||||
return <option key={x} value={x}>{x}</option>
|
||||
})
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Source Speaker Id</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={options.srcId} onChange={(e) => { onSrcIdChanged(Number(e.target.value)) }}>
|
||||
<select className="body-select" value={params.srcId} onChange={(e) => { onSrcIdChanged(Number(e.target.value)) }}>
|
||||
{
|
||||
options.speakers.map(x => {
|
||||
return <option key={x.id} value={x.id}>{x.name}({x.id})</option>
|
||||
@ -233,7 +282,7 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Destination Speaker Id</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={options.dstId} onChange={(e) => { onDstIdChanged(Number(e.target.value)) }}>
|
||||
<select className="body-select" value={params.dstId} onChange={(e) => { onDstIdChanged(Number(e.target.value)) }}>
|
||||
{
|
||||
options.speakers.map(x => {
|
||||
return <option key={x.id} value={x.id}>{x.name}({x.id})</option>
|
||||
@ -260,32 +309,10 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-3-4 left-padding-1 highlight">
|
||||
<div className="body-item-title">VF Enabled</div>
|
||||
<div>
|
||||
<input type="checkbox" checked={options.vfEnabled} onChange={(e) => onVfEnabledChange(e.target.checked)} />
|
||||
</div>
|
||||
<div className="body-button-container">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Voice Change Mode</div>
|
||||
<div className="body-select-container">
|
||||
<select className="body-select" value={options.voiceChangerMode} onChange={(e) => { onVoiceChangeModeChanged(e.target.value as VoiceChangerMode) }}>
|
||||
{
|
||||
Object.values(VoiceChangerMode).map(x => {
|
||||
return <option key={x} value={x}>{x}</option>
|
||||
})
|
||||
}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">GPU</div>
|
||||
<div className="body-input-container">
|
||||
<input type="number" min={-1} max={5} step={1} value={options.gpu} onChange={(e) => { onGpuChanged(Number(e.target.value)) }} />
|
||||
<input type="number" min={-1} max={5} step={1} value={params.gpu} onChange={(e) => { onGpuChanged(Number(e.target.value)) }} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -293,30 +320,32 @@ export const useMicrophoneOptions = (): MicrophoneOptionsComponent => {
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Cross Fade Lower Val</div>
|
||||
<div className="body-input-container">
|
||||
<input type="number" min={0} max={1} step={0.1} value={options.crossFadeLowerValue} onChange={(e) => { onCrossFadeLowerValueChanged(Number(e.target.value)) }} />
|
||||
<input type="number" min={0} max={1} step={0.1} value={params.crossFadeLowerValue} onChange={(e) => { onCrossFadeLowerValueChanged(Number(e.target.value)) }} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Cross Fade Offset Rate</div>
|
||||
<div className="body-input-container">
|
||||
<input type="number" min={0} max={1} step={0.1} value={options.crossFadeOffsetRate} onChange={(e) => { onCrossFadeOffsetRateChanged(Number(e.target.value)) }} />
|
||||
<input type="number" min={0} max={1} step={0.1} value={params.crossFadeOffsetRate} onChange={(e) => { onCrossFadeOffsetRateChanged(Number(e.target.value)) }} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="body-row split-3-7 left-padding-1 highlight">
|
||||
<div className="body-item-title">Cross Fade End Rate</div>
|
||||
<div className="body-input-container">
|
||||
<input type="number" min={0} max={1} step={0.1} value={options.crossFadeEndRate} onChange={(e) => { onCrossFadeEndRateChanged(Number(e.target.value)) }} />
|
||||
<input type="number" min={0} max={1} step={0.1} value={params.crossFadeEndRate} onChange={(e) => { onCrossFadeEndRateChanged(Number(e.target.value)) }} />
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}, [audioDeviceInfo, editSpeakerTargetId, editSpeakerTargetName, options])
|
||||
}, [audioDeviceInfo, editSpeakerTargetId, editSpeakerTargetName, startButtonRow, params, options])
|
||||
|
||||
return {
|
||||
component: settings,
|
||||
options: options
|
||||
params: params,
|
||||
options: options,
|
||||
isStarted
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { io, Socket } from "socket.io-client";
|
||||
import { DefaultEventsMap } from "@socket.io/component-emitter";
|
||||
import { Duplex, DuplexOptions } from "readable-stream";
|
||||
import { DefaultVoiceChangerRequestParamas, MajarModeTypes, VoiceChangerMode, VoiceChangerRequestParamas } from "./const";
|
||||
import { DefaultVoiceChangerRequestParamas, Protocol, VoiceChangerMode, VoiceChangerRequestParamas, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
|
||||
|
||||
export type Callbacks = {
|
||||
onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer) => void
|
||||
@ -9,17 +9,17 @@ export type Callbacks = {
|
||||
export type AudioStreamerListeners = {
|
||||
notifySendBufferingTime: (time: number) => void
|
||||
notifyResponseTime: (time: number) => void
|
||||
notifyException: (message: string) => void
|
||||
notifyException: (code: VOICE_CHANGER_CLIENT_EXCEPTION, message: string) => void
|
||||
}
|
||||
export class AudioStreamer extends Duplex {
|
||||
private callbacks: Callbacks
|
||||
private audioStreamerListeners: AudioStreamerListeners
|
||||
private majarMode: MajarModeTypes
|
||||
private protocol: Protocol = "sio"
|
||||
private serverUrl = ""
|
||||
private socket: Socket<DefaultEventsMap, DefaultEventsMap> | null = null
|
||||
private voiceChangerMode: VoiceChangerMode = "realtime"
|
||||
private requestParamas: VoiceChangerRequestParamas = DefaultVoiceChangerRequestParamas
|
||||
private chunkNum = 8
|
||||
private inputChunkNum = 10
|
||||
private requestChunks: ArrayBuffer[] = []
|
||||
private recordChunks: ArrayBuffer[] = []
|
||||
private isRecording = false
|
||||
@ -27,9 +27,8 @@ export class AudioStreamer extends Duplex {
|
||||
// performance monitor
|
||||
private bufferStart = 0;
|
||||
|
||||
constructor(majarMode: MajarModeTypes, callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
|
||||
constructor(callbacks: Callbacks, audioStreamerListeners: AudioStreamerListeners, options?: DuplexOptions) {
|
||||
super(options);
|
||||
this.majarMode = majarMode
|
||||
this.callbacks = callbacks
|
||||
this.audioStreamerListeners = audioStreamerListeners
|
||||
}
|
||||
@ -38,17 +37,19 @@ export class AudioStreamer extends Duplex {
|
||||
if (this.socket) {
|
||||
this.socket.close()
|
||||
}
|
||||
if (this.majarMode === "sio") {
|
||||
if (this.protocol === "sio") {
|
||||
this.socket = io(this.serverUrl);
|
||||
this.socket.on('connect_error', (err) => {
|
||||
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_CONNECT_FAILED, `[SIO] rconnection failed ${err}`)
|
||||
})
|
||||
this.socket.on('connect', () => console.log(`[SIO] sonnect to ${this.serverUrl}`));
|
||||
this.socket.on('response', (response: any[]) => {
|
||||
const cur = Date.now()
|
||||
const responseTime = cur - response[0]
|
||||
const result = response[1] as ArrayBuffer
|
||||
if (result.byteLength < 128 * 2) {
|
||||
this.audioStreamerListeners.notifyException(`[SIO] recevied data is too short ${result.byteLength}`)
|
||||
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_SIO_INVALID_RESPONSE, `[SIO] recevied data is too short ${result.byteLength}`)
|
||||
} else {
|
||||
this.audioStreamerListeners.notifyException(``)
|
||||
this.callbacks.onVoiceReceived(this.voiceChangerMode, response[1])
|
||||
this.audioStreamerListeners.notifyResponseTime(responseTime)
|
||||
}
|
||||
@ -57,11 +58,13 @@ export class AudioStreamer extends Duplex {
|
||||
}
|
||||
|
||||
// Option Change
|
||||
setServerUrl = (serverUrl: string, mode: MajarModeTypes) => {
|
||||
setServerUrl = (serverUrl: string, mode: Protocol, openTab: boolean = false) => {
|
||||
this.serverUrl = serverUrl
|
||||
this.majarMode = mode
|
||||
window.open(serverUrl, '_blank')
|
||||
console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.majarMode}`)
|
||||
this.protocol = mode
|
||||
if (openTab) {
|
||||
window.open(serverUrl, '_blank')
|
||||
}
|
||||
console.log(`[AudioStreamer] Server Setting:${this.serverUrl} ${this.protocol}`)
|
||||
|
||||
this.createSocketIO()// mode check is done in the method.
|
||||
}
|
||||
@ -70,8 +73,8 @@ export class AudioStreamer extends Duplex {
|
||||
this.requestParamas = val
|
||||
}
|
||||
|
||||
setChunkNum = (num: number) => {
|
||||
this.chunkNum = num
|
||||
setInputChunkNum = (num: number) => {
|
||||
this.inputChunkNum = num
|
||||
}
|
||||
|
||||
setVoiceChangerMode = (val: VoiceChangerMode) => {
|
||||
@ -115,7 +118,7 @@ export class AudioStreamer extends Duplex {
|
||||
}
|
||||
|
||||
//// リクエストバッファの中身が、リクエスト送信数と違う場合は処理終了。
|
||||
if (this.requestChunks.length < this.chunkNum) {
|
||||
if (this.requestChunks.length < this.inputChunkNum) {
|
||||
return
|
||||
}
|
||||
|
||||
@ -131,7 +134,7 @@ export class AudioStreamer extends Duplex {
|
||||
return prev + cur.byteLength
|
||||
}, 0)
|
||||
|
||||
console.log("send buff length", newBuffer.length)
|
||||
// console.log("send buff length", newBuffer.length)
|
||||
|
||||
this.sendBuffer(newBuffer)
|
||||
this.requestChunks = []
|
||||
@ -189,14 +192,14 @@ export class AudioStreamer extends Duplex {
|
||||
}
|
||||
const timestamp = Date.now()
|
||||
// console.log("REQUEST_MESSAGE:", [this.gpu, this.srcId, this.dstId, timestamp, newBuffer.buffer])
|
||||
console.log("SERVER_URL", this.serverUrl, this.majarMode)
|
||||
// console.log("SERVER_URL", this.serverUrl, this.protocol)
|
||||
const convertChunkNum = this.voiceChangerMode === "realtime" ? this.requestParamas.convertChunkNum : 0
|
||||
if (this.majarMode === "sio") {
|
||||
if (this.protocol === "sio") {
|
||||
if (!this.socket) {
|
||||
console.warn(`sio is not initialized`)
|
||||
return
|
||||
}
|
||||
console.log("emit!")
|
||||
// console.log("emit!")
|
||||
this.socket.emit('request_message', [
|
||||
this.requestParamas.gpu,
|
||||
this.requestParamas.srcId,
|
||||
@ -221,9 +224,8 @@ export class AudioStreamer extends Duplex {
|
||||
newBuffer.buffer)
|
||||
|
||||
if (res.byteLength < 128 * 2) {
|
||||
this.audioStreamerListeners.notifyException(`[REST] recevied data is too short ${res.byteLength}`)
|
||||
this.audioStreamerListeners.notifyException(VOICE_CHANGER_CLIENT_EXCEPTION.ERR_REST_INVALID_RESPONSE, `[REST] recevied data is too short ${res.byteLength}`)
|
||||
} else {
|
||||
this.audioStreamerListeners.notifyException(``)
|
||||
this.callbacks.onVoiceReceived(this.voiceChangerMode, res)
|
||||
this.audioStreamerListeners.notifyResponseTime(Date.now() - timestamp)
|
||||
}
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
import { VoiceChangerWorkletNode } from "./VoiceChangerWorkletNode";
|
||||
import { VoiceChangerWorkletNode, VolumeListener } from "./VoiceChangerWorkletNode";
|
||||
// @ts-ignore
|
||||
import workerjs from "raw-loader!../worklet/dist/index.js";
|
||||
import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js";
|
||||
import { createDummyMediaStream } from "./util";
|
||||
import { BufferSize, MajarModeTypes, VoiceChangerMode, VoiceChangerRequestParamas } from "./const";
|
||||
import { BufferSize, DefaultVoiceChangerOptions, DefaultVoiceChangerRequestParamas, Protocol, VoiceChangerMode, VoiceChangerRequestParamas } from "./const";
|
||||
import MicrophoneStream from "microphone-stream";
|
||||
import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreamer";
|
||||
|
||||
@ -29,10 +29,11 @@ export class VoiceChnagerClient {
|
||||
private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode
|
||||
|
||||
private promiseForInitialize: Promise<void>
|
||||
private _isVoiceChanging = false
|
||||
|
||||
private callbacks: Callbacks = {
|
||||
onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer): void => {
|
||||
console.log(voiceChangerMode, data)
|
||||
// console.log(voiceChangerMode, data)
|
||||
if (voiceChangerMode === "realtime") {
|
||||
this.vcNode.postReceivedVoice(data)
|
||||
return
|
||||
@ -59,19 +60,21 @@ export class VoiceChnagerClient {
|
||||
}
|
||||
}
|
||||
|
||||
constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners) {
|
||||
constructor(ctx: AudioContext, vfEnable: boolean, audioStreamerListeners: AudioStreamerListeners, volumeListener: VolumeListener) {
|
||||
this.ctx = ctx
|
||||
this.vfEnable = vfEnable
|
||||
this.promiseForInitialize = new Promise<void>(async (resolve) => {
|
||||
const scriptUrl = URL.createObjectURL(new Blob([workerjs], { type: "text/javascript" }));
|
||||
await this.ctx.audioWorklet.addModule(scriptUrl)
|
||||
|
||||
this.vcNode = new VoiceChangerWorkletNode(this.ctx); // vc node
|
||||
this.vcNode = new VoiceChangerWorkletNode(this.ctx, volumeListener); // vc node
|
||||
this.currentMediaStreamAudioDestinationNode = this.ctx.createMediaStreamDestination() // output node
|
||||
this.vcNode.connect(this.currentMediaStreamAudioDestinationNode) // vc node -> output node
|
||||
// (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる)
|
||||
this.audioStreamer = new AudioStreamer("sio", this.callbacks, audioStreamerListeners, { objectMode: true, })
|
||||
|
||||
this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, })
|
||||
this.audioStreamer.setRequestParams(DefaultVoiceChangerRequestParamas)
|
||||
this.audioStreamer.setInputChunkNum(DefaultVoiceChangerOptions.inputChunkNum)
|
||||
this.audioStreamer.setVoiceChangerMode(DefaultVoiceChangerOptions.voiceChangerMode)
|
||||
|
||||
if (this.vfEnable) {
|
||||
this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' })
|
||||
@ -113,12 +116,17 @@ export class VoiceChnagerClient {
|
||||
}
|
||||
|
||||
// create mic stream
|
||||
if (this.micStream) {
|
||||
console.log("DESTROY!!!!!!!!!!!!!!!!!!!")
|
||||
// this.micStream.stop()
|
||||
this.micStream.destroy()
|
||||
this.micStream = null
|
||||
}
|
||||
this.micStream = new MicrophoneStream({
|
||||
objectMode: true,
|
||||
bufferSize: bufferSize,
|
||||
context: this.ctx
|
||||
})
|
||||
|
||||
// connect nodes.
|
||||
if (this.currentDevice && forceVfDisable == false) {
|
||||
this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream) // input node
|
||||
@ -128,6 +136,7 @@ export class VoiceChnagerClient {
|
||||
voiceFocusNode.end.connect(this.outputNodeFromVF!)
|
||||
this.micStream.setStream(this.outputNodeFromVF!.stream) // vf node -> mic stream
|
||||
} else {
|
||||
console.log("VF disabled")
|
||||
this.micStream.setStream(this.currentMediaStream) // input device -> mic stream
|
||||
}
|
||||
this.micStream.pipe(this.audioStreamer!) // mic stream -> audio streamer
|
||||
@ -138,22 +147,35 @@ export class VoiceChnagerClient {
|
||||
return this.currentMediaStreamAudioDestinationNode.stream
|
||||
}
|
||||
|
||||
|
||||
|
||||
start = () => {
|
||||
if (!this.micStream) { return }
|
||||
this.micStream.playRecording()
|
||||
this._isVoiceChanging = true
|
||||
}
|
||||
stop = () => {
|
||||
if (!this.micStream) { return }
|
||||
this.micStream.pauseRecording()
|
||||
this._isVoiceChanging = false
|
||||
}
|
||||
get isVoiceChanging(): boolean {
|
||||
return this._isVoiceChanging
|
||||
}
|
||||
// Audio Streamer Settingg
|
||||
setServerUrl = (serverUrl: string, mode: MajarModeTypes) => {
|
||||
this.audioStreamer.setServerUrl(serverUrl, mode)
|
||||
setServerUrl = (serverUrl: string, mode: Protocol, openTab: boolean = false) => {
|
||||
this.audioStreamer.setServerUrl(serverUrl, mode, openTab)
|
||||
}
|
||||
|
||||
setRequestParams = (val: VoiceChangerRequestParamas) => {
|
||||
this.audioStreamer.setRequestParams(val)
|
||||
}
|
||||
|
||||
setChunkNum = (num: number) => {
|
||||
this.audioStreamer.setChunkNum(num)
|
||||
setInputChunkNum = (num: number) => {
|
||||
this.audioStreamer.setInputChunkNum(num)
|
||||
}
|
||||
|
||||
setVoiceChangerMode = (val: VoiceChangerMode) => {
|
||||
this.audioStreamer.setVoiceChangerMode(val)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@ -1,7 +1,13 @@
|
||||
export type VolumeListener = {
|
||||
notifyVolume: (vol: number) => void
|
||||
}
|
||||
|
||||
export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
constructor(context: AudioContext) {
|
||||
private listener: VolumeListener
|
||||
constructor(context: AudioContext, listener: VolumeListener) {
|
||||
super(context, "voice-changer-worklet-processor");
|
||||
this.port.onmessage = this.handleMessage.bind(this);
|
||||
this.listener = listener
|
||||
console.log(`[worklet_node][voice-changer-worklet-processor] created.`);
|
||||
}
|
||||
|
||||
@ -12,6 +18,7 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
|
||||
}
|
||||
|
||||
handleMessage(event: any) {
|
||||
console.log(`[Node:handleMessage_] `, event.data.volume);
|
||||
// console.log(`[Node:handleMessage_] `, event.data.volume);
|
||||
this.listener.notifyVolume(event.data.volume as number)
|
||||
}
|
||||
}
|
||||
@ -5,7 +5,7 @@
|
||||
|
||||
// types
|
||||
export type VoiceChangerRequestParamas = {
|
||||
convertChunkNum: number,
|
||||
convertChunkNum: number, // VITSに入力する変換サイズ。(入力データの2倍以上の大きさで指定。それより小さいものが指定された場合は、サーバ側で自動的に入力の2倍のサイズが設定される。)
|
||||
srcId: number,
|
||||
dstId: number,
|
||||
gpu: number,
|
||||
@ -15,18 +15,14 @@ export type VoiceChangerRequestParamas = {
|
||||
crossFadeEndRate: number,
|
||||
}
|
||||
|
||||
export type VoiceChangerRequest = VoiceChangerRequestParamas & {
|
||||
data: ArrayBuffer,
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
export type VoiceChangerOptions = {
|
||||
audioInputDeviceId: string | null,
|
||||
mediaStream: MediaStream | null,
|
||||
mmvcServerUrl: string,
|
||||
protocol: Protocol,
|
||||
sampleRate: SampleRate, // 48000Hz
|
||||
bufferSize: BufferSize, // 256, 512, 1024, 2048, 4096, 8192, 16384 (for mic stream)
|
||||
chunkNum: number, // n of (256 x n) for send buffer
|
||||
inputChunkNum: number, // n of (256 x n) for send buffer
|
||||
speakers: Speaker[],
|
||||
forceVfDisable: boolean,
|
||||
voiceChangerMode: VoiceChangerMode,
|
||||
@ -40,11 +36,11 @@ export type Speaker = {
|
||||
|
||||
// Consts
|
||||
|
||||
export const MajarModeTypes = {
|
||||
export const Protocol = {
|
||||
"sio": "sio",
|
||||
"rest": "rest",
|
||||
} as const
|
||||
export type MajarModeTypes = typeof MajarModeTypes[keyof typeof MajarModeTypes]
|
||||
export type Protocol = typeof Protocol[keyof typeof Protocol]
|
||||
|
||||
export const VoiceChangerMode = {
|
||||
"realtime": "realtime",
|
||||
@ -58,42 +54,69 @@ export const SampleRate = {
|
||||
export type SampleRate = typeof SampleRate[keyof typeof SampleRate]
|
||||
|
||||
export const BufferSize = {
|
||||
"256": 256,
|
||||
"512": 512,
|
||||
"1024": 1024,
|
||||
"2048": 2048,
|
||||
"4096": 4096,
|
||||
"8192": 8192,
|
||||
"16384": 16384
|
||||
} as const
|
||||
export type BufferSize = typeof BufferSize[keyof typeof BufferSize]
|
||||
|
||||
|
||||
// Defaults
|
||||
export const DefaultVoiceChangerRequestParamas: VoiceChangerRequestParamas = {
|
||||
convertChunkNum: 12, //（★１）
|
||||
convertChunkNum: 1, //（★１）
|
||||
srcId: 107,
|
||||
dstId: 100,
|
||||
gpu: 0,
|
||||
crossFadeLowerValue: 0.1,
|
||||
crossFadeOffsetRate: 0.3,
|
||||
crossFadeEndRate: 0.6
|
||||
crossFadeOffsetRate: 0.1,
|
||||
crossFadeEndRate: 0.9
|
||||
}
|
||||
|
||||
export const DefaultSpeakders: Speaker[] = [
|
||||
{
|
||||
"id": 100,
|
||||
"name": "ずんだもん"
|
||||
},
|
||||
{
|
||||
"id": 107,
|
||||
"name": "user"
|
||||
},
|
||||
{
|
||||
"id": 101,
|
||||
"name": "そら"
|
||||
},
|
||||
{
|
||||
"id": 102,
|
||||
"name": "めたん"
|
||||
},
|
||||
{
|
||||
"id": 103,
|
||||
"name": "つむぎ"
|
||||
}
|
||||
]
|
||||
export const DefaultVoiceChangerOptions: VoiceChangerOptions = {
|
||||
audioInputDeviceId: null,
|
||||
mediaStream: null,
|
||||
mmvcServerUrl: "https://192.168.0.3:18888/test",
|
||||
protocol: "sio",
|
||||
sampleRate: 48000,
|
||||
bufferSize: 1024,
|
||||
inputChunkNum: 48,
|
||||
speakers: [
|
||||
{
|
||||
"id": 100,
|
||||
"name": "ずんだもん"
|
||||
},
|
||||
{
|
||||
"id": 107,
|
||||
"name": "user"
|
||||
},
|
||||
{
|
||||
"id": 101,
|
||||
"name": "そら"
|
||||
},
|
||||
{
|
||||
"id": 102,
|
||||
"name": "めたん"
|
||||
},
|
||||
{
|
||||
"id": 103,
|
||||
"name": "つむぎ"
|
||||
}
|
||||
],
|
||||
forceVfDisable: false,
|
||||
voiceChangerMode: "realtime"
|
||||
}
|
||||
|
||||
|
||||
export const VOICE_CHANGER_CLIENT_EXCEPTION = {
|
||||
ERR_SIO_CONNECT_FAILED: "ERR_SIO_CONNECT_FAILED",
|
||||
ERR_SIO_INVALID_RESPONSE: "ERR_SIO_INVALID_RESPONSE",
|
||||
ERR_REST_INVALID_RESPONSE: "ERR_REST_INVALID_RESPONSE"
|
||||
|
||||
} as const
|
||||
export type VOICE_CHANGER_CLIENT_EXCEPTION = typeof VOICE_CHANGER_CLIENT_EXCEPTION[keyof typeof VOICE_CHANGER_CLIENT_EXCEPTION]
|
||||
|
||||
|
||||
|
||||
@ -19,7 +19,7 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
|
||||
// データは(int16)で受信
|
||||
const i16Data = new Int16Array(arrayBuffer)
|
||||
const f32Data = new Float32Array(i16Data.length)
|
||||
console.log(`[worklet] f32DataLength${f32Data.length} i16DataLength${i16Data.length}`)
|
||||
// console.log(`[worklet] f32DataLength${f32Data.length} i16DataLength${i16Data.length}`)
|
||||
i16Data.forEach((x, i) => {
|
||||
const float = (x >= 0x8000) ? -(0x10000 - x) / 0x8000 : x / 0x7FFF;
|
||||
f32Data[i] = float
|
||||
|
||||
Loading…
Reference in New Issue
Block a user