PublicMirrors/voice-changer
1
0
Fork 0
mirror of https://github.com/w-okada/voice-changer.git synced 2025-02-03 00:33:57 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #123 from w-okada/v.1.3.1

Browse Source 
V.1.3.1
This commit is contained in:
w-okada 2023-01-12 17:24:33 +09:00 committed by GitHub
commit 50a818f7ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 951 additions and 660 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/demo/dist/index.js vendored
View File

File diff suppressed because one or more lines are too long

4
client/demo/src/100_options_microphone.tsx
View File

@ -1,7 +1,7 @@
import * as React from "react";
import { useEffect, useMemo, useState } from "react";
import { AUDIO_ELEMENT_FOR_PLAY_RESULT } from "./const";
import { useServerSetting } from "./101_server_setting";
import { useServerSettingArea } from "./101_server_setting";
import { useDeviceSetting } from "./102_device_setting";
import { useConvertSetting } from "./104_convert_setting";
import { useAdvancedSetting } from "./105_advanced_setting";
@ -17,7 +17,7 @@ export const useMicrophoneOptions = () => {
audioOutputElementId: AUDIO_ELEMENT_FOR_PLAY_RESULT
})
const serverSetting = useServerSetting({ clientState })
const serverSetting = useServerSettingArea({ clientState })
const deviceSetting = useDeviceSetting(audioContext, { clientState })
const speakerSetting = useSpeakerSetting({ clientState })
const convertSetting = useConvertSetting({ clientState })

115
client/demo/src/101_server_setting.tsx
View File

@ -11,28 +11,7 @@ export type ServerSettingState = {
serverSetting: JSX.Element;
}
export const useServerSetting = (props: UseServerSettingProps): ServerSettingState => {
const mmvcServerUrlRow = useMemo(() => {
const onSetServerClicked = async () => {
const input = document.getElementById("mmvc-server-url") as HTMLInputElement
props.clientState.setSettingState({
...props.clientState.settingState,
mmvcServerUrl: input.value
})
}
return (
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">MMVC Server</div>
<div className="body-input-container">
<input type="text" defaultValue={props.clientState.settingState.mmvcServerUrl} id="mmvc-server-url" className="body-item-input" />
</div>
<div className="body-button-container">
<div className="body-button" onClick={onSetServerClicked}>set</div>
</div>
</div>
)
}, [props.clientState.settingState])
export const useServerSettingArea = (props: UseServerSettingProps): ServerSettingState => {
const uploadeModelRow = useMemo(() => {
const onPyTorchFileLoadClicked = async () => {
const file = await fileSelector("")
@ -40,14 +19,14 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
alert("モデルファイルの拡張子はpthである必要があります。")
return
}
props.clientState.setSettingState({
...props.clientState.settingState,
props.clientState.serverSetting.setFileUploadSetting({
...props.clientState.serverSetting.fileUploadSetting,
pyTorchModel: file
})
}
const onPyTorchFileClearClicked = () => {
props.clientState.setSettingState({
...props.clientState.settingState,
props.clientState.serverSetting.setFileUploadSetting({
...props.clientState.serverSetting.fileUploadSetting,
pyTorchModel: null
})
}
@ -57,14 +36,14 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
alert("モデルファイルの拡張子はjsonである必要があります。")
return
}
props.clientState.setSettingState({
...props.clientState.settingState,
props.clientState.serverSetting.setFileUploadSetting({
...props.clientState.serverSetting.fileUploadSetting,
configFile: file
})
}
const onConfigFileClearClicked = () => {
props.clientState.setSettingState({
...props.clientState.settingState,
props.clientState.serverSetting.setFileUploadSetting({
...props.clientState.serverSetting.fileUploadSetting,
configFile: null
})
}
@ -74,19 +53,19 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
alert("モデルファイルの拡張子はonnxである必要があります。")
return
}
props.clientState.setSettingState({
...props.clientState.settingState,
props.clientState.serverSetting.setFileUploadSetting({
...props.clientState.serverSetting.fileUploadSetting,
onnxModel: file
})
}
const onOnnxFileClearClicked = () => {
props.clientState.setSettingState({
...props.clientState.settingState,
props.clientState.serverSetting.setFileUploadSetting({
...props.clientState.serverSetting.fileUploadSetting,
onnxModel: null
})
}
const onModelUploadClicked = async () => {
props.clientState.loadModel()
props.clientState.serverSetting.loadModel()
}
return (
@ -100,30 +79,30 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
<div></div>
</div>
</div>
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-2">PyTorch(.pth)</div>
<div className="body-item-text">
<div>{props.clientState.settingState.pyTorchModel?.name}</div>
</div>
<div className="body-button-container">
<div className="body-button" onClick={onPyTorchFileLoadClicked}>select</div>
<div className="body-button left-margin-1" onClick={onPyTorchFileClearClicked}>clear</div>
</div>
</div>
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-2">Config(.json)</div>
<div className="body-item-text">
<div>{props.clientState.settingState.configFile?.name}</div>
<div>{props.clientState.serverSetting.fileUploadSetting.configFile?.name}</div>
</div>
<div className="body-button-container">
<div className="body-button" onClick={onConfigFileLoadClicked}>select</div>
<div className="body-button left-margin-1" onClick={onConfigFileClearClicked}>clear</div>
</div>
</div>
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-2">PyTorch(.pth)</div>
<div className="body-item-text">
<div>{props.clientState.serverSetting.fileUploadSetting.pyTorchModel?.name}</div>
</div>
<div className="body-button-container">
<div className="body-button" onClick={onPyTorchFileLoadClicked}>select</div>
<div className="body-button left-margin-1" onClick={onPyTorchFileClearClicked}>clear</div>
</div>
</div>
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-2">Onnx(.onnx)</div>
<div className="body-item-text">
<div>{props.clientState.settingState.onnxModel?.name}</div>
<div>{props.clientState.serverSetting.fileUploadSetting.onnxModel?.name}</div>
</div>
<div className="body-button-container">
<div className="body-button" onClick={onOnnxFileLoadClicked}>select</div>
@ -133,7 +112,7 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-2"></div>
<div className="body-item-text">
{props.clientState.isUploading ? `uploading.... ${props.clientState.uploadProgress}%` : ""}
{props.clientState.serverSetting.isUploading ? `uploading.... ${props.clientState.serverSetting.uploadProgress}%` : ""}
</div>
<div className="body-button-container">
<div className="body-button" onClick={onModelUploadClicked}>upload</div>
@ -142,23 +121,20 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
</>
)
}, [
props.clientState.settingState,
props.clientState.loadModel,
props.clientState.isUploading,
props.clientState.uploadProgress])
props.clientState.serverSetting.fileUploadSetting,
props.clientState.serverSetting.loadModel,
props.clientState.serverSetting.isUploading,
props.clientState.serverSetting.uploadProgress])
const protocolRow = useMemo(() => {
const onProtocolChanged = async (val: Protocol) => {
props.clientState.setSettingState({
...props.clientState.settingState,
protocol: val
})
props.clientState.clientSetting.setProtocol(val)
}
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Protocol</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.settingState.protocol} onChange={(e) => {
<select className="body-select" value={props.clientState.clientSetting.setting.protocol} onChange={(e) => {
onProtocolChanged(e.target.value as
Protocol)
}}>
@ -171,20 +147,17 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.clientSetting.setting.protocol, props.clientState.clientSetting.setProtocol])
const frameworkRow = useMemo(() => {
const onFrameworkChanged = async (val: Framework) => {
props.clientState.setSettingState({
...props.clientState.settingState,
framework: val
})
props.clientState.serverSetting.setFramework(val)
}
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Framework</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.settingState.framework} onChange={(e) => {
<select className="body-select" value={props.clientState.serverSetting.setting.framework} onChange={(e) => {
onFrameworkChanged(e.target.value as
Framework)
}}>
@ -197,23 +170,20 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.serverSetting.setting.framework, props.clientState.serverSetting.setFramework])
const onnxExecutionProviderRow = useMemo(() => {
if (props.clientState.settingState.framework != "ONNX") {
if (props.clientState.serverSetting.setting.framework != "ONNX") {
return
}
const onOnnxExecutionProviderChanged = async (val: OnnxExecutionProvider) => {
props.clientState.setSettingState({
...props.clientState.settingState,
onnxExecutionProvider: val
})
props.clientState.serverSetting.setOnnxExecutionProvider(val)
}
return (
<div className="body-row split-3-7 left-padding-1">
<div className="body-item-title left-padding-2">OnnxExecutionProvider</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.settingState.onnxExecutionProvider} onChange={(e) => {
<select className="body-select" value={props.clientState.serverSetting.setting.onnxExecutionProvider} onChange={(e) => {
onOnnxExecutionProviderChanged(e.target.value as
OnnxExecutionProvider)
}}>
@ -226,7 +196,7 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.serverSetting.setting.framework, props.clientState.serverSetting.setting.onnxExecutionProvider, props.clientState.serverSetting.setOnnxExecutionProvider])
const serverSetting = useMemo(() => {
return (
@ -236,14 +206,13 @@ export const useServerSetting = (props: UseServerSettingProps): ServerSettingSta
<div className="body-select-container">
</div>
</div>
{mmvcServerUrlRow}
{uploadeModelRow}
{frameworkRow}
{onnxExecutionProviderRow}
{protocolRow}
</>
)
}, [mmvcServerUrlRow, uploadeModelRow, frameworkRow, onnxExecutionProviderRow, protocolRow])
}, [uploadeModelRow, frameworkRow, onnxExecutionProviderRow, protocolRow])
return {

51
client/demo/src/102_device_setting.tsx
View File

@ -68,7 +68,9 @@ export const useDeviceSetting = (audioContext: AudioContext | null, props: UseDe
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">AudioInput</div>
<div className="body-select-container">
<select className="body-select" value={audioInputForGUI} onChange={(e) => { setAudioInputForGUI(e.target.value) }}>
<select className="body-select" value={audioInputForGUI} onChange={(e) => {
setAudioInputForGUI(e.target.value)
}}>
{
inputAudioDeviceInfo.map(x => {
return <option key={x.deviceId} value={x.deviceId}>{x.label}</option>
@ -85,21 +87,16 @@ export const useDeviceSetting = (audioContext: AudioContext | null, props: UseDe
if (!audioContext) {
return
}
if (audioInputForGUI == "none") {
const ms = createDummyMediaStream(audioContext)
props.clientState.setSettingState({
...props.clientState.settingState,
audioInput: ms
})
props.clientState.clientSetting.setAudioInput(ms)
} else if (audioInputForGUI == "file") {
// file selector (audioMediaInputRow)
} else {
props.clientState.setSettingState({
...props.clientState.settingState,
audioInput: audioInputForGUI
})
props.clientState.clientSetting.setAudioInput(audioInputForGUI)
}
}, [audioContext, audioInputForGUI])
}, [audioContext, audioInputForGUI, props.clientState.clientSetting.setAudioInput])
const audioMediaInputRow = useMemo(() => {
if (audioInputForGUI != "file") {
@ -116,10 +113,7 @@ export const useDeviceSetting = (audioContext: AudioContext | null, props: UseDe
const src = audioContext!.createMediaElementSource(audio);
const dst = audioContext!.createMediaStreamDestination()
src.connect(dst)
props.clientState.setSettingState({
...props.clientState.settingState,
audioInput: dst.stream
})
props.clientState.clientSetting.setAudioInput(dst.stream)
// original stream to play.
const audio_org = document.getElementById(AUDIO_ELEMENT_FOR_TEST_ORIGINAL) as HTMLAudioElement
audio_org.src = url
@ -148,7 +142,7 @@ export const useDeviceSetting = (audioContext: AudioContext | null, props: UseDe
</div>
</div>
)
}, [audioInputForGUI])
}, [audioInputForGUI, props.clientState.clientSetting.setAudioInput])
@ -181,30 +175,6 @@ export const useDeviceSetting = (audioContext: AudioContext | null, props: UseDe
})
}, [audioOutputForGUI])
const sampleRateRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Sample Rate</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.settingState.sampleRate} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
sampleRate: Number(e.target.value) as SampleRate
})
}}>
{
Object.values(SampleRate).map(x => {
return <option key={x} value={x}>{x}</option>
})
}
</select>
</div>
</div>
)
}, [props.clientState.settingState])
const deviceSetting = useMemo(() => {
return (
@ -216,11 +186,10 @@ export const useDeviceSetting = (audioContext: AudioContext | null, props: UseDe
</div>
{audioInputRow}
{audioMediaInputRow}
{sampleRateRow}
{audioOutputRow}
</>
)
}, [audioInputRow, audioMediaInputRow, sampleRateRow, audioOutputRow])
}, [audioInputRow, audioMediaInputRow, audioOutputRow])
return {
deviceSetting,

69
client/demo/src/103_speaker_setting.tsx
View File

@ -1,4 +1,4 @@
import React, { useMemo } from "react"
import React, { useMemo, useState } from "react"
import { ClientState } from "./hooks/useClient"
export type UseSpeakerSettingProps = {
@ -6,20 +6,19 @@ export type UseSpeakerSettingProps = {
}
export const useSpeakerSetting = (props: UseSpeakerSettingProps) => {
const [editSpeakerTargetId, setEditSpeakerTargetId] = useState<number>(0)
const [editSpeakerTargetName, setEditSpeakerTargetName] = useState<string>("")
const srcIdRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Source Speaker Id</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.settingState.srcId} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
srcId: Number(e.target.value)
})
<select className="body-select" value={props.clientState.serverSetting.setting.srcId} onChange={(e) => {
props.clientState.serverSetting.setSrcId(Number(e.target.value))
}}>
{
props.clientState.settingState.speakers.map(x => {
props.clientState.clientSetting.setting.speakers.map(x => {
return <option key={x.id} value={x.id}>{x.name}({x.id})</option>
})
}
@ -27,21 +26,18 @@ export const useSpeakerSetting = (props: UseSpeakerSettingProps) => {
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.clientSetting.setting.speakers, props.clientState.serverSetting.setting.srcId, props.clientState.serverSetting.setSrcId])
const dstIdRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Destination Speaker Id</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.settingState.dstId} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
dstId: Number(e.target.value)
})
<select className="body-select" value={props.clientState.serverSetting.setting.dstId} onChange={(e) => {
props.clientState.serverSetting.setDstId(Number(e.target.value))
}}>
{
props.clientState.settingState.speakers.map(x => {
props.clientState.clientSetting.setting.speakers.map(x => {
return <option key={x.id} value={x.id}>{x.name}({x.id})</option>
})
}
@ -49,38 +45,29 @@ export const useSpeakerSetting = (props: UseSpeakerSettingProps) => {
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.clientSetting.setting.speakers, props.clientState.serverSetting.setting.dstId, props.clientState.serverSetting.setDstId])
const editSpeakerIdMappingRow = useMemo(() => {
const onSetSpeakerMappingClicked = async () => {
const targetId = props.clientState.settingState.editSpeakerTargetId
const targetName = props.clientState.settingState.editSpeakerTargetName
const targetSpeaker = props.clientState.settingState.speakers.find(x => { return x.id == targetId })
const targetId = editSpeakerTargetId
const targetName = editSpeakerTargetName
const targetSpeaker = props.clientState.clientSetting.setting.speakers.find(x => { return x.id == targetId })
if (targetSpeaker) {
if (targetName.length == 0) { // Delete
const newSpeakers = props.clientState.settingState.speakers.filter(x => { return x.id != targetId })
props.clientState.setSettingState({
...props.clientState.settingState,
speakers: newSpeakers
})
const newSpeakers = props.clientState.clientSetting.setting.speakers.filter(x => { return x.id != targetId })
props.clientState.clientSetting.setSpeakers(newSpeakers)
} else { // Update
targetSpeaker.name = targetName
props.clientState.setSettingState({
...props.clientState.settingState,
speakers: props.clientState.settingState.speakers
})
props.clientState.clientSetting.setSpeakers([...props.clientState.clientSetting.setting.speakers])
}
} else {
if (targetName.length == 0) { // Noop
} else {// add
props.clientState.settingState.speakers.push({
props.clientState.clientSetting.setting.speakers.push({
id: targetId,
name: targetName
})
props.clientState.setSettingState({
...props.clientState.settingState,
speakers: props.clientState.settingState.speakers
})
props.clientState.clientSetting.setSpeakers([...props.clientState.clientSetting.setting.speakers])
}
}
}
@ -88,21 +75,15 @@ export const useSpeakerSetting = (props: UseSpeakerSettingProps) => {
<div className="body-row split-3-1-2-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">Edit Speaker Mapping</div>
<div className="body-input-container">
<input type="number" min={1} max={256} step={1} value={props.clientState.settingState.editSpeakerTargetId} onChange={(e) => {
<input type="number" min={1} max={256} step={1} value={editSpeakerTargetId} onChange={(e) => {
const id = Number(e.target.value)
props.clientState.setSettingState({
...props.clientState.settingState,
editSpeakerTargetId: id,
editSpeakerTargetName: props.clientState.settingState.speakers.find(x => { return x.id == id })?.name || ""
})
setEditSpeakerTargetId(id)
setEditSpeakerTargetName(props.clientState.clientSetting.setting.speakers.find(x => { return x.id == id })?.name || "")
}} />
</div>
<div className="body-input-container">
<input type="text" value={props.clientState.settingState.editSpeakerTargetName} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
editSpeakerTargetName: e.target.value
})
<input type="text" value={editSpeakerTargetName} onChange={(e) => {
setEditSpeakerTargetName(e.target.value)
}} />
</div>
<div className="body-button-container">
@ -110,7 +91,7 @@ export const useSpeakerSetting = (props: UseSpeakerSettingProps) => {
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.clientSetting.setting.speakers, editSpeakerTargetId, editSpeakerTargetName])
const speakerSetting = useMemo(() => {

98
client/demo/src/104_convert_setting.tsx
View File

@ -1,4 +1,4 @@
import { DefaultVoiceChangerRequestParamas, DefaultVoiceChangerOptions, BufferSize } from "@dannadori/voice-changer-client-js"
import { BufferSize } from "@dannadori/voice-changer-client-js"
import React, { useMemo, useState } from "react"
import { ClientState } from "./hooks/useClient"
@ -12,109 +12,32 @@ export type ConvertSettingState = {
export const useConvertSetting = (props: UseConvertSettingProps): ConvertSettingState => {
const bufferSizeRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Buffer Size</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.settingState.bufferSize} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
bufferSize: Number(e.target.value) as BufferSize
})
}}>
{
Object.values(BufferSize).map(x => {
return <option key={x} value={x}>{x}</option>
})
}
</select>
</div>
</div>
)
}, [props.clientState.settingState])
const inputChunkNumRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Input Chunk Num(128sample/chunk)</div>
<div className="body-input-container">
<input type="number" min={1} max={256} step={1} value={props.clientState.settingState.inputChunkNum} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
inputChunkNum: Number(e.target.value)
})
<input type="number" min={1} max={256} step={1} value={props.clientState.clientSetting.setting.inputChunkNum} onChange={(e) => {
props.clientState.clientSetting.setInputChunkNum(Number(e.target.value))
}} />
</div>
</div>
)
}, [props.clientState.settingState])
const convertChunkNumRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Convert Chunk Num(128sample/chunk)</div>
<div className="body-input-container">
<input type="number" min={1} max={256} step={1} value={props.clientState.settingState.convertChunkNum} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
convertChunkNum: Number(e.target.value)
})
}} />
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.clientSetting.setting.inputChunkNum, props.clientState.clientSetting.setInputChunkNum])
const gpuRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">GPU</div>
<div className="body-input-container">
<input type="number" min={-2} max={5} step={1} value={props.clientState.settingState.gpu} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
gpu: Number(e.target.value)
})
<input type="number" min={-2} max={5} step={1} value={props.clientState.serverSetting.setting.gpu} onChange={(e) => {
props.clientState.serverSetting.setGpu(Number(e.target.value))
}} />
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.serverSetting.setting.gpu, props.clientState.serverSetting.setGpu])
const crossFadeOffsetRateRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Cross Fade Offset Rate</div>
<div className="body-input-container">
<input type="number" min={0} max={1} step={0.1} value={props.clientState.settingState.crossFadeOffsetRate} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
crossFadeOffsetRate: Number(e.target.value)
})
}} />
</div>
</div>
)
}, [props.clientState.settingState])
const crossFadeEndRateRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Cross Fade End Rate</div>
<div className="body-input-container">
<input type="number" min={0} max={1} step={0.1} value={props.clientState.settingState.crossFadeEndRate} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
crossFadeEndRate: Number(e.target.value)
})
}} />
</div>
</div>
)
}, [props.clientState.settingState])
const convertSetting = useMemo(() => {
return (
@ -124,15 +47,12 @@ export const useConvertSetting = (props: UseConvertSettingProps): ConvertSetting
<div className="body-select-container">
</div>
</div>
{bufferSizeRow}
{inputChunkNumRow}
{convertChunkNumRow}
{gpuRow}
{crossFadeOffsetRateRow}
{crossFadeEndRateRow}
</>
)
}, [bufferSizeRow, inputChunkNumRow, convertChunkNumRow, gpuRow, crossFadeOffsetRateRow, crossFadeEndRateRow])
}, [inputChunkNumRow, gpuRow])
return {
convertSetting,

212
client/demo/src/105_advanced_setting.tsx
View File

@ -1,4 +1,4 @@
import { VoiceChangerMode } from "@dannadori/voice-changer-client-js"
import { BufferSize, SampleRate, VoiceChangerMode } from "@dannadori/voice-changer-client-js"
import React, { useMemo, useState } from "react"
import { ClientState } from "./hooks/useClient"
@ -12,35 +12,140 @@ export type AdvancedSettingState = {
}
export const useAdvancedSetting = (props: UseAdvancedSettingProps): AdvancedSettingState => {
const [showAdvancedSetting, setShowAdvancedSetting] = useState<boolean>(false)
const mmvcServerUrlRow = useMemo(() => {
const onSetServerClicked = async () => {
const input = document.getElementById("mmvc-server-url") as HTMLInputElement
props.clientState.clientSetting.setServerUrl(input.value)
}
return (
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">MMVC Server</div>
<div className="body-input-container">
<input type="text" defaultValue={props.clientState.clientSetting.setting.mmvcServerUrl} id="mmvc-server-url" className="body-item-input" />
</div>
<div className="body-button-container">
<div className="body-button" onClick={onSetServerClicked}>set</div>
</div>
</div>
)
}, [props.clientState.clientSetting.setting.mmvcServerUrl, props.clientState.clientSetting.setServerUrl])
const sampleRateRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Sample Rate</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.clientSetting.setting.sampleRate} onChange={(e) => {
props.clientState.clientSetting.setSampleRate(Number(e.target.value) as SampleRate)
}}>
{
Object.values(SampleRate).map(x => {
return <option key={x} value={x}>{x}</option>
})
}
</select>
</div>
</div>
)
}, [props.clientState.clientSetting.setting.sampleRate, props.clientState.clientSetting.setSampleRate])
const bufferSizeRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Buffer Size</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.clientSetting.setting.bufferSize} onChange={(e) => {
props.clientState.clientSetting.setBufferSize(Number(e.target.value) as BufferSize)
}}>
{
Object.values(BufferSize).map(x => {
return <option key={x} value={x}>{x}</option>
})
}
</select>
</div>
</div>
)
}, [props.clientState.clientSetting.setting.bufferSize, props.clientState.clientSetting.setBufferSize])
const convertChunkNumRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Convert Chunk Num(128sample/chunk)</div>
<div className="body-input-container">
<input type="number" min={1} max={256} step={1} value={props.clientState.serverSetting.setting.convertChunkNum} onChange={(e) => {
props.clientState.serverSetting.setConvertChunkNum(Number(e.target.value))
}} />
</div>
</div>
)
}, [props.clientState.serverSetting.setting.convertChunkNum, props.clientState.serverSetting.setConvertChunkNum])
const crossFadeOverlapRateRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Cross Fade Overlap Rate</div>
<div className="body-input-container">
<input type="number" min={0.1} max={1} step={0.1} value={props.clientState.serverSetting.setting.crossFadeOverlapRate} onChange={(e) => {
props.clientState.serverSetting.setCrossFadeOverlapRate(Number(e.target.value))
}} />
</div>
</div>
)
}, [props.clientState.serverSetting.setting.crossFadeOverlapRate, props.clientState.serverSetting.setCrossFadeOverlapRate])
const crossFadeOffsetRateRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Cross Fade Offset Rate</div>
<div className="body-input-container">
<input type="number" min={0} max={1} step={0.1} value={props.clientState.serverSetting.setting.crossFadeOffsetRate} onChange={(e) => {
props.clientState.serverSetting.setCrossFadeOffsetRate(Number(e.target.value))
}} />
</div>
</div>
)
}, [props.clientState.serverSetting.setting.crossFadeOffsetRate, props.clientState.serverSetting.setCrossFadeOffsetRate])
const crossFadeEndRateRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Cross Fade End Rate</div>
<div className="body-input-container">
<input type="number" min={0} max={1} step={0.1} value={props.clientState.serverSetting.setting.crossFadeEndRate} onChange={(e) => {
props.clientState.serverSetting.setCrossFadeEndRate(Number(e.target.value))
}} />
</div>
</div>
)
}, [props.clientState.serverSetting.setting.crossFadeEndRate, props.clientState.serverSetting.setCrossFadeEndRate])
const vfForceDisableRow = useMemo(() => {
return (
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-1 ">VF Disabled</div>
<div>
<input type="checkbox" checked={props.clientState.settingState.vfForceDisabled} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
vfForceDisabled: e.target.checked
})
<input type="checkbox" checked={props.clientState.clientSetting.setting.forceVfDisable} onChange={(e) => {
props.clientState.clientSetting.setVfForceDisabled(e.target.checked)
}} />
</div>
<div className="body-button-container">
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.clientSetting.setting.forceVfDisable, props.clientState.clientSetting.setVfForceDisabled])
const voiceChangeModeRow = useMemo(() => {
return (
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1 ">Voice Change Mode</div>
<div className="body-select-container">
<select className="body-select" value={props.clientState.settingState.voiceChangerMode} onChange={(e) => {
props.clientState.setSettingState({
...props.clientState.settingState,
voiceChangerMode: e.target.value as VoiceChangerMode
})
<select className="body-select" value={props.clientState.clientSetting.setting.voiceChangerMode} onChange={(e) => {
props.clientState.clientSetting.setVoiceChangerMode(e.target.value as VoiceChangerMode)
}}>
{
Object.values(VoiceChangerMode).map(x => {
@ -51,21 +156,94 @@ export const useAdvancedSetting = (props: UseAdvancedSettingProps): AdvancedSett
</div>
</div>
)
}, [props.clientState.settingState])
}, [props.clientState.clientSetting.setting.voiceChangerMode, props.clientState.clientSetting.setVoiceChangerMode])
const workletSettingRow = useMemo(() => {
return (
<>
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Trancate Num</div>
<div className="body-input-container">
<input type="number" min={50} max={300} step={1} value={props.clientState.workletSetting.setting.numTrancateTreshold} onChange={(e) => {
props.clientState.workletSetting.setSetting({
...props.clientState.workletSetting.setting,
numTrancateTreshold: Number(e.target.value)
})
}} />
</div>
</div>
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Trancate Vol</div>
<div className="body-input-container">
<input type="number" min={0.0001} max={0.0009} step={0.0001} value={props.clientState.workletSetting.setting.volTrancateThreshold} onChange={(e) => {
props.clientState.workletSetting.setSetting({
...props.clientState.workletSetting.setting,
volTrancateThreshold: Number(e.target.value)
})
}} />
</div>
</div>
<div className="body-row split-3-7 left-padding-1 guided">
<div className="body-item-title left-padding-1">Trancate Vol Length</div>
<div className="body-input-container">
<input type="number" min={16} max={128} step={1} value={props.clientState.workletSetting.setting.volTrancateLength} onChange={(e) => {
props.clientState.workletSetting.setSetting({
...props.clientState.workletSetting.setting,
volTrancateLength: Number(e.target.value)
})
}} />
</div>
</div>
</>
)
}, [props.clientState.workletSetting.setting, props.clientState.workletSetting.setSetting])
const advanceSettingContent = useMemo(() => {
if (!showAdvancedSetting) return <></>
return (
<>
<div className="body-row divider"></div>
{mmvcServerUrlRow}
<div className="body-row divider"></div>
{sampleRateRow}
{bufferSizeRow}
<div className="body-row divider"></div>
{convertChunkNumRow}
{crossFadeOverlapRateRow}
{crossFadeOffsetRateRow}
{crossFadeEndRateRow}
<div className="body-row divider"></div>
{vfForceDisableRow}
{voiceChangeModeRow}
<div className="body-row divider"></div>
{workletSettingRow}
<div className="body-row divider"></div>
</>
)
}, [showAdvancedSetting, mmvcServerUrlRow, sampleRateRow, bufferSizeRow, convertChunkNumRow, crossFadeOverlapRateRow, crossFadeOffsetRateRow, crossFadeEndRateRow, vfForceDisableRow, voiceChangeModeRow, workletSettingRow])
const advancedSetting = useMemo(() => {
return (
<>
<div className="body-row split-3-7 left-padding-1">
<div className="body-sub-section-title">Advanced Setting</div>
<div className="body-select-container">
<div>
<input type="checkbox" checked={showAdvancedSetting} onChange={(e) => {
setShowAdvancedSetting(e.target.checked)
}} /> show
</div>
</div>
{vfForceDisableRow}
{voiceChangeModeRow}
{advanceSettingContent}
</>
)
}, [vfForceDisableRow, voiceChangeModeRow])
}, [showAdvancedSetting, advanceSettingContent])
return {
advancedSetting,

14
client/demo/src/106_server_control.tsx
View File

@ -11,11 +11,11 @@ export const useServerControl = (props: UseServerControlProps) => {
const startButtonRow = useMemo(() => {
const onStartClicked = async () => {
setIsStarted(true)
await props.clientState.start()
await props.clientState.clientSetting.start()
}
const onStopClicked = async () => {
setIsStarted(false)
await props.clientState.stop()
await props.clientState.clientSetting.stop()
}
const startClassName = isStarted ? "body-button-active" : "body-button-stanby"
const stopClassName = isStarted ? "body-button-stanby" : "body-button-active"
@ -32,7 +32,7 @@ export const useServerControl = (props: UseServerControlProps) => {
</div>
)
}, [isStarted, props.clientState.start, props.clientState.stop])
}, [isStarted, props.clientState.clientSetting.start, props.clientState.clientSetting.stop])
const performanceRow = useMemo(() => {
return (
@ -60,9 +60,9 @@ export const useServerControl = (props: UseServerControlProps) => {
<div className="body-row split-3-3-4 left-padding-1 guided">
<div className="body-item-title left-padding-1">Model Info:</div>
<div className="body-item-text">
<span className="body-item-text-item">{props.clientState.serverInfo?.configFile || ""}</span>
<span className="body-item-text-item">{props.clientState.serverInfo?.pyTorchModelFile || ""}</span>
<span className="body-item-text-item">{props.clientState.serverInfo?.onnxModelFile || ""}</span>
<span className="body-item-text-item">{props.clientState.serverSetting.serverInfo?.configFile || ""}</span>
<span className="body-item-text-item">{props.clientState.serverSetting.serverInfo?.pyTorchModelFile || ""}</span>
<span className="body-item-text-item">{props.clientState.serverSetting.serverInfo?.onnxModelFile || ""}</span>
</div>
@ -72,7 +72,7 @@ export const useServerControl = (props: UseServerControlProps) => {
</div>
</>
)
}, [props.clientState.getInfo, props.clientState.serverInfo])
}, [props.clientState.getInfo, props.clientState.serverSetting.serverInfo])

1
client/demo/src/const.ts
View File

@ -1,6 +1,5 @@
export const CHROME_EXTENSION = false
export const AUDIO_ELEMENT_FOR_PLAY_RESULT = "audio-result"
export const AUDIO_ELEMENT_FOR_TEST_ORIGINAL = "audio-test-original"
export const AUDIO_ELEMENT_FOR_TEST_CONVERTED = "audio-test-converted"

6
client/demo/src/css/App.css
View File

@ -209,6 +209,12 @@ body {
/* border-bottom: 1px solid rgba(9, 133, 67, 0.3); */
}
.divider {
height:4px;
/* background-color: rgba(16, 210, 113, 0.1); */
background-color: rgba(31, 42, 36, 0.1);
}
.body-top-title {
font-size: 3rem;
}

352
client/demo/src/hooks/useClient.ts
View File

@ -1,115 +1,60 @@
import { ServerInfo, BufferSize, createDummyMediaStream, DefaultVoiceChangerOptions, DefaultVoiceChangerRequestParamas, Framework, OnnxExecutionProvider, Protocol, SampleRate, ServerSettingKey, Speaker, VoiceChangerMode, VoiceChnagerClient } from "@dannadori/voice-changer-client-js"
import { VoiceChangerClient } from "@dannadori/voice-changer-client-js"
import { useEffect, useMemo, useRef, useState } from "react"
import { ClientSettingState, useClientSetting } from "./useClientSetting"
import { ServerSettingState, useServerSetting } from "./useServerSetting"
import { useWorkletSetting, WorkletSettingState } from "./useWorkletSetting"
export type UseClientProps = {
audioContext: AudioContext | null
audioOutputElementId: string
}
export type SettingState = {
// server setting
mmvcServerUrl: string
pyTorchModel: File | null
configFile: File | null
onnxModel: File | null
protocol: Protocol
framework: Framework
onnxExecutionProvider: OnnxExecutionProvider
// device setting
audioInput: string | MediaStream | null;
sampleRate: SampleRate;
// speaker setting
speakers: Speaker[]
editSpeakerTargetId: number
editSpeakerTargetName: string
srcId: number
dstId: number
// convert setting
bufferSize: BufferSize
inputChunkNum: number
convertChunkNum: number
gpu: number
crossFadeOffsetRate: number
crossFadeEndRate: number
// advanced setting
vfForceDisabled: boolean
voiceChangerMode: VoiceChangerMode
}
const InitialSettingState: SettingState = {
mmvcServerUrl: DefaultVoiceChangerOptions.mmvcServerUrl,
pyTorchModel: null,
configFile: null,
onnxModel: null,
protocol: DefaultVoiceChangerOptions.protocol,
framework: DefaultVoiceChangerOptions.framework,
onnxExecutionProvider: DefaultVoiceChangerOptions.onnxExecutionProvider,
audioInput: "none",
sampleRate: DefaultVoiceChangerOptions.sampleRate,
speakers: DefaultVoiceChangerOptions.speakers,
editSpeakerTargetId: 0,
editSpeakerTargetName: "",
srcId: DefaultVoiceChangerRequestParamas.srcId,
dstId: DefaultVoiceChangerRequestParamas.dstId,
bufferSize: DefaultVoiceChangerOptions.bufferSize,
inputChunkNum: DefaultVoiceChangerOptions.inputChunkNum,
convertChunkNum: DefaultVoiceChangerRequestParamas.convertChunkNum,
gpu: DefaultVoiceChangerRequestParamas.gpu,
crossFadeOffsetRate: DefaultVoiceChangerRequestParamas.crossFadeOffsetRate,
crossFadeEndRate: DefaultVoiceChangerRequestParamas.crossFadeEndRate,
vfForceDisabled: DefaultVoiceChangerOptions.forceVfDisable,
voiceChangerMode: DefaultVoiceChangerOptions.voiceChangerMode
}
export type ClientState = {
clientInitialized: boolean
workletSetting: WorkletSettingState
clientSetting: ClientSettingState
serverSetting: ServerSettingState
bufferingTime: number;
responseTime: number;
volume: number;
uploadProgress: number;
isUploading: boolean
// Setting
settingState: SettingState
serverInfo: ServerInfo | undefined
setSettingState: (setting: SettingState) => void
// Client Control
loadModel: () => Promise<void>
start: () => Promise<void>;
stop: () => Promise<void>;
getInfo: () => Promise<void>
}
export const useClient = (props: UseClientProps): ClientState => {
// (1) クライアント初期化
const voiceChangerClientRef = useRef<VoiceChnagerClient | null>(null)
const [clientInitialized, setClientInitialized] = useState<boolean>(false)
// (1-1) クライアント
const voiceChangerClientRef = useRef<VoiceChangerClient | null>(null)
const [voiceChangerClient, setVoiceChangerClient] = useState<VoiceChangerClient | null>(voiceChangerClientRef.current)
//// クライアント初期化待ち用フラグ
const initializedResolveRef = useRef<(value: void | PromiseLike<void>) => void>()
const initializedPromise = useMemo(() => {
return new Promise<void>((resolve) => {
initializedResolveRef.current = resolve
})
}, [])
// (1-2) 各種設定
const clientSetting = useClientSetting({ voiceChangerClient, audioContext: props.audioContext })
const workletSetting = useWorkletSetting({ voiceChangerClient })
const serverSetting = useServerSetting({ voiceChangerClient })
// (1-3) ステータス
const [bufferingTime, setBufferingTime] = useState<number>(0)
const [responseTime, setResponseTime] = useState<number>(0)
const [volume, setVolume] = useState<number>(0)
// (2-1) 初期化処理
useEffect(() => {
const initialized = async () => {
if (!props.audioContext) {
return
}
const voiceChangerClient = new VoiceChnagerClient(props.audioContext, true, {
const voiceChangerClient = new VoiceChangerClient(props.audioContext, true, {
notifySendBufferingTime: (val: number) => {
setBufferingTime(val)
},
@ -129,8 +74,8 @@ export const useClient = (props: UseClientProps): ClientState => {
await voiceChangerClient.isInitialized()
voiceChangerClientRef.current = voiceChangerClient
setVoiceChangerClient(voiceChangerClientRef.current)
console.log("[useClient] client initialized")
setClientInitialized(true)
const audio = document.getElementById(props.audioOutputElementId) as HTMLAudioElement
audio.srcObject = voiceChangerClientRef.current.stream
@ -141,255 +86,26 @@ export const useClient = (props: UseClientProps): ClientState => {
}, [props.audioContext])
// (2) 設定
const [settingState, setSettingState] = useState<SettingState>(InitialSettingState)
const [displaySettingState, setDisplaySettingState] = useState<SettingState>(InitialSettingState)
const [serverInfo, setServerInfo] = useState<ServerInfo>()
const [uploadProgress, setUploadProgress] = useState<number>(0)
const [isUploading, setIsUploading] = useState<boolean>(false)
// (2-1) server setting
// (a) サーバURL設定
useEffect(() => {
(async () => {
await initializedPromise
voiceChangerClientRef.current!.setServerUrl(settingState.mmvcServerUrl, true)
voiceChangerClientRef.current!.stop()
getInfo()
})()
}, [settingState.mmvcServerUrl])
// (b) プロトコル設定
useEffect(() => {
(async () => {
await initializedPromise
voiceChangerClientRef.current!.setProtocol(settingState.protocol)
})()
}, [settingState.protocol])
// (c) フレームワーク設定
useEffect(() => {
(async () => {
await initializedPromise
const info = await voiceChangerClientRef.current!.updateServerSettings(ServerSettingKey.framework, "" + settingState.framework)
setServerInfo(info)
})()
}, [settingState.framework])
// (d) OnnxExecutionProvider設定
useEffect(() => {
(async () => {
await initializedPromise
const info = await voiceChangerClientRef.current!.updateServerSettings(ServerSettingKey.onnxExecutionProvider, settingState.onnxExecutionProvider)
setServerInfo(info)
})()
}, [settingState.onnxExecutionProvider])
// (e) モデルアップロード
const uploadFile = useMemo(() => {
return async (file: File, onprogress: (progress: number, end: boolean) => void) => {
await initializedPromise
const num = await voiceChangerClientRef.current!.uploadFile(file, onprogress)
const res = await voiceChangerClientRef.current!.concatUploadedFile(file, num)
console.log("uploaded", num, res)
}
}, [])
const loadModel = useMemo(() => {
return async () => {
if (!settingState.pyTorchModel && !settingState.onnxModel) {
alert("PyTorchモデルとONNXモデルのどちらか一つ以上指定する必要があります。")
return
}
if (!settingState.configFile) {
alert("Configファイルを指定する必要があります。")
return
}
await initializedPromise
setUploadProgress(0)
setIsUploading(true)
const models = [settingState.pyTorchModel, settingState.onnxModel].filter(x => { return x != null }) as File[]
for (let i = 0; i < models.length; i++) {
const progRate = 1 / models.length
const progOffset = 100 * i * progRate
await uploadFile(models[i], (progress: number, end: boolean) => {
// console.log(progress * progRate + progOffset, end, progRate,)
setUploadProgress(progress * progRate + progOffset)
})
}
await uploadFile(settingState.configFile, (progress: number, end: boolean) => {
console.log(progress, end)
})
const serverInfo = await voiceChangerClientRef.current!.loadModel(settingState.configFile, settingState.pyTorchModel, settingState.onnxModel)
console.log(serverInfo)
setUploadProgress(0)
setIsUploading(false)
}
}, [settingState.pyTorchModel, settingState.onnxModel, settingState.configFile])
// (2-2) device setting
// (a) インプット設定。audio nodes の設定の都合上、バッファサイズの変更も併せて反映させる。
useEffect(() => {
(async () => {
await initializedPromise
if (!settingState.audioInput || settingState.audioInput == "none") {
console.log("[useClient] setup!(1)", settingState.audioInput)
const ms = createDummyMediaStream(props.audioContext!)
await voiceChangerClientRef.current!.setup(ms, settingState.bufferSize, settingState.vfForceDisabled)
} else {
console.log("[useClient] setup!(2)", settingState.audioInput)
await voiceChangerClientRef.current!.setup(settingState.audioInput, settingState.bufferSize, settingState.vfForceDisabled)
}
})()
}, [settingState.audioInput, settingState.bufferSize, settingState.vfForceDisabled])
// (2-3) speaker setting
// (a) srcId設定。
useEffect(() => {
(async () => {
await initializedPromise
const info = await voiceChangerClientRef.current!.updateServerSettings(ServerSettingKey.srcId, "" + settingState.srcId)
setServerInfo(info)
})()
}, [settingState.srcId])
// (b) dstId設定。
useEffect(() => {
(async () => {
await initializedPromise
const info = await voiceChangerClientRef.current!.updateServerSettings(ServerSettingKey.dstId, "" + settingState.dstId)
setServerInfo(info)
})()
}, [settingState.dstId])
// (2-4) convert setting
// (a) input chunk num設定
useEffect(() => {
(async () => {
await initializedPromise
voiceChangerClientRef.current!.setInputChunkNum(settingState.inputChunkNum)
})()
}, [settingState.inputChunkNum])
// (b) convert chunk num設定
useEffect(() => {
(async () => {
await initializedPromise
const info = await voiceChangerClientRef.current!.updateServerSettings(ServerSettingKey.convertChunkNum, "" + settingState.convertChunkNum)
setServerInfo(info)
})()
}, [settingState.convertChunkNum])
// (c) gpu設定
useEffect(() => {
(async () => {
await initializedPromise
const info = await voiceChangerClientRef.current!.updateServerSettings(ServerSettingKey.gpu, "" + settingState.gpu)
setServerInfo(info)
})()
}, [settingState.gpu])
// (d) crossfade設定1
useEffect(() => {
(async () => {
await initializedPromise
const info = await voiceChangerClientRef.current!.updateServerSettings(ServerSettingKey.crossFadeOffsetRate, "" + settingState.crossFadeOffsetRate)
setServerInfo(info)
})()
}, [settingState.crossFadeOffsetRate])
// (e) crossfade設定2
useEffect(() => {
(async () => {
await initializedPromise
const info = await voiceChangerClientRef.current!.updateServerSettings(ServerSettingKey.crossFadeEndRate, "" + settingState.crossFadeEndRate)
setServerInfo(info)
})()
}, [settingState.crossFadeEndRate])
// (2-5) advanced setting
//// VFDisableはinput設定で合わせて設定。
// (a) voice changer mode
useEffect(() => {
(async () => {
await initializedPromise
voiceChangerClientRef.current!.setVoiceChangerMode(settingState.voiceChangerMode)
voiceChangerClientRef.current!.stop()
})()
}, [settingState.voiceChangerMode])
// (2-6) server control
// (1) start
const start = useMemo(() => {
return async () => {
await initializedPromise
voiceChangerClientRef.current!.setServerUrl(settingState.mmvcServerUrl, true)
voiceChangerClientRef.current!.start()
}
}, [settingState.mmvcServerUrl])
// (2) stop
const stop = useMemo(() => {
return async () => {
await initializedPromise
voiceChangerClientRef.current!.stop()
}
}, [])
// (3) get info
// (2-2) 情報リロード
const getInfo = useMemo(() => {
return async () => {
await initializedPromise
const serverSettings = await voiceChangerClientRef.current!.getServerSettings()
const clientSettings = await voiceChangerClientRef.current!.getClientSettings()
setServerInfo(serverSettings)
console.log(serverSettings, clientSettings)
await clientSetting.reloadClientSetting()
await serverSetting.reloadServerInfo()
}
}, [])
}, [clientSetting, serverSetting])
// (x)
useEffect(() => {
if (serverInfo && serverInfo.status == "OK") {
setDisplaySettingState({
...settingState,
convertChunkNum: serverInfo.convertChunkNum,
crossFadeOffsetRate: serverInfo.crossFadeOffsetRate,
crossFadeEndRate: serverInfo.crossFadeEndRate,
gpu: serverInfo.gpu,
srcId: serverInfo.srcId,
dstId: serverInfo.dstId,
framework: serverInfo.framework,
onnxExecutionProvider: serverInfo.providers.length > 0 ? serverInfo.providers[0] as OnnxExecutionProvider : "CPUExecutionProvider"
})
} else {
setDisplaySettingState({
...settingState,
})
}
}, [settingState, serverInfo])
return {
clientInitialized,
bufferingTime,
responseTime,
volume,
uploadProgress,
isUploading,
settingState: displaySettingState,
serverInfo,
setSettingState,
loadModel,
start,
stop,
getInfo,
clientSetting,
workletSetting,
serverSetting,
}
}

184
client/demo/src/hooks/useClientSetting.ts Normal file
View File

@ -0,0 +1,184 @@
import { BufferSize, createDummyMediaStream, DefaultVoiceChangerClientSetting, Protocol, SampleRate, Speaker, VoiceChangerClient, VoiceChangerClientSetting, VoiceChangerMode } from "@dannadori/voice-changer-client-js"
import { useState, useMemo, useRef, useEffect } from "react"
export type UseClientSettingProps = {
voiceChangerClient: VoiceChangerClient | null
audioContext: AudioContext | null
}
export type ClientSettingState = {
setting: VoiceChangerClientSetting;
setServerUrl: (url: string) => void;
setProtocol: (proto: Protocol) => void;
setAudioInput: (audioInput: string | MediaStream | null) => Promise<void>
setBufferSize: (bufferSize: BufferSize) => Promise<void>
setVfForceDisabled: (vfForceDisabled: boolean) => Promise<void>
setInputChunkNum: (num: number) => void;
setVoiceChangerMode: (mode: VoiceChangerMode) => void
setSampleRate: (num: SampleRate) => void
setSpeakers: (speakers: Speaker[]) => void
start: () => Promise<void>
stop: () => Promise<void>
reloadClientSetting: () => Promise<void>
}
export const useClientSetting = (props: UseClientSettingProps): ClientSettingState => {
const settingRef = useRef<VoiceChangerClientSetting>(DefaultVoiceChangerClientSetting)
const [setting, _setSetting] = useState<VoiceChangerClientSetting>(settingRef.current)
//////////////
// 設定
/////////////
const setServerUrl = useMemo(() => {
return (url: string) => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.setServerUrl(url, true)
settingRef.current.mmvcServerUrl = url
_setSetting({ ...settingRef.current })
}
}, [props.voiceChangerClient])
const setProtocol = useMemo(() => {
return (proto: Protocol) => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.setProtocol(proto)
settingRef.current.protocol = proto
_setSetting({ ...settingRef.current })
}
}, [props.voiceChangerClient])
const _setInput = async () => {
if (!props.voiceChangerClient) return
if (!settingRef.current.audioInput || settingRef.current.audioInput == "none") {
console.log("[useClient] setup!(1)", settingRef.current.audioInput)
const ms = createDummyMediaStream(props.audioContext!)
await props.voiceChangerClient.setup(ms, settingRef.current.bufferSize, settingRef.current.forceVfDisable)
} else {
console.log("[useClient] setup!(2)", settingRef.current.audioInput)
await props.voiceChangerClient.setup(settingRef.current.audioInput, settingRef.current.bufferSize, settingRef.current.forceVfDisable)
}
}
const setAudioInput = useMemo(() => {
return async (audioInput: string | MediaStream | null) => {
if (!props.voiceChangerClient) return
settingRef.current.audioInput = audioInput
await _setInput()
_setSetting({ ...settingRef.current })
}
}, [props.voiceChangerClient])
const setBufferSize = useMemo(() => {
return async (bufferSize: BufferSize) => {
if (!props.voiceChangerClient) return
settingRef.current.bufferSize = bufferSize
await _setInput()
_setSetting({ ...settingRef.current })
}
}, [props.voiceChangerClient])
const setVfForceDisabled = useMemo(() => {
return async (vfForceDisabled: boolean) => {
if (!props.voiceChangerClient) return
settingRef.current.forceVfDisable = vfForceDisabled
await _setInput()
_setSetting({ ...settingRef.current })
}
}, [props.voiceChangerClient])
const setInputChunkNum = useMemo(() => {
return (num: number) => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.setInputChunkNum(num)
settingRef.current.inputChunkNum = num
_setSetting({ ...settingRef.current })
}
}, [props.voiceChangerClient])
const setVoiceChangerMode = useMemo(() => {
return (mode: VoiceChangerMode) => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.setVoiceChangerMode(mode)
settingRef.current.voiceChangerMode = mode
_setSetting({ ...settingRef.current })
}
}, [props.voiceChangerClient])
const setSampleRate = useMemo(() => {
return (num: SampleRate) => {
if (!props.voiceChangerClient) return
//props.voiceChangerClient.setSampleRate(num) // Not Implemented
settingRef.current.sampleRate = num
_setSetting({ ...settingRef.current })
}
}, [props.voiceChangerClient])
const setSpeakers = useMemo(() => {
return (speakers: Speaker[]) => {
if (!props.voiceChangerClient) return
settingRef.current.speakers = speakers
_setSetting({ ...settingRef.current })
}
}, [props.voiceChangerClient])
//////////////
// 操作
/////////////
// (1) start
const start = useMemo(() => {
return async () => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.setServerUrl(setting.mmvcServerUrl, true)
props.voiceChangerClient.start()
}
}, [setting.mmvcServerUrl, props.voiceChangerClient])
// (2) stop
const stop = useMemo(() => {
return async () => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.stop()
}
}, [props.voiceChangerClient])
const reloadClientSetting = useMemo(() => {
return async () => {
if (!props.voiceChangerClient) return
await props.voiceChangerClient.getClientSettings()
}
}, [props.voiceChangerClient])
//////////////
// Colab対応
/////////////
useEffect(() => {
const params = new URLSearchParams(location.search);
const colab = params.get("colab")
if (colab == "true") {
settingRef.current.protocol = "rest"
settingRef.current.inputChunkNum = 64
_setSetting({ ...settingRef.current })
}
}, [])
return {
setting,
setServerUrl,
setProtocol,
setAudioInput,
setBufferSize,
setVfForceDisabled,
setInputChunkNum,
setVoiceChangerMode,
setSampleRate,
setSpeakers,
start,
stop,
reloadClientSetting
}
}

217
client/demo/src/hooks/useServerSetting.ts Normal file
View File

@ -0,0 +1,217 @@
import { DefaultVoiceChangerServerSetting, Framework, OnnxExecutionProvider, ServerInfo, ServerSettingKey, VoiceChangerClient, VoiceChangerServerSetting, } from "@dannadori/voice-changer-client-js"
import { useState, useMemo, useRef, } from "react"
export type FileUploadSetting = {
pyTorchModel: File | null
configFile: File | null
onnxModel: File | null
}
const InitialFileUploadSetting: FileUploadSetting = {
pyTorchModel: null,
configFile: null,
onnxModel: null,
}
export type UseServerSettingProps = {
voiceChangerClient: VoiceChangerClient | null
}
export type ServerSettingState = {
setting: VoiceChangerServerSetting;
serverInfo: ServerInfo | undefined;
fileUploadSetting: FileUploadSetting
setFramework: (framework: Framework) => Promise<boolean>;
setOnnxExecutionProvider: (provider: OnnxExecutionProvider) => Promise<boolean>;
setSrcId: (num: number) => Promise<boolean>;
setDstId: (num: number) => Promise<boolean>;
setConvertChunkNum: (num: number) => Promise<boolean>;
setGpu: (num: number) => Promise<boolean>;
setCrossFadeOffsetRate: (num: number) => Promise<boolean>;
setCrossFadeEndRate: (num: number) => Promise<boolean>;
setCrossFadeOverlapRate: (num: number) => Promise<boolean>;
reloadServerInfo: () => Promise<void>;
setFileUploadSetting: (val: FileUploadSetting) => void
loadModel: () => Promise<void>
uploadProgress: number
isUploading: boolean
}
export const useServerSetting = (props: UseServerSettingProps): ServerSettingState => {
const settingRef = useRef<VoiceChangerServerSetting>(DefaultVoiceChangerServerSetting)
const [setting, _setSetting] = useState<VoiceChangerServerSetting>(settingRef.current)
const [serverInfo, _setServerInfo] = useState<ServerInfo>()
const [fileUploadSetting, setFileUploadSetting] = useState<FileUploadSetting>(InitialFileUploadSetting)
//////////////
// 設定
/////////////
//// サーバに設定後、反映された情報と照合して値が一致していることを確認。一致していない場合はalert
const _set_and_store = async (key: ServerSettingKey, newVal: string) => {
if (!props.voiceChangerClient) return false
const res = await props.voiceChangerClient.updateServerSettings(key, "" + newVal)
_setServerInfo(res)
if (newVal == res[key]) {
_setSetting({
...settingRef.current,
convertChunkNum: res.convertChunkNum,
srcId: res.srcId,
dstId: res.dstId,
gpu: res.gpu,
crossFadeOffsetRate: res.crossFadeOffsetRate,
crossFadeEndRate: res.crossFadeEndRate,
crossFadeOverlapRate: res.crossFadeOverlapRate,
framework: res.framework,
onnxExecutionProvider: (!!res.onnxExecutionProvider && res.onnxExecutionProvider.length > 0) ? res.onnxExecutionProvider[0] as OnnxExecutionProvider : DefaultVoiceChangerServerSetting.onnxExecutionProvider
})
return true
} else {
alert(`[ServerSetting] setting failed. [key:${key}, new:${newVal}, res:${res[key]}]`)
return false
}
}
const setFramework = useMemo(() => {
return async (framework: Framework) => {
return await _set_and_store(ServerSettingKey.framework, "" + framework)
}
}, [props.voiceChangerClient])
const setOnnxExecutionProvider = useMemo(() => {
return async (provider: OnnxExecutionProvider) => {
return await _set_and_store(ServerSettingKey.onnxExecutionProvider, "" + provider)
}
}, [props.voiceChangerClient])
const setSrcId = useMemo(() => {
return async (num: number) => {
return await _set_and_store(ServerSettingKey.srcId, "" + num)
}
}, [props.voiceChangerClient])
const setDstId = useMemo(() => {
return async (num: number) => {
return await _set_and_store(ServerSettingKey.dstId, "" + num)
}
}, [props.voiceChangerClient])
const setConvertChunkNum = useMemo(() => {
return async (num: number) => {
return await _set_and_store(ServerSettingKey.convertChunkNum, "" + num)
}
}, [props.voiceChangerClient])
const setGpu = useMemo(() => {
return async (num: number) => {
return await _set_and_store(ServerSettingKey.gpu, "" + num)
}
}, [props.voiceChangerClient])
const setCrossFadeOffsetRate = useMemo(() => {
return async (num: number) => {
return await _set_and_store(ServerSettingKey.crossFadeOffsetRate, "" + num)
}
}, [props.voiceChangerClient])
const setCrossFadeEndRate = useMemo(() => {
return async (num: number) => {
return await _set_and_store(ServerSettingKey.crossFadeEndRate, "" + num)
}
}, [props.voiceChangerClient])
const setCrossFadeOverlapRate = useMemo(() => {
return async (num: number) => {
return await _set_and_store(ServerSettingKey.crossFadeOverlapRate, "" + num)
}
}, [props.voiceChangerClient])
//////////////
// 操作
/////////////
const [uploadProgress, setUploadProgress] = useState<number>(0)
const [isUploading, setIsUploading] = useState<boolean>(false)
// (e) モデルアップロード
const _uploadFile = useMemo(() => {
return async (file: File, onprogress: (progress: number, end: boolean) => void) => {
if (!props.voiceChangerClient) return
const num = await props.voiceChangerClient.uploadFile(file, onprogress)
const res = await props.voiceChangerClient.concatUploadedFile(file, num)
console.log("uploaded", num, res)
}
}, [props.voiceChangerClient])
const loadModel = useMemo(() => {
return async () => {
if (!fileUploadSetting.pyTorchModel && !fileUploadSetting.onnxModel) {
alert("PyTorchモデルとONNXモデルのどちらか一つ以上指定する必要があります。")
return
}
if (!fileUploadSetting.configFile) {
alert("Configファイルを指定する必要があります。")
return
}
if (!props.voiceChangerClient) return
setUploadProgress(0)
setIsUploading(true)
const models = [fileUploadSetting.pyTorchModel, fileUploadSetting.onnxModel].filter(x => { return x != null }) as File[]
for (let i = 0; i < models.length; i++) {
const progRate = 1 / models.length
const progOffset = 100 * i * progRate
await _uploadFile(models[i], (progress: number, end: boolean) => {
// console.log(progress * progRate + progOffset, end, progRate,)
setUploadProgress(progress * progRate + progOffset)
})
}
await _uploadFile(fileUploadSetting.configFile, (progress: number, end: boolean) => {
console.log(progress, end)
})
const serverInfo = await props.voiceChangerClient.loadModel(fileUploadSetting.configFile, fileUploadSetting.pyTorchModel, fileUploadSetting.onnxModel)
console.log(serverInfo)
setUploadProgress(0)
setIsUploading(false)
}
}, [fileUploadSetting, props.voiceChangerClient])
const reloadServerInfo = useMemo(() => {
return async () => {
if (!props.voiceChangerClient) return
const res = await props.voiceChangerClient.getServerSettings()
_setServerInfo(res)
_setSetting({
...settingRef.current,
convertChunkNum: res.convertChunkNum,
srcId: res.srcId,
dstId: res.dstId,
gpu: res.gpu,
crossFadeOffsetRate: res.crossFadeOffsetRate,
crossFadeEndRate: res.crossFadeEndRate,
crossFadeOverlapRate: res.crossFadeOverlapRate,
framework: res.framework,
onnxExecutionProvider: (!!res.onnxExecutionProvider && res.onnxExecutionProvider.length > 0) ? res.onnxExecutionProvider[0] as OnnxExecutionProvider : DefaultVoiceChangerServerSetting.onnxExecutionProvider
})
}
}, [props.voiceChangerClient])
return {
setting,
serverInfo,
fileUploadSetting,
setFramework,
setOnnxExecutionProvider,
setSrcId,
setDstId,
setConvertChunkNum,
setGpu,
setCrossFadeOffsetRate,
setCrossFadeEndRate,
setCrossFadeOverlapRate,
reloadServerInfo,
setFileUploadSetting,
loadModel,
uploadProgress,
isUploading,
}
}

29
client/demo/src/hooks/useWorkletSetting.ts Normal file
View File

@ -0,0 +1,29 @@
import { DefaultWorkletSetting, VoiceChangerClient, WorkletSetting } from "@dannadori/voice-changer-client-js"
import { useState, useMemo } from "react"
export type UseWorkletSettingProps = {
voiceChangerClient: VoiceChangerClient | null
}
export type WorkletSettingState = {
setting: WorkletSetting;
setSetting: (setting: WorkletSetting) => void;
}
export const useWorkletSetting = (props: UseWorkletSettingProps): WorkletSettingState => {
const [setting, _setSetting] = useState<WorkletSetting>(DefaultWorkletSetting)
const setSetting = useMemo(() => {
return (setting: WorkletSetting) => {
if (!props.voiceChangerClient) return
props.voiceChangerClient.configureWorklet(setting)
_setSetting(setting)
}
}, [props.voiceChangerClient])
return {
setting,
setSetting
}
}

4
client/lib/package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "@dannadori/voice-changer-client-js",
"version": "1.0.5",
"version": "1.0.6",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@dannadori/voice-changer-client-js",
"version": "1.0.5",
"version": "1.0.6",
"license": "ISC",
"dependencies": {
"@types/readable-stream": "^2.3.15",

2
client/lib/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@dannadori/voice-changer-client-js",
"version": "1.0.5",
"version": "1.0.6",
"description": "",
"main": "dist/index.js",
"directories": {

12
client/lib/src/@types/voice-changer-worklet-processor.d.ts vendored Normal file
View File

@ -0,0 +1,12 @@
export declare const RequestType: {
readonly voice: "voice";
readonly config: "config";
};
export type RequestType = typeof RequestType[keyof typeof RequestType];
export type VoiceChangerWorkletProcessorRequest = {
requestType: RequestType;
voice: ArrayBuffer;
numTrancateTreshold: number;
volTrancateThreshold: number;
volTrancateLength: number;
};

32
client/lib/src/VoiceChangerClient.ts
View File

@ -3,11 +3,11 @@ import { VoiceChangerWorkletNode, VolumeListener } from "./VoiceChangerWorkletNo
import workerjs from "raw-loader!../worklet/dist/index.js";
import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js";
import { createDummyMediaStream, validateUrl } from "./util";
import { BufferSize, DefaultVoiceChangerOptions, Protocol, ServerSettingKey, VoiceChangerMode, VOICE_CHANGER_CLIENT_EXCEPTION } from "./const";
import { BufferSize, DefaultVoiceChangerClientSetting, Protocol, ServerSettingKey, VoiceChangerMode, VOICE_CHANGER_CLIENT_EXCEPTION, WorkletSetting } from "./const";
import MicrophoneStream from "microphone-stream";
import { AudioStreamer, Callbacks, AudioStreamerListeners } from "./AudioStreamer";
import { ServerConfigurator } from "./ServerConfigurator";
import { VoiceChangerWorkletProcessorRequest } from "./@types/voice-changer-worklet-processor";
// オーディオデータの流れ
// input node(mic or MediaStream) -> [vf node] -> microphne stream -> audio streamer ->
@ -15,7 +15,7 @@ import { ServerConfigurator } from "./ServerConfigurator";
export class VoiceChnagerClient {
export class VoiceChangerClient {
private configurator: ServerConfigurator
private ctx: AudioContext
private vfEnable = false
@ -39,7 +39,15 @@ export class VoiceChnagerClient {
onVoiceReceived: (voiceChangerMode: VoiceChangerMode, data: ArrayBuffer): void => {
// console.log(voiceChangerMode, data)
if (voiceChangerMode === "realtime") {
this.vcNode.postReceivedVoice(data)
const req: VoiceChangerWorkletProcessorRequest = {
requestType: "voice",
voice: data,
numTrancateTreshold: 0,
volTrancateThreshold: 0,
volTrancateLength: 0
}
this.vcNode.postReceivedVoice(req)
return
}
@ -77,8 +85,8 @@ export class VoiceChnagerClient {
this.vcNode.connect(this.currentMediaStreamAudioDestinationNode) // vc node -> output node
// (vc nodeにはaudio streamerのcallbackでデータが投げ込まれる)
this.audioStreamer = new AudioStreamer(this.callbacks, audioStreamerListeners, { objectMode: true, })
this.audioStreamer.setInputChunkNum(DefaultVoiceChangerOptions.inputChunkNum)
this.audioStreamer.setVoiceChangerMode(DefaultVoiceChangerOptions.voiceChangerMode)
this.audioStreamer.setInputChunkNum(DefaultVoiceChangerClientSetting.inputChunkNum)
this.audioStreamer.setVoiceChangerMode(DefaultVoiceChangerClientSetting.voiceChangerMode)
if (this.vfEnable) {
this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' })
@ -201,6 +209,18 @@ export class VoiceChnagerClient {
this.audioStreamer.setVoiceChangerMode(val)
}
// configure worklet
configureWorklet = (setting: WorkletSetting) => {
const req: VoiceChangerWorkletProcessorRequest = {
requestType: "config",
voice: new ArrayBuffer(1),
numTrancateTreshold: setting.numTrancateTreshold,
volTrancateThreshold: setting.volTrancateThreshold,
volTrancateLength: setting.volTrancateLength
}
this.vcNode.postReceivedVoice(req)
}
// Configurator Method
uploadFile = (file: File, onprogress: (progress: number, end: boolean) => void) => {
return this.configurator.uploadFile(file, onprogress)

8
client/lib/src/VoiceChangerWorkletNode.ts
View File

@ -1,3 +1,5 @@
import { VoiceChangerWorkletProcessorRequest } from "./@types/voice-changer-worklet-processor";
export type VolumeListener = {
notifyVolume: (vol: number) => void
}
@ -11,10 +13,10 @@ export class VoiceChangerWorkletNode extends AudioWorkletNode {
console.log(`[worklet_node][voice-changer-worklet-processor] created.`);
}
postReceivedVoice = (data: ArrayBuffer) => {
postReceivedVoice = (req: VoiceChangerWorkletProcessorRequest) => {
this.port.postMessage({
data: data,
}, [data]);
request: req
}, [req.voice]);
}
handleMessage(event: any) {

39
client/lib/src/const.ts
View File

@ -1,10 +1,11 @@
// (★1) chunk sizeは 128サンプル, 256byte(int16)と定義。
// (★2) 256byte(最低バッファサイズ256から間引いた個数x2byte)をchunkとして管理。
// 24000sample -> 1sec, 128sample(1chunk) -> 5.333msec
// 187.5chunk -> 1sec
// types
export type VoiceChangerRequestParamas = {
export type VoiceChangerServerSetting = {
convertChunkNum: number, // VITSに入力する変換サイズ。(入力データの2倍以上の大きさで指定。それより小さいものが指定された場合は、サーバ側で自動的に入力の2倍のサイズが設定される。)
srcId: number,
dstId: number,
@ -13,10 +14,13 @@ export type VoiceChangerRequestParamas = {
crossFadeLowerValue: number,
crossFadeOffsetRate: number,
crossFadeEndRate: number,
crossFadeOverlapRate: number,
framework: Framework
onnxExecutionProvider: OnnxExecutionProvider,
}
export type VoiceChangerOptions = {
export type VoiceChangerClientSetting = {
audioInput: string | MediaStream | null,
mmvcServerUrl: string,
protocol: Protocol,
@ -26,10 +30,13 @@ export type VoiceChangerOptions = {
speakers: Speaker[],
forceVfDisable: boolean,
voiceChangerMode: VoiceChangerMode,
onnxExecutionProvider: OnnxExecutionProvider,
framework: Framework
}
export type WorkletSetting = {
numTrancateTreshold: number,
volTrancateThreshold: number,
volTrancateLength: number
}
export type Speaker = {
"id": number,
@ -45,11 +52,12 @@ export type ServerInfo = {
convertChunkNum: number,
crossFadeOffsetRate: number,
crossFadeEndRate: number,
crossFadeOverlapRate: number,
gpu: number,
srcId: number,
dstId: number,
framework: Framework,
providers: string[]
onnxExecutionProvider: string[]
}
@ -106,23 +114,28 @@ export const ServerSettingKey = {
"gpu": "gpu",
"crossFadeOffsetRate": "crossFadeOffsetRate",
"crossFadeEndRate": "crossFadeEndRate",
"crossFadeOverlapRate": "crossFadeOverlapRate",
"framework": "framework",
"onnxExecutionProvider": "onnxExecutionProvider"
} as const
export type ServerSettingKey = typeof ServerSettingKey[keyof typeof ServerSettingKey]
// Defaults
export const DefaultVoiceChangerRequestParamas: VoiceChangerRequestParamas = {
export const DefaultVoiceChangerServerSetting: VoiceChangerServerSetting = {
convertChunkNum: 32, //（★１）
srcId: 107,
dstId: 100,
gpu: 0,
crossFadeLowerValue: 0.1,
crossFadeOffsetRate: 0.1,
crossFadeEndRate: 0.9
crossFadeEndRate: 0.9,
crossFadeOverlapRate: 0.5,
framework: "PyTorch",
onnxExecutionProvider: "CPUExecutionProvider"
}
export const DefaultVoiceChangerOptions: VoiceChangerOptions = {
export const DefaultVoiceChangerClientSetting: VoiceChangerClientSetting = {
audioInput: null,
mmvcServerUrl: "",
protocol: "sio",
@ -153,11 +166,13 @@ export const DefaultVoiceChangerOptions: VoiceChangerOptions = {
],
forceVfDisable: false,
voiceChangerMode: "realtime",
framework: "PyTorch",
onnxExecutionProvider: "CPUExecutionProvider"
}
export const DefaultWorkletSetting: WorkletSetting = {
numTrancateTreshold: 188,
volTrancateThreshold: 0.0005,
volTrancateLength: 32
}
export const VOICE_CHANGER_CLIENT_EXCEPTION = {
ERR_SIO_CONNECT_FAILED: "ERR_SIO_CONNECT_FAILED",

2
client/lib/tsconfig.json
View File

@ -29,6 +29,6 @@
"skipLibCheck": true
},
/* tscコマンドで読み込むファイルを指定 */
"include": ["src/*.ts"],
"include": ["src/**/*.ts"],
"exclude": ["node_modules"]
}

2
client/lib/tsconfig.worklet.json
View File

@ -2,6 +2,8 @@
"compilerOptions": {
"target": "ES2020",
"lib":["ES2020"],
"outDir": "./worklet/dist",
"declaration": true,
/* ファイル名の大文字小文字を区別 */
"forceConsistentCasingInFileNames": true,

73
client/lib/worklet/src/voice-changer-worklet-processor.ts
View File

@ -1,8 +1,26 @@
export const RequestType = {
"voice": "voice",
"config": "config"
} as const
export type RequestType = typeof RequestType[keyof typeof RequestType]
export type VoiceChangerWorkletProcessorRequest = {
requestType: RequestType,
voice: ArrayBuffer,
numTrancateTreshold: number
volTrancateThreshold: number
volTrancateLength: number
}
class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
private BLOCK_SIZE = 128
private initialized = false;
private volume = 0
private numTrancateTreshold = 150
private volTrancateThreshold = 0.0005
private volTrancateLength = 32
private volTrancateCount = 0
playBuffer: Float32Array[] = []
/**
* @constructor
@ -13,9 +31,25 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
this.port.onmessage = this.handleMessage.bind(this);
}
calcVol = (data: Float32Array, prevVol: number) => {
const sum = data.reduce((prev, cur) => {
return prev + cur * cur
}, 0)
const rms = Math.sqrt(sum / data.length)
return Math.max(rms, prevVol * 0.95)
}
handleMessage(event: any) {
// noop
const arrayBuffer = event.data.data as ArrayBuffer
const request = event.data.request as VoiceChangerWorkletProcessorRequest
if (request.requestType === "config") {
this.numTrancateTreshold = request.numTrancateTreshold
this.volTrancateLength = request.volTrancateLength
this.volTrancateThreshold = request.volTrancateThreshold
console.log("[worklet] worklet configured", request)
return
}
const arrayBuffer = request.voice
// データは(int16)で受信
const i16Data = new Int16Array(arrayBuffer)
const f32Data = new Float32Array(i16Data.length)
@ -25,7 +59,7 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
f32Data[i] = float
})
if (this.playBuffer.length > 50) {
if (this.playBuffer.length > this.numTrancateTreshold) {
console.log("[worklet] Buffer truncated")
while (this.playBuffer.length > 2) {
this.playBuffer.shift()
@ -62,19 +96,32 @@ class VoiceChangerWorkletProcessor extends AudioWorkletProcessor {
return true
}
const data = this.playBuffer.shift()!
const sum = data.reduce((prev, cur) => {
return prev + cur * cur
}, 0)
const rms = Math.sqrt(sum / data.length)
this.volume = Math.max(rms, this.volume * 0.95)
this.port.postMessage({ volume: this.volume });
//// 一定期間無音状態が続いている場合はスキップ。
let voice: Float32Array | undefined
while (true) {
voice = this.playBuffer.shift()
if (!voice) {
break
}
this.volume = this.calcVol(voice, this.volume)
if (this.volume < this.volTrancateThreshold) {
this.volTrancateCount += 1
} else {
this.volTrancateCount = 0
}
if (this.volTrancateCount < this.volTrancateLength || this.volTrancateLength < 0) {
break
} else {
// console.log("silent...skip")
}
}
outputs[0][0].set(data)
if (voice) {
this.port.postMessage({ volume: this.volume });
outputs[0][0].set(voice)
}
return true;
}

3
server/MMVCServerSIO.py
View File

@ -78,6 +78,9 @@ if __name__ == thisFilename or args.colab == True:
MODEL = args.m if args.m != None else None
ONNX_MODEL = args.o if args.o != None else None
if args.colab == True:
os.environ["colab"] = "True"
# if os.getenv("EX_TB_PORT"):
# EX_TB_PORT = os.environ["EX_TB_PORT"]
# exApplitionInfo.external_tensorboard_port = int(EX_TB_PORT)

80
server/voice_changer/VoiceChanger.py
View File

@ -24,6 +24,7 @@ class VocieChangerSettings():
dstId:int = 100
crossFadeOffsetRate:float = 0.1
crossFadeEndRate:float = 0.9
crossFadeOverlapRate:float = 0.9
convertChunkNum:int = 32
framework:str = "PyTorch" # PyTorch or ONNX
pyTorchModelFile:str = ""
@ -31,7 +32,7 @@ class VocieChangerSettings():
configFile:str = ""
# ↓mutableな物だけ列挙
intData = ["gpu","srcId", "dstId", "convertChunkNum"]
floatData = [ "crossFadeOffsetRate", "crossFadeEndRate",]
floatData = [ "crossFadeOffsetRate", "crossFadeEndRate", "crossFadeOverlapRate"]
strData = ["framework"]
class VoiceChanger():
@ -44,6 +45,8 @@ class VoiceChanger():
self.onnx_session = None
self.currentCrossFadeOffsetRate=0
self.currentCrossFadeEndRate=0
self.currentCrossFadeOverlapRate=0
# 共通で使用する情報を収集
self.hps = utils.get_hparams_from_file(config)
self.gpu_num = torch.cuda.device_count()
@ -92,7 +95,7 @@ class VoiceChanger():
def get_info(self):
data = asdict(self.settings)
data["providers"] = self.onnx_session.get_providers() if self.onnx_session != None else []
data["onnxExecutionProvider"] = self.onnx_session.get_providers() if self.onnx_session != None else []
files = ["configFile", "pyTorchModelFile", "onnxModelFile"]
for f in files:
if data[f]!=None and os.path.exists(data[f]):
@ -133,20 +136,24 @@ class VoiceChanger():
def _generate_strength(self, unpackedData):
if self.unpackedData_length != unpackedData.shape[0] or self.currentCrossFadeOffsetRate != self.settings.crossFadeOffsetRate or self.currentCrossFadeEndRate != self.settings.crossFadeEndRate :
if self.unpackedData_length != unpackedData.shape[0] or self.currentCrossFadeOffsetRate != self.settings.crossFadeOffsetRate or self.currentCrossFadeEndRate != self.settings.crossFadeEndRate or self.currentCrossFadeOverlapRate != self.settings.crossFadeOverlapRate:
self.unpackedData_length = unpackedData.shape[0]
self.currentCrossFadeOffsetRate = self.settings.crossFadeOffsetRate
self.currentCrossFadeEndRate = self.settings.crossFadeEndRate
cf_offset = int(unpackedData.shape[0] * self.settings.crossFadeOffsetRate)
cf_end = int(unpackedData.shape[0] * self.settings.crossFadeEndRate)
self.currentCrossFadeOverlapRate = self.settings.crossFadeOverlapRate
overlapSize = int(unpackedData.shape[0] * self.settings.crossFadeOverlapRate)
cf_offset = int(overlapSize * self.settings.crossFadeOffsetRate)
cf_end = int(overlapSize * self.settings.crossFadeEndRate)
cf_range = cf_end - cf_offset
percent = np.arange(cf_range) / cf_range
np_prev_strength = np.cos(percent * 0.5 * np.pi) ** 2
np_cur_strength = np.cos((1-percent) * 0.5 * np.pi) ** 2
self.np_prev_strength = np.concatenate([np.ones(cf_offset), np_prev_strength, np.zeros(unpackedData.shape[0]-cf_offset-len(np_prev_strength))])
self.np_cur_strength = np.concatenate([np.zeros(cf_offset), np_cur_strength, np.ones(unpackedData.shape[0]-cf_offset-len(np_cur_strength))])
self.np_prev_strength = np.concatenate([np.ones(cf_offset), np_prev_strength, np.zeros(overlapSize - cf_offset - len(np_prev_strength))])
self.np_cur_strength = np.concatenate([np.zeros(cf_offset), np_cur_strength, np.ones(overlapSize - cf_offset - len(np_cur_strength))])
self.prev_strength = torch.FloatTensor(self.np_prev_strength)
self.cur_strength = torch.FloatTensor(self.np_cur_strength)
@ -199,16 +206,19 @@ class VoiceChanger():
"sid_tgt": sid_tgt1.numpy()
})[0][0,0] * self.hps.data.max_wav_value
if hasattr(self, 'np_prev_audio1') == True:
prev = self.np_prev_audio1[-1*inputSize:]
cur = audio1[-2*inputSize:-1*inputSize]
# print(prev.shape, self.np_prev_strength.shape, cur.shape, self.np_cur_strength.shape)
powered_prev = prev * self.np_prev_strength
powered_cur = cur * self.np_cur_strength
result = powered_prev + powered_cur
#result = prev * self.np_prev_strength + cur * self.np_cur_strength
overlapSize = int(inputSize * self.settings.crossFadeOverlapRate)
prev_overlap = self.np_prev_audio1[-1*overlapSize:]
cur_overlap = audio1[-1*(inputSize + overlapSize) :-1*inputSize]
# print(prev_overlap.shape, self.np_prev_strength.shape, cur_overlap.shape, self.np_cur_strength.shape)
# print(">>>>>>>>>>>", -1*(inputSize + overlapSize) , -1*inputSize)
powered_prev = prev_overlap * self.np_prev_strength
powered_cur = cur_overlap * self.np_cur_strength
powered_result = powered_prev + powered_cur
cur = audio1[-1*inputSize:-1*overlapSize]
result = np.concatenate([powered_result, cur],axis=0)
else:
cur = audio1[-2*inputSize:-1*inputSize]
result = cur
result = np.zeros(1).astype(np.int16)
self.np_prev_audio1 = audio1
return result
@ -230,10 +240,17 @@ class VoiceChanger():
print(f"cur_strength move from {self.cur_strength.device} to cpu")
self.cur_strength = self.cur_strength.cpu()
if hasattr(self, 'prev_audio1') == True and self.prev_audio1.device == torch.device('cpu'):
prev = self.prev_audio1[-1*inputSize:]
cur = audio1[-2*inputSize:-1*inputSize]
result = prev * self.prev_strength + cur * self.cur_strength
if hasattr(self, 'prev_audio1') == True and self.prev_audio1.device == torch.device('cpu'): # prev_audio1が所望のデバイスに無い場合は一回休み。
overlapSize = int(inputSize * self.settings.crossFadeOverlapRate)
prev_overlap = self.prev_audio1[-1*overlapSize:]
cur_overlap = audio1[-1*(inputSize + overlapSize) :-1*inputSize]
powered_prev = prev_overlap * self.prev_strength
powered_cur = cur_overlap * self.cur_strength
powered_result = powered_prev + powered_cur
cur = audio1[-1*inputSize:-1*overlapSize] # 今回のインプットの生部分。(インプット - 次回のCrossfade部分)。
result = torch.cat([powered_result, cur],axis=0) # Crossfadeと今回のインプットの生部分を結合
else:
cur = audio1[-2*inputSize:-1*inputSize]
result = cur
@ -257,27 +274,32 @@ class VoiceChanger():
if hasattr(self, 'prev_audio1') == True and self.prev_audio1.device == torch.device('cuda', self.settings.gpu):
prev = self.prev_audio1[-1*inputSize:]
cur = audio1[-2*inputSize:-1*inputSize]
result = prev * self.prev_strength + cur * self.cur_strength
# print("merging...", prev.shape, cur.shape)
overlapSize = int(inputSize * self.settings.crossFadeOverlapRate)
prev_overlap = self.prev_audio1[-1*overlapSize:]
cur_overlap = audio1[-1*(inputSize + overlapSize) :-1*inputSize]
powered_prev = prev_overlap * self.prev_strength
powered_cur = cur_overlap * self.cur_strength
powered_result = powered_prev + powered_cur
cur = audio1[-1*inputSize:-1*overlapSize] # 今回のインプットの生部分。(インプット - 次回のCrossfade部分)。
result = torch.cat([powered_result, cur],axis=0) # Crossfadeと今回のインプットの生部分を結合
else:
cur = audio1[-2*inputSize:-1*inputSize]
result = cur
# print("no merging...", cur.shape)
self.prev_audio1 = audio1
#print(result)
result = result.cpu().float().numpy()
return result
def on_request(self, unpackedData:any):
convertSize = self.settings.convertChunkNum * 128 # 128sample/1chunk
if unpackedData.shape[0] * 2 > convertSize:
convertSize = unpackedData.shape[0] * 2
# print("convert Size", convertChunkNum, convertSize)
if unpackedData.shape[0]*(1 + self.settings.crossFadeOverlapRate) + 1024 > convertSize:
convertSize = int(unpackedData.shape[0]*(1 + self.settings.crossFadeOverlapRate)) + 1024
# print("convert Size", unpackedData.shape[0], unpackedData.shape[0]*(1 + self.settings.crossFadeOverlapRate), convertSize)
self._generate_strength(unpackedData)
data = self._generate_input(unpackedData, convertSize)