import { VoiceChangerWorkletNode, VoiceChangerWorkletListener } from "./VoiceChangerWorkletNode"; // @ts-ignore import workerjs from "raw-loader!../worklet/dist/index.js"; import { VoiceFocusDeviceTransformer, VoiceFocusTransformDevice } from "amazon-chime-sdk-js"; import { createDummyMediaStream, validateUrl } from "./util"; import { DefaultClientSettng, MergeModelRequest, ServerSettingKey, VoiceChangerClientSetting, WorkletNodeSetting, WorkletSetting } from "./const"; import { ServerConfigurator } from "./ServerConfigurator"; // オーディオデータの流れ // input node(mic or MediaStream) -> [vf node] -> [vc node] -> // sio/rest server -> [vc node] -> output node import { BlockingQueue } from "./utils/BlockingQueue"; export class VoiceChangerClient { private configurator: ServerConfigurator private ctx: AudioContext private vfEnable = false private vf: VoiceFocusDeviceTransformer | null = null private currentDevice: VoiceFocusTransformDevice | null = null private currentMediaStream: MediaStream | null = null private currentMediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null private inputGainNode: GainNode | null = null private outputGainNode: GainNode | null = null private monitorGainNode: GainNode | null = null private vcInNode!: VoiceChangerWorkletNode private vcOutNode!: VoiceChangerWorkletNode private currentMediaStreamAudioDestinationNode!: MediaStreamAudioDestinationNode private currentMediaStreamAudioDestinationMonitorNode!: MediaStreamAudioDestinationNode private promiseForInitialize: Promise private _isVoiceChanging = false private setting: VoiceChangerClientSetting = DefaultClientSettng.voiceChangerClientSetting private sslCertified: string[] = [] private sem = new BlockingQueue(); constructor(ctx: AudioContext, vfEnable: boolean, voiceChangerWorkletListener: VoiceChangerWorkletListener) { this.sem.enqueue(0); this.configurator = new ServerConfigurator() this.ctx = ctx this.vfEnable = vfEnable this.promiseForInitialize = new Promise(async (resolve) => { const scriptUrl = URL.createObjectURL(new Blob([workerjs], { type: "text/javascript" })); // await this.ctx.audioWorklet.addModule(scriptUrl) // this.vcInNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node try { this.vcInNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node } catch (err) { await this.ctx.audioWorklet.addModule(scriptUrl) this.vcInNode = new VoiceChangerWorkletNode(this.ctx, voiceChangerWorkletListener); // vc node } // const ctx44k = new AudioContext({ sampleRate: 44100 }) // これでもプチプチが残る const ctx44k = new AudioContext({ sampleRate: 48000 }) // 結局これが一番まし。 console.log("audio out:", ctx44k) try { this.vcOutNode = new VoiceChangerWorkletNode(ctx44k, voiceChangerWorkletListener); // vc node } catch (err) { await ctx44k.audioWorklet.addModule(scriptUrl) this.vcOutNode = new VoiceChangerWorkletNode(ctx44k, voiceChangerWorkletListener); // vc node } this.currentMediaStreamAudioDestinationNode = ctx44k.createMediaStreamDestination() // output node this.outputGainNode = ctx44k.createGain() this.outputGainNode.gain.value = this.setting.outputGain this.vcOutNode.connect(this.outputGainNode) // vc node -> output node this.outputGainNode.connect(this.currentMediaStreamAudioDestinationNode) this.currentMediaStreamAudioDestinationMonitorNode = ctx44k.createMediaStreamDestination() // output node this.monitorGainNode = ctx44k.createGain() this.monitorGainNode.gain.value = this.setting.monitorGain this.vcOutNode.connect(this.monitorGainNode) // vc node -> monitor node this.monitorGainNode.connect(this.currentMediaStreamAudioDestinationMonitorNode) if (this.vfEnable) { this.vf = await VoiceFocusDeviceTransformer.create({ variant: 'c20' }) const dummyMediaStream = createDummyMediaStream(this.ctx) this.currentDevice = (await this.vf.createTransformDevice(dummyMediaStream)) || null; } resolve() }) } private lock = async () => { const num = await this.sem.dequeue(); return num; }; private unlock = (num: number) => { this.sem.enqueue(num + 1); }; isInitialized = async () => { if (this.promiseForInitialize) { await this.promiseForInitialize } return true } ///////////////////////////////////////////////////// // オペレーション ///////////////////////////////////////////////////// /// Operations /// setup = async () => { const lockNum = await this.lock() console.log(`Input Setup=> echo: ${this.setting.echoCancel}, noise1: ${this.setting.noiseSuppression}, noise2: ${this.setting.noiseSuppression2}`) // condition check if (!this.vcInNode) { console.warn("vc node is not initialized.") throw "vc node is not initialized." } // Main Process //// shutdown & re-generate mediastream if (this.currentMediaStream) { this.currentMediaStream.getTracks().forEach(x => { x.stop() }) this.currentMediaStream = null } //// Input デバイスがnullの時はmicStreamを止めてリターン if (!this.setting.audioInput) { console.log(`Input Setup=> client mic is disabled. ${this.setting.audioInput}`) this.vcInNode.stop() await this.unlock(lockNum) return } if (typeof this.setting.audioInput == "string") { try { if (this.setting.audioInput == "none") { this.currentMediaStream = createDummyMediaStream(this.ctx) } else { this.currentMediaStream = await navigator.mediaDevices.getUserMedia({ audio: { deviceId: this.setting.audioInput, channelCount: 1, sampleRate: this.setting.sampleRate, sampleSize: 16, autoGainControl: false, echoCancellation: this.setting.echoCancel, noiseSuppression: this.setting.noiseSuppression } }) } } catch (e) { console.warn(e) this.vcInNode.stop() await this.unlock(lockNum) throw e } // this.currentMediaStream.getAudioTracks().forEach((x) => { // console.log("MIC Setting(cap)", x.getCapabilities()) // console.log("MIC Setting(const)", x.getConstraints()) // console.log("MIC Setting(setting)", x.getSettings()) // }) } else { this.currentMediaStream = this.setting.audioInput } // connect nodes. this.currentMediaStreamAudioSourceNode = this.ctx.createMediaStreamSource(this.currentMediaStream) this.inputGainNode = this.ctx.createGain() this.inputGainNode.gain.value = this.setting.inputGain this.currentMediaStreamAudioSourceNode.connect(this.inputGainNode) if (this.currentDevice && this.setting.noiseSuppression2) { this.currentDevice.chooseNewInnerDevice(this.currentMediaStream) const voiceFocusNode = await this.currentDevice.createAudioNode(this.ctx); // vf node this.inputGainNode.connect(voiceFocusNode.start) // input node -> vf node voiceFocusNode.end.connect(this.vcInNode) } else { // console.log("input___ media stream", this.currentMediaStream) // this.currentMediaStream.getTracks().forEach(x => { // console.log("input___ media stream set", x.getSettings()) // console.log("input___ media stream con", x.getConstraints()) // console.log("input___ media stream cap", x.getCapabilities()) // }) // console.log("input___ media node", this.currentMediaStreamAudioSourceNode) // console.log("input___ gain node", this.inputGainNode.channelCount, this.inputGainNode) this.inputGainNode.connect(this.vcInNode) } this.vcInNode.setOutputNode(this.vcOutNode) console.log("Input Setup=> success") await this.unlock(lockNum) } get stream(): MediaStream { return this.currentMediaStreamAudioDestinationNode.stream } get monitorStream(): MediaStream { return this.currentMediaStreamAudioDestinationMonitorNode.stream } start = async () => { await this.vcInNode.start() this._isVoiceChanging = true } stop = async () => { await this.vcInNode.stop() this._isVoiceChanging = false } get isVoiceChanging(): boolean { return this._isVoiceChanging } //////////////////////// /// 設定 ////////////////////////////// setServerUrl = (serverUrl: string, openTab: boolean = false) => { const url = validateUrl(serverUrl) const pageUrl = `${location.protocol}//${location.host}` if (url != pageUrl && url.length != 0 && location.protocol == "https:" && this.sslCertified.includes(url) == false) { if (openTab) { const value = window.confirm("MMVC Server is different from this page's origin. Open tab to open ssl connection. OK? (You can close the opened tab after ssl connection succeed.)"); if (value) { window.open(url, '_blank') this.sslCertified.push(url) } else { alert("Your voice conversion may fail...") } } } this.vcInNode.updateSetting({ ...this.vcInNode.getSettings(), serverUrl: url }) this.configurator.setServerUrl(url) } updateClientSetting = async (setting: VoiceChangerClientSetting) => { let reconstructInputRequired = false if ( this.setting.audioInput != setting.audioInput || this.setting.echoCancel != setting.echoCancel || this.setting.noiseSuppression != setting.noiseSuppression || this.setting.noiseSuppression2 != setting.noiseSuppression2 || this.setting.sampleRate != setting.sampleRate ) { reconstructInputRequired = true } if (this.setting.inputGain != setting.inputGain) { this.setInputGain(setting.inputGain) } if (this.setting.outputGain != setting.outputGain) { this.setOutputGain(setting.outputGain) } if (this.setting.monitorGain != setting.monitorGain) { this.setMonitorGain(setting.monitorGain) } this.setting = setting if (reconstructInputRequired) { await this.setup() } } setInputGain = (val: number) => { this.setting.inputGain = val if (!this.inputGainNode) { return } if(!val){ return } this.inputGainNode.gain.value = val } setOutputGain = (val: number) => { if (!this.outputGainNode) { return } if(!val){ return } this.outputGainNode.gain.value = val } setMonitorGain = (val: number) => { if (!this.monitorGainNode) { return } if(!val){ return } this.monitorGainNode.gain.value = val } ///////////////////////////////////////////////////// // コンポーネント設定、操作 ///////////////////////////////////////////////////// //## Server ##// getModelType = () => { return this.configurator.getModelType() } getOnnx = async () => { return this.configurator.export2onnx() } mergeModel = async (req: MergeModelRequest) => { return this.configurator.mergeModel(req) } updateModelDefault = async () => { return this.configurator.updateModelDefault() } updateModelInfo = async (slot: number, key: string, val: string) => { return this.configurator.updateModelInfo(slot, key, val) } updateServerSettings = (key: ServerSettingKey, val: string) => { return this.configurator.updateSettings(key, val) } uploadFile = (buf: ArrayBuffer, filename: string, onprogress: (progress: number, end: boolean) => void) => { return this.configurator.uploadFile(buf, filename, onprogress) } uploadFile2 = (dir: string, file: File, onprogress: (progress: number, end: boolean) => void) => { return this.configurator.uploadFile2(dir, file, onprogress) } concatUploadedFile = (filename: string, chunkNum: number) => { return this.configurator.concatUploadedFile(filename, chunkNum) } loadModel = ( slot: number, isHalf: boolean, params: string, ) => { return this.configurator.loadModel(slot, isHalf, params) } uploadAssets = (params: string) => { return this.configurator.uploadAssets(params) } //## Worklet ##// configureWorklet = (setting: WorkletSetting) => { this.vcInNode.configure(setting) this.vcOutNode.configure(setting) } startOutputRecording = () => { this.vcOutNode.startOutputRecording() } stopOutputRecording = () => { return this.vcOutNode.stopOutputRecording() } trancateBuffer = () => { this.vcOutNode.trancateBuffer() } //## Worklet Node ##// updateWorkletNodeSetting = (setting: WorkletNodeSetting) => { this.vcInNode.updateSetting(setting) this.vcOutNode.updateSetting(setting) } ///////////////////////////////////////////////////// // 情報取得 ///////////////////////////////////////////////////// // Information getClientSettings = () => { return this.vcInNode.getSettings() } getServerSettings = () => { return this.configurator.getSettings() } getPerformance = () => { return this.configurator.getPerformance() } getSocketId = () => { return this.vcInNode.getSocketId() } }