TEST: screen capture

client/demo/src/components/demo/001_GuiStateProvider.tsx

@@ -64,12 +64,14 @@ type GuiStateAndMethod = {
     audioInputForGUI: string;
     audioOutputForGUI: string;
     fileInputEchoback: boolean | undefined;
+    shareScreenEnabled: boolean;
     audioOutputForAnalyzer: string;
     setInputAudioDeviceInfo: (val: MediaDeviceInfo[]) => void;
     setOutputAudioDeviceInfo: (val: MediaDeviceInfo[]) => void;
     setAudioInputForGUI: (val: string) => void;
     setAudioOutputForGUI: (val: string) => void;
     setFileInputEchoback: (val: boolean) => void;
+    setShareScreenEnabled: (val: boolean) => void;
     setAudioOutputForAnalyzer: (val: string) => void;
 
     modelSlotNum: number;
@@ -105,6 +107,7 @@ export const GuiStateProvider = ({ children }: Props) => {
     const [audioInputForGUI, setAudioInputForGUI] = useState<string>("none");
     const [audioOutputForGUI, setAudioOutputForGUI] = useState<string>("none");
     const [fileInputEchoback, setFileInputEchoback] = useState<boolean>(false); //最初のmuteが有効になるように。undefined <-- ??? falseしておけばよさそう。undefinedだとwarningがでる。
+    const [shareScreenEnabled, setShareScreenEnabled] = useState<boolean>(false);
     const [audioOutputForAnalyzer, setAudioOutputForAnalyzer] = useState<string>("default");
 
     const [textInputResolve, setTextInputResolve] = useState<TextInputResolveType | null>(null);
@@ -137,6 +140,13 @@ export const GuiStateProvider = ({ children }: Props) => {
             label: "file",
             toJSON: () => {},
         });
+        audioInputs.push({
+            deviceId: "screen",
+            groupId: "screen",
+            kind: "audioinput",
+            label: "screen",
+            toJSON: () => {},
+        });
         const audioOutputs = mediaDeviceInfos.filter((x) => {
             return x.kind == "audiooutput";
         });
@@ -261,12 +271,14 @@ export const GuiStateProvider = ({ children }: Props) => {
         audioInputForGUI,
         audioOutputForGUI,
         fileInputEchoback,
+        shareScreenEnabled,
         audioOutputForAnalyzer,
         setInputAudioDeviceInfo,
         setOutputAudioDeviceInfo,
         setAudioInputForGUI,
         setAudioOutputForGUI,
         setFileInputEchoback,
+        setShareScreenEnabled,
         setAudioOutputForAnalyzer,
 
         modelSlotNum,

client/demo/src/components/demo/components2/102-2_ConvertArea.tsx

@@ -30,8 +30,8 @@ export const ConvertArea = (props: ConvertProps) => {
             memory: 0,
         });
 
-        const onClassName = serverSetting.serverSetting.gpu == 0 ? "config-sub-area-button-active" : "config-sub-area-button";
+        // const onClassName = serverSetting.serverSetting.gpu == 0 ? "config-sub-area-button-active" : "config-sub-area-button";
-        const offClassName = serverSetting.serverSetting.gpu == 0 ? "config-sub-area-button" : "config-sub-area-button-active";
+        // const offClassName = serverSetting.serverSetting.gpu == 0 ? "config-sub-area-button" : "config-sub-area-button-active";
 
         const cpuClassName = serverSetting.serverSetting.gpu == -1 ? "config-sub-area-button-active" : "config-sub-area-button";
         const gpu0ClassName = serverSetting.serverSetting.gpu == 0 ? "config-sub-area-button-active" : "config-sub-area-button";

client/demo/src/components/demo/components2/102-3_DeviceArea.tsx

@@ -8,11 +8,12 @@ export type DeviceAreaProps = {};
 
 export const DeviceArea = (_props: DeviceAreaProps) => {
     const { setting, serverSetting, audioContext, setAudioOutputElementId, initializedRef, setVoiceChangerClientSetting, startOutputRecording, stopOutputRecording } = useAppState();
-    const { isConverting, audioInputForGUI, inputAudioDeviceInfo, setAudioInputForGUI, fileInputEchoback, setFileInputEchoback, setAudioOutputForGUI, audioOutputForGUI, outputAudioDeviceInfo } = useGuiState();
+    const { isConverting, audioInputForGUI, inputAudioDeviceInfo, setAudioInputForGUI, fileInputEchoback, setFileInputEchoback, setAudioOutputForGUI, audioOutputForGUI, outputAudioDeviceInfo, shareScreenEnabled, setShareScreenEnabled } = useGuiState();
     const [inputHostApi, setInputHostApi] = useState<string>("ALL");
     const [outputHostApi, setOutputHostApi] = useState<string>("ALL");
     const [monitorHostApi, setMonitorHostApi] = useState<string>("ALL");
     const audioSrcNode = useRef<MediaElementAudioSourceNode>();
+    const displayMediaStream = useRef<MediaStream | null>(null);
 
     const { getItem, setItem } = useIndexedDB({ clientType: null });
     const [outputRecordingStarted, setOutputRecordingStarted] = useState<boolean>(false);
@@ -97,7 +98,7 @@ export const DeviceArea = (_props: DeviceAreaProps) => {
                         value={audioInputForGUI}
                         onChange={async (e) => {
                             setAudioInputForGUI(e.target.value);
-                            if (e.target.value != "file") {
+                            if (e.target.value != "file" && e.target.value != "screen") {
                                 try {
                                     await setVoiceChangerClientSetting({ ...setting.voiceChangerClientSetting, audioInput: e.target.value });
                                 } catch (e) {
@@ -132,7 +133,7 @@ export const DeviceArea = (_props: DeviceAreaProps) => {
         const hostAPIs = new Set(
             devices.map((x) => {
                 return x.hostAPI;
-            })
+            }),
         );
         const hostAPIOptions = Array.from(hostAPIs).map((x, index) => {
             return (
@@ -275,6 +276,74 @@ export const DeviceArea = (_props: DeviceAreaProps) => {
         );
     }, [audioInputForGUI, fileInputEchoback, setting, serverSetting.serverSetting]);
 
+    const audioInputScreenRow = useMemo(() => {
+        if (audioInputForGUI != "screen" || serverSetting.serverSetting.enableServerAudio == 1) {
+            return <></>;
+        }
+
+        const onSelectScreenClicked = async () => {
+            // 既存msをクローズ
+            if (displayMediaStream.current) {
+                displayMediaStream.current.getTracks().forEach((x) => {
+                    x.stop();
+                });
+                displayMediaStream.current = null;
+            }
+
+            // 共有ストップ
+            if (shareScreenEnabled == true) {
+                setShareScreenEnabled(false);
+                return;
+            }
+
+            // 共有スタート
+            try {
+                displayMediaStream.current = await navigator.mediaDevices.getDisplayMedia({
+                    video: true,
+                    audio: true,
+                });
+            } catch (e) {
+                return;
+            }
+            if (!displayMediaStream.current) {
+                return;
+            }
+            if (displayMediaStream.current?.getAudioTracks().length == 0) {
+                displayMediaStream.current.getTracks().forEach((x) => {
+                    x.stop();
+                });
+                displayMediaStream.current = null;
+            }
+
+            try {
+                setVoiceChangerClientSetting({ ...setting.voiceChangerClientSetting, audioInput: displayMediaStream.current });
+            } catch (e) {
+                console.error(e);
+            }
+
+            setShareScreenEnabled(!shareScreenEnabled);
+        };
+
+        const echobackClass = shareScreenEnabled ? "config-sub-area-control-field-screen-select-button-active" : "config-sub-area-control-field-screen-select-button";
+        return (
+            <div className="config-sub-area-control">
+                <div className="config-sub-area-control-title left-padding-1"></div>
+                <div className="config-sub-area-control-field">
+                    <div className="config-sub-area-control-field-screen-select">
+                        <div
+                            className={echobackClass}
+                            onClick={() => {
+                                onSelectScreenClicked();
+                            }}
+                        >
+                            capture
+                        </div>
+                    </div>
+                </div>
+            </div>
+        );
+    }, [audioInputForGUI, setting, serverSetting.serverSetting, shareScreenEnabled, setShareScreenEnabled]);
+
     // (3) Audio Output
     useEffect(() => {
         const loadCache = async () => {
@@ -375,7 +444,7 @@ export const DeviceArea = (_props: DeviceAreaProps) => {
         const hostAPIs = new Set(
             devices.map((x) => {
                 return x.hostAPI;
-            })
+            }),
         );
         const hostAPIOptions = Array.from(hostAPIs).map((x, index) => {
             return (
@@ -516,7 +585,7 @@ export const DeviceArea = (_props: DeviceAreaProps) => {
         const hostAPIs = new Set(
             devices.map((x) => {
                 return x.hostAPI;
-            })
+            }),
         );
         const hostAPIOptions = Array.from(hostAPIs).map((x, index) => {
             return (
@@ -541,7 +610,7 @@ export const DeviceArea = (_props: DeviceAreaProps) => {
         filteredDevice.unshift(
             <option value={-1} key={-1}>
                 none
-            </option>
+            </option>,
         );
 
         const currentValue = devices.find((x) => {
@@ -591,6 +660,7 @@ export const DeviceArea = (_props: DeviceAreaProps) => {
             {clientAudioInputRow}
             {serverAudioInputRow}
             {audioInputMediaRow}
+            {audioInputScreenRow}
             {clientAudioOutputRow}
             {serverAudioOutputRow}
             {serverMonitorRow}

client/demo/src/css/App.css

@@ -1455,6 +1455,31 @@ audio::-webkit-media-controls-overlay-enclosure{
                         max-width: 70%;
                     }
                 }
+                .config-sub-area-control-field-screen-select {
+                    display: flex;
+                    flex-direction: row-reverse;
+                    gap: 5px;
+                    .config-sub-area-control-field-screen-select-button-active {
+                        font-size: 0.8rem;
+                        border: solid 1px #333;
+                        border-radius: 5px;
+                        background: #ada;
+                        height: 1.2rem;
+                        padding-left: 2px;
+                        padding-right: 2px;
+                        cursor: pointer;
+                    }
+                    .config-sub-area-control-field-screen-select-button {
+                        border: solid 1px #333;
+                        background: #fff;
+                        font-size: 0.8rem;
+                        border-radius: 5px;
+                        height: 1.2rem;
+                        padding-left: 2px;
+                        padding-right: 2px;
+                        cursor: pointer;
+                    }
+                }
                 .config-sub-area-control-field-wav-file {
                     display: flex;
                     flex-direction: row;