From c4cb04f0112d7657cb16779ab9e61142e8821137 Mon Sep 17 00:00:00 2001 From: Josh Knapp Date: Thu, 5 Nov 2026 06:24:11 -0800 Subject: [PATCH] Fix speech-to-text microphone on Linux AppImage The mic button failed on Linux with a chain of distinct problems, each hidden behind the previous one: - The AppImage bundled GStreamer libraries but no plugins, so WebKitGTK found 0 capture devices. Enable bundleMediaFramework and install the GStreamer plugin packages on the CI runners. - The bundled libpipewire had the builder's SPA plugin paths compiled in and failed on hosts with a different layout. finalize-appimage.sh now drops it so the host's is used (host-coupled, like libwayland-client). - WebKitGTK denies getUserMedia until the embedder enables media streams and answers the permission request; wry does neither. Add a Linux-only handler for the main window that grants audio-only requests and denies everything else. - getUserMedia now steps down from the saved device to the default device to bare audio:true on OverconstrainedError/NotFoundError, and the STT error shows the DOMException name. Verified on CachyOS: permission request allowed, recording works. Co-Authored-By: Claude Sonnet 5.5 --- .gitea/workflows/build-app-preview.yml | 8 +++ .gitea/workflows/build-app.yml | 4 ++ app/src-tauri/Cargo.lock | 1 + app/src-tauri/Cargo.toml | 6 ++ app/src-tauri/src/lib.rs | 5 ++ app/src-tauri/src/linux_media.rs | 77 ++++++++++++++++++++++++++ app/src-tauri/tauri.conf.json | 7 ++- app/src/hooks/useSTT.ts | 11 ++-- app/src/hooks/useVoice.ts | 8 +-- app/src/lib/get-microphone.ts | 35 ++++++++++++ scripts/finalize-appimage.sh | 36 ++++++++++++ 11 files changed, 186 insertions(+), 12 deletions(-) create mode 100644 app/src-tauri/src/linux_media.rs create mode 100644 app/src/lib/get-microphone.ts diff --git a/.gitea/workflows/build-app-preview.yml b/.gitea/workflows/build-app-preview.yml index fb13e68..f5942a8 100644 --- a/.gitea/workflows/build-app-preview.yml +++ b/.gitea/workflows/build-app-preview.yml @@ -284,6 +284,10 @@ jobs: sudo env NEEDRESTART_MODE=a DEBIAN_FRONTEND=noninteractive apt-get install -y \ libgtk-3-dev \ libwebkit2gtk-4.1-dev \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-pipewire \ + gstreamer1.0-pulseaudio \ libayatana-appindicator3-dev \ librsvg2-dev \ libsoup-3.0-dev \ @@ -378,6 +382,10 @@ jobs: sudo env NEEDRESTART_MODE=a DEBIAN_FRONTEND=noninteractive apt-get install -y \ libgtk-3-dev \ libwebkit2gtk-4.1-dev \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-pipewire \ + gstreamer1.0-pulseaudio \ libayatana-appindicator3-dev \ librsvg2-dev \ libsoup-3.0-dev \ diff --git a/.gitea/workflows/build-app.yml b/.gitea/workflows/build-app.yml index 913e535..2ad49a7 100644 --- a/.gitea/workflows/build-app.yml +++ b/.gitea/workflows/build-app.yml @@ -151,6 +151,10 @@ jobs: sudo env NEEDRESTART_MODE=a DEBIAN_FRONTEND=noninteractive apt-get install -y \ libgtk-3-dev \ libwebkit2gtk-4.1-dev \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-pipewire \ + gstreamer1.0-pulseaudio \ libayatana-appindicator3-dev \ librsvg2-dev \ libsoup-3.0-dev \ diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 22b1543..f14dc98 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -6644,6 +6644,7 @@ dependencies = [ "tower-http", "url", "uuid", + "webkit2gtk", "zeroize", ] diff --git a/app/src-tauri/Cargo.toml b/app/src-tauri/Cargo.toml index 9ffeb45..bcd8928 100644 --- a/app/src-tauri/Cargo.toml +++ b/app/src-tauri/Cargo.toml @@ -48,6 +48,12 @@ similar = "2" # Blocking client + rustls: no git binary or OpenSSL needed on the host. gix = { version = "0.88", default-features = false, features = ["blocking-network-client", "blocking-http-transport-reqwest-rust-tls", "credentials", "sha1"] } +# Linux only: WebKitGTK refuses getUserMedia until the embedder enables media +# streams and answers the permission request (see src/linux_media.rs). Same +# version tauri/wry already resolve, so this adds no second copy. +[target.'cfg(target_os = "linux")'.dependencies] +webkit2gtk = "2.0.2" + [dev-dependencies] # `test-util` (not part of tokio's `full`) lets the auto-start retry tests run # their backoff schedule under a paused clock instead of in real seconds. diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index 08a2bba..3155056 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -4,6 +4,8 @@ mod browser_view; mod command_census; mod commands; mod container_watch; +#[cfg(target_os = "linux")] +mod linux_media; mod docker; pub mod file_viewer; mod install_helper; @@ -255,6 +257,9 @@ pub fn run() { }) .manage(file_viewer::registry::ViewerRegistry::default()) .setup(move |app| { + #[cfg(target_os = "linux")] + linux_media::enable_microphone(app.handle()); + match tauri::image::Image::from_bytes(include_bytes!("../icons/icon.png")) { Ok(icon) => { if let Some(window) = app.get_webview_window("main") { diff --git a/app/src-tauri/src/linux_media.rs b/app/src-tauri/src/linux_media.rs new file mode 100644 index 0000000..a4d1ae0 --- /dev/null +++ b/app/src-tauri/src/linux_media.rs @@ -0,0 +1,77 @@ +//! Microphone access for the main window on Linux (WebKitGTK). +//! +//! WebKitGTK denies `getUserMedia` unless the embedder (a) has the +//! `enable-media-stream` setting on and (b) answers the webview's +//! `permission-request` signal. wry does neither, so without this the speech- +//! to-text button fails with +//! `NotAllowedError: The request is not allowed by the user agent ...` even +//! when GStreamer can see the microphone. +//! +//! Scope is deliberately narrow: +//! * only the `main` window — the file-viewer windows and any embedded +//! browser view load different content and get no grant; +//! * only audio capture — a request that also asks for a camera, and every +//! other permission type (geolocation, notifications, ...), is denied. + +use tauri::{AppHandle, Manager}; +use webkit2gtk::glib::prelude::*; +use webkit2gtk::{ + PermissionRequestExt, SettingsExt, UserMediaPermissionRequest, UserMediaPermissionRequestExt, + WebView, WebViewExt, +}; + +/// Whether a media permission request should be granted: microphone only. +pub(crate) fn allow_media_request(for_audio: bool, for_video: bool) -> bool { + for_audio && !for_video +} + +pub fn enable_microphone(app: &AppHandle) { + let Some(window) = app.get_webview_window("main") else { + log::warn!("Microphone: no main window, capture will stay disabled"); + return; + }; + let result = window.with_webview(|platform| { + let webview: WebView = platform.inner(); + if let Some(settings) = WebViewExt::settings(&webview) { + settings.set_enable_media_stream(true); + } + webview.connect_permission_request(|_, request| { + if let Some(media) = request.downcast_ref::() { + let ok = + allow_media_request(media.is_for_audio_device(), media.is_for_video_device()); + if ok { + request.allow(); + } else { + request.deny(); + } + log::info!( + "Microphone: media permission request {}", + if ok { "allowed" } else { "denied" } + ); + } else { + request.deny(); + } + true // handled; do not fall through to WebKit's default + }); + }); + if let Err(e) = result { + log::warn!("Microphone: could not configure the webview: {e}"); + } +} + +#[cfg(test)] +mod tests { + use super::allow_media_request; + + #[test] + fn grants_audio_only() { + assert!(allow_media_request(true, false)); + } + + #[test] + fn denies_camera_and_mixed_requests() { + assert!(!allow_media_request(false, true)); + assert!(!allow_media_request(true, true)); + assert!(!allow_media_request(false, false)); + } +} diff --git a/app/src-tauri/tauri.conf.json b/app/src-tauri/tauri.conf.json index 65e36e6..48fce27 100644 --- a/app/src-tauri/tauri.conf.json +++ b/app/src-tauri/tauri.conf.json @@ -35,7 +35,12 @@ "icons/icon.ico", "icons/icon.icns", "icons/icon.png" - ] + ], + "linux": { + "appimage": { + "bundleMediaFramework": true + } + } }, "plugins": {} } diff --git a/app/src/hooks/useSTT.ts b/app/src/hooks/useSTT.ts index 8bff90f..69e57af 100644 --- a/app/src/hooks/useSTT.ts +++ b/app/src/hooks/useSTT.ts @@ -2,6 +2,7 @@ import { useCallback, useRef, useState } from "react"; import * as commands from "../lib/tauri-commands"; import { encodeWav } from "../lib/wav"; import { useAppState } from "../store/appState"; +import { getMicrophoneStream } from "../lib/get-microphone"; export type SttState = "idle" | "recording" | "transcribing" | "error"; @@ -36,11 +37,8 @@ export function useSTT(sessionId: string, sendInput: (sessionId: string, data: s noiseSuppression: true, autoGainControl: true, }; - if (deviceId) { - audioConstraints.deviceId = { exact: deviceId }; - } - const stream = await navigator.mediaDevices.getUserMedia({ audio: audioConstraints }); + const stream = await getMicrophoneStream(audioConstraints, deviceId); streamRef.current = stream; const audioContext = new AudioContext({ sampleRate: 16000 }); @@ -59,7 +57,10 @@ export function useSTT(sessionId: string, sendInput: (sessionId: string, data: s source.connect(processor); processor.connect(audioContext.destination); } catch (e) { - const msg = e instanceof Error ? e.message : String(e); + // Include the DOMException name: "Invalid constraint" alone doesn't say + // whether it was OverconstrainedError, NotAllowedError, NotFoundError, etc. + const msg = e instanceof Error ? `${e.name}: ${e.message}` : String(e); + console.error("STT startRecording failed:", e); setError(msg); setState("error"); } diff --git a/app/src/hooks/useVoice.ts b/app/src/hooks/useVoice.ts index b7800df..cfb6787 100644 --- a/app/src/hooks/useVoice.ts +++ b/app/src/hooks/useVoice.ts @@ -1,5 +1,6 @@ import { useCallback, useRef, useState } from "react"; import * as commands from "../lib/tauri-commands"; +import { getMicrophoneStream } from "../lib/get-microphone"; type VoiceState = "inactive" | "starting" | "active" | "error"; @@ -27,13 +28,8 @@ export function useVoice(sessionId: string, deviceId?: string | null) { noiseSuppression: true, autoGainControl: true, }; - if (deviceId) { - audioConstraints.deviceId = { exact: deviceId }; - } - const stream = await navigator.mediaDevices.getUserMedia({ - audio: audioConstraints, - }); + const stream = await getMicrophoneStream(audioConstraints, deviceId); streamRef.current = stream; // 3. Create AudioContext at 16kHz (browser handles resampling) diff --git a/app/src/lib/get-microphone.ts b/app/src/lib/get-microphone.ts new file mode 100644 index 0000000..0707b3f --- /dev/null +++ b/app/src/lib/get-microphone.ts @@ -0,0 +1,35 @@ +/** + * Open the microphone, relaxing constraints until the webview accepts them. + * + * Attempts, in order: + * 1. saved device (exact) + base constraints + * 2. base constraints, system default device + * 3. bare `audio: true` + * + * Step 2 covers a stale saved deviceId (unplugged / regenerated IDs). Step 3 + * covers webviews (notably WebKitGTK on Linux) that reject one of the optional + * processing constraints with `OverconstrainedError: Invalid constraint`. + * Only constraint/device errors fall through; permission errors still throw. + */ +const RETRYABLE = new Set(["OverconstrainedError", "NotFoundError"]); + +export async function getMicrophoneStream( + base: MediaTrackConstraints, + deviceId?: string | null, +): Promise { + const attempts: (MediaTrackConstraints | boolean)[] = []; + if (deviceId) attempts.push({ ...base, deviceId: { exact: deviceId } }); + attempts.push(base, true); + + let lastError: unknown; + for (const audio of attempts) { + try { + return await navigator.mediaDevices.getUserMedia({ audio }); + } catch (e) { + lastError = e; + if (!(e instanceof DOMException) || !RETRYABLE.has(e.name)) throw e; + console.warn("getUserMedia attempt failed, relaxing constraints:", e.name, e.message, audio); + } + } + throw lastError; +} diff --git a/scripts/finalize-appimage.sh b/scripts/finalize-appimage.sh index ed4e41d..77589d1 100755 --- a/scripts/finalize-appimage.sh +++ b/scripts/finalize-appimage.sh @@ -55,6 +55,28 @@ # lands last: a fallback, never an override. # # --------------------------------------------------------------------------- +# 1b. The bundled libpipewire (microphone capture) +# --------------------------------------------------------------------------- +# +# `bundleMediaFramework` puts the GStreamer plugins in the image, including +# libgstpipewire.so, which is what WebKitGTK uses to reach the microphone. That +# plugin pulls libpipewire-0.3.so.0 in with it, and the copy linuxdeploy +# bundles has the *builder's* plugin directories compiled in +# (/usr/lib/x86_64-linux-gnu/spa-0.2 and .../pipewire-0.3). Neither is bundled +# and neither exists on, say, Arch (/usr/lib/spa-0.2), so on the host: +# +# [E] pw.loop [loop.c:67 pw_loop_new()] can't make support.system handle: +# No such file or directory +# +# and the microphone never opens. libpipewire is host-coupled in the same way +# libwayland-client is: it has to agree with the SPA plugins, client config and +# daemon actually running. Its client ABI is backward compatible, so the +# bundled libgstpipewire.so links happily against a newer host copy. Unlike the +# wayland case there is no useful fallback — without a host PipeWire daemon +# there is nothing to capture from — so the bundled copy is simply removed; +# GStreamer skips a plugin whose library will not load and the app still starts. +# +# --------------------------------------------------------------------------- # 2. Metadata an AppImage manager needs # --------------------------------------------------------------------------- # @@ -202,6 +224,17 @@ else echo "$LIB is not bundled — nothing to demote." fi +# Independent of the wayland demotion above, and equally unconditional on it. +PW_LIB="libpipewire-0.3.so.0" +pw_removed=false +if [ -e "$root/usr/lib/$PW_LIB" ]; then + rm -f "$root/usr/lib/$PW_LIB" + pw_removed=true + echo "Removed bundled $PW_LIB (the host's is used)." +else + echo "$PW_LIB is not bundled — nothing to remove." +fi + # --- metadata ------------------------------------------------------------- # Version comes from the artifact rather than a second source that could drift. @@ -279,6 +312,9 @@ if [ "$demoted" = true ]; then [ -e "$out/$HOOK" ] || fail "the fallback hook is missing." grep -q "triple-c-wayland-fallback" "$out/AppRun" || fail "AppRun does not source the hook." fi +if [ "$pw_removed" = true ]; then + [ -e "$out/usr/lib/$PW_LIB" ] && fail "$PW_LIB is still bundled." +fi [ -x "$out/usr/bin/triple-c" ] || fail "no executable usr/bin/triple-c." # Every file readable by all, every program executable by all — what the -- 2.52.0