Fix speech-to-text microphone on Linux AppImage #72

Merged
jknapp merged 1 commits from fix/linux-microphone into main 2026-11-05 14:42:33 +00:00
11 changed files with 186 additions and 12 deletions

No files matched your search

+8
View File
@@ -284,6 +284,10 @@ jobs:
sudo env NEEDRESTART_MODE=a DEBIAN_FRONTEND=noninteractive apt-get install -y \
libgtk-3-dev \
libwebkit2gtk-4.1-dev \
gstreamer1.0-plugins-base \
gstreamer1.0-plugins-good \
gstreamer1.0-pipewire \
gstreamer1.0-pulseaudio \
libayatana-appindicator3-dev \
librsvg2-dev \
libsoup-3.0-dev \
@@ -378,6 +382,10 @@ jobs:
sudo env NEEDRESTART_MODE=a DEBIAN_FRONTEND=noninteractive apt-get install -y \
libgtk-3-dev \
libwebkit2gtk-4.1-dev \
gstreamer1.0-plugins-base \
gstreamer1.0-plugins-good \
gstreamer1.0-pipewire \
gstreamer1.0-pulseaudio \
libayatana-appindicator3-dev \
librsvg2-dev \
libsoup-3.0-dev \
+4
View File
@@ -151,6 +151,10 @@ jobs:
sudo env NEEDRESTART_MODE=a DEBIAN_FRONTEND=noninteractive apt-get install -y \
libgtk-3-dev \
libwebkit2gtk-4.1-dev \
gstreamer1.0-plugins-base \
gstreamer1.0-plugins-good \
gstreamer1.0-pipewire \
gstreamer1.0-pulseaudio \
libayatana-appindicator3-dev \
librsvg2-dev \
libsoup-3.0-dev \
+1
View File
@@ -6644,6 +6644,7 @@ dependencies = [
"tower-http",
"url",
"uuid",
"webkit2gtk",
"zeroize",
]
+6
View File
@@ -48,6 +48,12 @@ similar = "2"
# Blocking client + rustls: no git binary or OpenSSL needed on the host.
gix = { version = "0.88", default-features = false, features = ["blocking-network-client", "blocking-http-transport-reqwest-rust-tls", "credentials", "sha1"] }
# Linux only: WebKitGTK refuses getUserMedia until the embedder enables media
# streams and answers the permission request (see src/linux_media.rs). Same
# version tauri/wry already resolve, so this adds no second copy.
[target.'cfg(target_os = "linux")'.dependencies]
webkit2gtk = "2.0.2"
[dev-dependencies]
# `test-util` (not part of tokio's `full`) lets the auto-start retry tests run
# their backoff schedule under a paused clock instead of in real seconds.
+5
View File
@@ -4,6 +4,8 @@ mod browser_view;
mod command_census;
mod commands;
mod container_watch;
#[cfg(target_os = "linux")]
mod linux_media;
mod docker;
pub mod file_viewer;
mod install_helper;
@@ -255,6 +257,9 @@ pub fn run() {
})
.manage(file_viewer::registry::ViewerRegistry::default())
.setup(move |app| {
#[cfg(target_os = "linux")]
linux_media::enable_microphone(app.handle());
match tauri::image::Image::from_bytes(include_bytes!("../icons/icon.png")) {
Ok(icon) => {
if let Some(window) = app.get_webview_window("main") {
+77
View File
@@ -0,0 +1,77 @@
//! Microphone access for the main window on Linux (WebKitGTK).
//!
//! WebKitGTK denies `getUserMedia` unless the embedder (a) has the
//! `enable-media-stream` setting on and (b) answers the webview's
//! `permission-request` signal. wry does neither, so without this the speech-
//! to-text button fails with
//! `NotAllowedError: The request is not allowed by the user agent ...` even
//! when GStreamer can see the microphone.
//!
//! Scope is deliberately narrow:
//! * only the `main` window — the file-viewer windows and any embedded
//! browser view load different content and get no grant;
//! * only audio capture — a request that also asks for a camera, and every
//! other permission type (geolocation, notifications, ...), is denied.
use tauri::{AppHandle, Manager};
use webkit2gtk::glib::prelude::*;
use webkit2gtk::{
PermissionRequestExt, SettingsExt, UserMediaPermissionRequest, UserMediaPermissionRequestExt,
WebView, WebViewExt,
};
/// Whether a media permission request should be granted: microphone only.
pub(crate) fn allow_media_request(for_audio: bool, for_video: bool) -> bool {
for_audio && !for_video
}
pub fn enable_microphone(app: &AppHandle) {
let Some(window) = app.get_webview_window("main") else {
log::warn!("Microphone: no main window, capture will stay disabled");
return;
};
let result = window.with_webview(|platform| {
let webview: WebView = platform.inner();
if let Some(settings) = WebViewExt::settings(&webview) {
settings.set_enable_media_stream(true);
}
webview.connect_permission_request(|_, request| {
if let Some(media) = request.downcast_ref::<UserMediaPermissionRequest>() {
let ok =
allow_media_request(media.is_for_audio_device(), media.is_for_video_device());
if ok {
request.allow();
} else {
request.deny();
}
log::info!(
"Microphone: media permission request {}",
if ok { "allowed" } else { "denied" }
);
} else {
request.deny();
}
true // handled; do not fall through to WebKit's default
});
});
if let Err(e) = result {
log::warn!("Microphone: could not configure the webview: {e}");
}
}
#[cfg(test)]
mod tests {
use super::allow_media_request;
#[test]
fn grants_audio_only() {
assert!(allow_media_request(true, false));
}
#[test]
fn denies_camera_and_mixed_requests() {
assert!(!allow_media_request(false, true));
assert!(!allow_media_request(true, true));
assert!(!allow_media_request(false, false));
}
}
+6 -1
View File
@@ -35,7 +35,12 @@
"icons/icon.ico",
"icons/icon.icns",
"icons/icon.png"
]
],
"linux": {
"appimage": {
"bundleMediaFramework": true
}
}
},
"plugins": {}
}
+6 -5
View File
@@ -2,6 +2,7 @@ import { useCallback, useRef, useState } from "react";
import * as commands from "../lib/tauri-commands";
import { encodeWav } from "../lib/wav";
import { useAppState } from "../store/appState";
import { getMicrophoneStream } from "../lib/get-microphone";
export type SttState = "idle" | "recording" | "transcribing" | "error";
@@ -36,11 +37,8 @@ export function useSTT(sessionId: string, sendInput: (sessionId: string, data: s
noiseSuppression: true,
autoGainControl: true,
};
if (deviceId) {
audioConstraints.deviceId = { exact: deviceId };
}
const stream = await navigator.mediaDevices.getUserMedia({ audio: audioConstraints });
const stream = await getMicrophoneStream(audioConstraints, deviceId);
streamRef.current = stream;
const audioContext = new AudioContext({ sampleRate: 16000 });
@@ -59,7 +57,10 @@ export function useSTT(sessionId: string, sendInput: (sessionId: string, data: s
source.connect(processor);
processor.connect(audioContext.destination);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
// Include the DOMException name: "Invalid constraint" alone doesn't say
// whether it was OverconstrainedError, NotAllowedError, NotFoundError, etc.
const msg = e instanceof Error ? `${e.name}: ${e.message}` : String(e);
console.error("STT startRecording failed:", e);
setError(msg);
setState("error");
}
+2 -6
View File
@@ -1,5 +1,6 @@
import { useCallback, useRef, useState } from "react";
import * as commands from "../lib/tauri-commands";
import { getMicrophoneStream } from "../lib/get-microphone";
type VoiceState = "inactive" | "starting" | "active" | "error";
@@ -27,13 +28,8 @@ export function useVoice(sessionId: string, deviceId?: string | null) {
noiseSuppression: true,
autoGainControl: true,
};
if (deviceId) {
audioConstraints.deviceId = { exact: deviceId };
}
const stream = await navigator.mediaDevices.getUserMedia({
audio: audioConstraints,
});
const stream = await getMicrophoneStream(audioConstraints, deviceId);
streamRef.current = stream;
// 3. Create AudioContext at 16kHz (browser handles resampling)
+35
View File
@@ -0,0 +1,35 @@
/**
* Open the microphone, relaxing constraints until the webview accepts them.
*
* Attempts, in order:
* 1. saved device (exact) + base constraints
* 2. base constraints, system default device
* 3. bare `audio: true`
*
* Step 2 covers a stale saved deviceId (unplugged / regenerated IDs). Step 3
* covers webviews (notably WebKitGTK on Linux) that reject one of the optional
* processing constraints with `OverconstrainedError: Invalid constraint`.
* Only constraint/device errors fall through; permission errors still throw.
*/
const RETRYABLE = new Set(["OverconstrainedError", "NotFoundError"]);
export async function getMicrophoneStream(
base: MediaTrackConstraints,
deviceId?: string | null,
): Promise<MediaStream> {
const attempts: (MediaTrackConstraints | boolean)[] = [];
if (deviceId) attempts.push({ ...base, deviceId: { exact: deviceId } });
attempts.push(base, true);
let lastError: unknown;
for (const audio of attempts) {
try {
return await navigator.mediaDevices.getUserMedia({ audio });
} catch (e) {
lastError = e;
if (!(e instanceof DOMException) || !RETRYABLE.has(e.name)) throw e;
console.warn("getUserMedia attempt failed, relaxing constraints:", e.name, e.message, audio);
}
}
throw lastError;
}
+36
View File
@@ -55,6 +55,28 @@
# lands last: a fallback, never an override.
#
# ---------------------------------------------------------------------------
# 1b. The bundled libpipewire (microphone capture)
# ---------------------------------------------------------------------------
#
# `bundleMediaFramework` puts the GStreamer plugins in the image, including
# libgstpipewire.so, which is what WebKitGTK uses to reach the microphone. That
# plugin pulls libpipewire-0.3.so.0 in with it, and the copy linuxdeploy
# bundles has the *builder's* plugin directories compiled in
# (/usr/lib/x86_64-linux-gnu/spa-0.2 and .../pipewire-0.3). Neither is bundled
# and neither exists on, say, Arch (/usr/lib/spa-0.2), so on the host:
#
# [E] pw.loop [loop.c:67 pw_loop_new()] can't make support.system handle:
# No such file or directory
#
# and the microphone never opens. libpipewire is host-coupled in the same way
# libwayland-client is: it has to agree with the SPA plugins, client config and
# daemon actually running. Its client ABI is backward compatible, so the
# bundled libgstpipewire.so links happily against a newer host copy. Unlike the
# wayland case there is no useful fallback — without a host PipeWire daemon
# there is nothing to capture from — so the bundled copy is simply removed;
# GStreamer skips a plugin whose library will not load and the app still starts.
#
# ---------------------------------------------------------------------------
# 2. Metadata an AppImage manager needs
# ---------------------------------------------------------------------------
#
@@ -202,6 +224,17 @@ else
echo "$LIB is not bundled — nothing to demote."
fi
# Independent of the wayland demotion above, and equally unconditional on it.
PW_LIB="libpipewire-0.3.so.0"
pw_removed=false
if [ -e "$root/usr/lib/$PW_LIB" ]; then
rm -f "$root/usr/lib/$PW_LIB"
pw_removed=true
echo "Removed bundled $PW_LIB (the host's is used)."
else
echo "$PW_LIB is not bundled — nothing to remove."
fi
# --- metadata -------------------------------------------------------------
# Version comes from the artifact rather than a second source that could drift.
@@ -279,6 +312,9 @@ if [ "$demoted" = true ]; then
[ -e "$out/$HOOK" ] || fail "the fallback hook is missing."
grep -q "triple-c-wayland-fallback" "$out/AppRun" || fail "AppRun does not source the hook."
fi
if [ "$pw_removed" = true ]; then
[ -e "$out/usr/lib/$PW_LIB" ] && fail "$PW_LIB is still bundled."
fi
[ -x "$out/usr/bin/triple-c" ] || fail "no executable usr/bin/triple-c."
# Every file readable by all, every program executable by all — what the