Fix speech-to-text microphone on Linux AppImage #72
No files matched your search
@@ -284,6 +284,10 @@ jobs:
|
||||
sudo env NEEDRESTART_MODE=a DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
libgtk-3-dev \
|
||||
libwebkit2gtk-4.1-dev \
|
||||
gstreamer1.0-plugins-base \
|
||||
gstreamer1.0-plugins-good \
|
||||
gstreamer1.0-pipewire \
|
||||
gstreamer1.0-pulseaudio \
|
||||
libayatana-appindicator3-dev \
|
||||
librsvg2-dev \
|
||||
libsoup-3.0-dev \
|
||||
@@ -378,6 +382,10 @@ jobs:
|
||||
sudo env NEEDRESTART_MODE=a DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
libgtk-3-dev \
|
||||
libwebkit2gtk-4.1-dev \
|
||||
gstreamer1.0-plugins-base \
|
||||
gstreamer1.0-plugins-good \
|
||||
gstreamer1.0-pipewire \
|
||||
gstreamer1.0-pulseaudio \
|
||||
libayatana-appindicator3-dev \
|
||||
librsvg2-dev \
|
||||
libsoup-3.0-dev \
|
||||
|
||||
@@ -151,6 +151,10 @@ jobs:
|
||||
sudo env NEEDRESTART_MODE=a DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
libgtk-3-dev \
|
||||
libwebkit2gtk-4.1-dev \
|
||||
gstreamer1.0-plugins-base \
|
||||
gstreamer1.0-plugins-good \
|
||||
gstreamer1.0-pipewire \
|
||||
gstreamer1.0-pulseaudio \
|
||||
libayatana-appindicator3-dev \
|
||||
librsvg2-dev \
|
||||
libsoup-3.0-dev \
|
||||
|
||||
Generated
+1
@@ -6644,6 +6644,7 @@ dependencies = [
|
||||
"tower-http",
|
||||
"url",
|
||||
"uuid",
|
||||
"webkit2gtk",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
|
||||
@@ -48,6 +48,12 @@ similar = "2"
|
||||
# Blocking client + rustls: no git binary or OpenSSL needed on the host.
|
||||
gix = { version = "0.88", default-features = false, features = ["blocking-network-client", "blocking-http-transport-reqwest-rust-tls", "credentials", "sha1"] }
|
||||
|
||||
# Linux only: WebKitGTK refuses getUserMedia until the embedder enables media
|
||||
# streams and answers the permission request (see src/linux_media.rs). Same
|
||||
# version tauri/wry already resolve, so this adds no second copy.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
webkit2gtk = "2.0.2"
|
||||
|
||||
[dev-dependencies]
|
||||
# `test-util` (not part of tokio's `full`) lets the auto-start retry tests run
|
||||
# their backoff schedule under a paused clock instead of in real seconds.
|
||||
|
||||
@@ -4,6 +4,8 @@ mod browser_view;
|
||||
mod command_census;
|
||||
mod commands;
|
||||
mod container_watch;
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux_media;
|
||||
mod docker;
|
||||
pub mod file_viewer;
|
||||
mod install_helper;
|
||||
@@ -255,6 +257,9 @@ pub fn run() {
|
||||
})
|
||||
.manage(file_viewer::registry::ViewerRegistry::default())
|
||||
.setup(move |app| {
|
||||
#[cfg(target_os = "linux")]
|
||||
linux_media::enable_microphone(app.handle());
|
||||
|
||||
match tauri::image::Image::from_bytes(include_bytes!("../icons/icon.png")) {
|
||||
Ok(icon) => {
|
||||
if let Some(window) = app.get_webview_window("main") {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
//! Microphone access for the main window on Linux (WebKitGTK).
|
||||
//!
|
||||
//! WebKitGTK denies `getUserMedia` unless the embedder (a) has the
|
||||
//! `enable-media-stream` setting on and (b) answers the webview's
|
||||
//! `permission-request` signal. wry does neither, so without this the speech-
|
||||
//! to-text button fails with
|
||||
//! `NotAllowedError: The request is not allowed by the user agent ...` even
|
||||
//! when GStreamer can see the microphone.
|
||||
//!
|
||||
//! Scope is deliberately narrow:
|
||||
//! * only the `main` window — the file-viewer windows and any embedded
|
||||
//! browser view load different content and get no grant;
|
||||
//! * only audio capture — a request that also asks for a camera, and every
|
||||
//! other permission type (geolocation, notifications, ...), is denied.
|
||||
|
||||
use tauri::{AppHandle, Manager};
|
||||
use webkit2gtk::glib::prelude::*;
|
||||
use webkit2gtk::{
|
||||
PermissionRequestExt, SettingsExt, UserMediaPermissionRequest, UserMediaPermissionRequestExt,
|
||||
WebView, WebViewExt,
|
||||
};
|
||||
|
||||
/// Whether a media permission request should be granted: microphone only.
|
||||
pub(crate) fn allow_media_request(for_audio: bool, for_video: bool) -> bool {
|
||||
for_audio && !for_video
|
||||
}
|
||||
|
||||
pub fn enable_microphone(app: &AppHandle) {
|
||||
let Some(window) = app.get_webview_window("main") else {
|
||||
log::warn!("Microphone: no main window, capture will stay disabled");
|
||||
return;
|
||||
};
|
||||
let result = window.with_webview(|platform| {
|
||||
let webview: WebView = platform.inner();
|
||||
if let Some(settings) = WebViewExt::settings(&webview) {
|
||||
settings.set_enable_media_stream(true);
|
||||
}
|
||||
webview.connect_permission_request(|_, request| {
|
||||
if let Some(media) = request.downcast_ref::<UserMediaPermissionRequest>() {
|
||||
let ok =
|
||||
allow_media_request(media.is_for_audio_device(), media.is_for_video_device());
|
||||
if ok {
|
||||
request.allow();
|
||||
} else {
|
||||
request.deny();
|
||||
}
|
||||
log::info!(
|
||||
"Microphone: media permission request {}",
|
||||
if ok { "allowed" } else { "denied" }
|
||||
);
|
||||
} else {
|
||||
request.deny();
|
||||
}
|
||||
true // handled; do not fall through to WebKit's default
|
||||
});
|
||||
});
|
||||
if let Err(e) = result {
|
||||
log::warn!("Microphone: could not configure the webview: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::allow_media_request;
|
||||
|
||||
#[test]
|
||||
fn grants_audio_only() {
|
||||
assert!(allow_media_request(true, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn denies_camera_and_mixed_requests() {
|
||||
assert!(!allow_media_request(false, true));
|
||||
assert!(!allow_media_request(true, true));
|
||||
assert!(!allow_media_request(false, false));
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,12 @@
|
||||
"icons/icon.ico",
|
||||
"icons/icon.icns",
|
||||
"icons/icon.png"
|
||||
]
|
||||
],
|
||||
"linux": {
|
||||
"appimage": {
|
||||
"bundleMediaFramework": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"plugins": {}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { useCallback, useRef, useState } from "react";
|
||||
import * as commands from "../lib/tauri-commands";
|
||||
import { encodeWav } from "../lib/wav";
|
||||
import { useAppState } from "../store/appState";
|
||||
import { getMicrophoneStream } from "../lib/get-microphone";
|
||||
|
||||
export type SttState = "idle" | "recording" | "transcribing" | "error";
|
||||
|
||||
@@ -36,11 +37,8 @@ export function useSTT(sessionId: string, sendInput: (sessionId: string, data: s
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
};
|
||||
if (deviceId) {
|
||||
audioConstraints.deviceId = { exact: deviceId };
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: audioConstraints });
|
||||
const stream = await getMicrophoneStream(audioConstraints, deviceId);
|
||||
streamRef.current = stream;
|
||||
|
||||
const audioContext = new AudioContext({ sampleRate: 16000 });
|
||||
@@ -59,7 +57,10 @@ export function useSTT(sessionId: string, sendInput: (sessionId: string, data: s
|
||||
source.connect(processor);
|
||||
processor.connect(audioContext.destination);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
// Include the DOMException name: "Invalid constraint" alone doesn't say
|
||||
// whether it was OverconstrainedError, NotAllowedError, NotFoundError, etc.
|
||||
const msg = e instanceof Error ? `${e.name}: ${e.message}` : String(e);
|
||||
console.error("STT startRecording failed:", e);
|
||||
setError(msg);
|
||||
setState("error");
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
import * as commands from "../lib/tauri-commands";
|
||||
import { getMicrophoneStream } from "../lib/get-microphone";
|
||||
|
||||
type VoiceState = "inactive" | "starting" | "active" | "error";
|
||||
|
||||
@@ -27,13 +28,8 @@ export function useVoice(sessionId: string, deviceId?: string | null) {
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
};
|
||||
if (deviceId) {
|
||||
audioConstraints.deviceId = { exact: deviceId };
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: audioConstraints,
|
||||
});
|
||||
const stream = await getMicrophoneStream(audioConstraints, deviceId);
|
||||
streamRef.current = stream;
|
||||
|
||||
// 3. Create AudioContext at 16kHz (browser handles resampling)
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Open the microphone, relaxing constraints until the webview accepts them.
|
||||
*
|
||||
* Attempts, in order:
|
||||
* 1. saved device (exact) + base constraints
|
||||
* 2. base constraints, system default device
|
||||
* 3. bare `audio: true`
|
||||
*
|
||||
* Step 2 covers a stale saved deviceId (unplugged / regenerated IDs). Step 3
|
||||
* covers webviews (notably WebKitGTK on Linux) that reject one of the optional
|
||||
* processing constraints with `OverconstrainedError: Invalid constraint`.
|
||||
* Only constraint/device errors fall through; permission errors still throw.
|
||||
*/
|
||||
const RETRYABLE = new Set(["OverconstrainedError", "NotFoundError"]);
|
||||
|
||||
export async function getMicrophoneStream(
|
||||
base: MediaTrackConstraints,
|
||||
deviceId?: string | null,
|
||||
): Promise<MediaStream> {
|
||||
const attempts: (MediaTrackConstraints | boolean)[] = [];
|
||||
if (deviceId) attempts.push({ ...base, deviceId: { exact: deviceId } });
|
||||
attempts.push(base, true);
|
||||
|
||||
let lastError: unknown;
|
||||
for (const audio of attempts) {
|
||||
try {
|
||||
return await navigator.mediaDevices.getUserMedia({ audio });
|
||||
} catch (e) {
|
||||
lastError = e;
|
||||
if (!(e instanceof DOMException) || !RETRYABLE.has(e.name)) throw e;
|
||||
console.warn("getUserMedia attempt failed, relaxing constraints:", e.name, e.message, audio);
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
@@ -55,6 +55,28 @@
|
||||
# lands last: a fallback, never an override.
|
||||
#
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1b. The bundled libpipewire (microphone capture)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# `bundleMediaFramework` puts the GStreamer plugins in the image, including
|
||||
# libgstpipewire.so, which is what WebKitGTK uses to reach the microphone. That
|
||||
# plugin pulls libpipewire-0.3.so.0 in with it, and the copy linuxdeploy
|
||||
# bundles has the *builder's* plugin directories compiled in
|
||||
# (/usr/lib/x86_64-linux-gnu/spa-0.2 and .../pipewire-0.3). Neither is bundled
|
||||
# and neither exists on, say, Arch (/usr/lib/spa-0.2), so on the host:
|
||||
#
|
||||
# [E] pw.loop [loop.c:67 pw_loop_new()] can't make support.system handle:
|
||||
# No such file or directory
|
||||
#
|
||||
# and the microphone never opens. libpipewire is host-coupled in the same way
|
||||
# libwayland-client is: it has to agree with the SPA plugins, client config and
|
||||
# daemon actually running. Its client ABI is backward compatible, so the
|
||||
# bundled libgstpipewire.so links happily against a newer host copy. Unlike the
|
||||
# wayland case there is no useful fallback — without a host PipeWire daemon
|
||||
# there is nothing to capture from — so the bundled copy is simply removed;
|
||||
# GStreamer skips a plugin whose library will not load and the app still starts.
|
||||
#
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Metadata an AppImage manager needs
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
@@ -202,6 +224,17 @@ else
|
||||
echo "$LIB is not bundled — nothing to demote."
|
||||
fi
|
||||
|
||||
# Independent of the wayland demotion above, and equally unconditional on it.
|
||||
PW_LIB="libpipewire-0.3.so.0"
|
||||
pw_removed=false
|
||||
if [ -e "$root/usr/lib/$PW_LIB" ]; then
|
||||
rm -f "$root/usr/lib/$PW_LIB"
|
||||
pw_removed=true
|
||||
echo "Removed bundled $PW_LIB (the host's is used)."
|
||||
else
|
||||
echo "$PW_LIB is not bundled — nothing to remove."
|
||||
fi
|
||||
|
||||
# --- metadata -------------------------------------------------------------
|
||||
|
||||
# Version comes from the artifact rather than a second source that could drift.
|
||||
@@ -279,6 +312,9 @@ if [ "$demoted" = true ]; then
|
||||
[ -e "$out/$HOOK" ] || fail "the fallback hook is missing."
|
||||
grep -q "triple-c-wayland-fallback" "$out/AppRun" || fail "AppRun does not source the hook."
|
||||
fi
|
||||
if [ "$pw_removed" = true ]; then
|
||||
[ -e "$out/usr/lib/$PW_LIB" ] && fail "$PW_LIB is still bundled."
|
||||
fi
|
||||
[ -x "$out/usr/bin/triple-c" ] || fail "no executable usr/bin/triple-c."
|
||||
|
||||
# Every file readable by all, every program executable by all — what the
|
||||
|
||||
Reference in new issue
Block a user