Files
Triple-C/app/src/hooks/useVoice.ts
T
shadowdaoandClaude Sonnet 5.5 c4cb04f011
Secret Scan / scan (push) Successful in 6s
Secret Scan / scan (pull_request) Successful in 6s
Build App (Preview) / compute-version (pull_request) Successful in 5s
Build App (Preview) / create-release (pull_request) Successful in 6s
Build App (Preview) / test (pull_request) Successful in 4m8s
Build App (Preview) / build-macos (pull_request) Successful in 3m18s
Build App (Preview) / build-windows (pull_request) Successful in 8m8s
Build App (Preview) / build-linux (pull_request) Successful in 6m48s
Build App (Preview) / prune-previews (pull_request) Successful in 2s
Fix speech-to-text microphone on Linux AppImage
The mic button failed on Linux with a chain of distinct problems, each
hidden behind the previous one:

- The AppImage bundled GStreamer libraries but no plugins, so WebKitGTK
  found 0 capture devices. Enable bundleMediaFramework and install the
  GStreamer plugin packages on the CI runners.
- The bundled libpipewire had the builder's SPA plugin paths compiled in
  and failed on hosts with a different layout. finalize-appimage.sh now
  drops it so the host's is used (host-coupled, like libwayland-client).
- WebKitGTK denies getUserMedia until the embedder enables media streams
  and answers the permission request; wry does neither. Add a Linux-only
  handler for the main window that grants audio-only requests and denies
  everything else.
- getUserMedia now steps down from the saved device to the default device
  to bare audio:true on OverconstrainedError/NotFoundError, and the STT
  error shows the DOMException name.

Verified on CachyOS: permission request allowed, recording works.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-05 06:24:11 -08:00

100 lines
3.2 KiB
TypeScript

import { useCallback, useRef, useState } from "react";
import * as commands from "../lib/tauri-commands";
import { getMicrophoneStream } from "../lib/get-microphone";
type VoiceState = "inactive" | "starting" | "active" | "error";
export function useVoice(sessionId: string, deviceId?: string | null) {
const [state, setState] = useState<VoiceState>("inactive");
const [error, setError] = useState<string | null>(null);
const audioContextRef = useRef<AudioContext | null>(null);
const streamRef = useRef<MediaStream | null>(null);
const workletRef = useRef<AudioWorkletNode | null>(null);
const start = useCallback(async () => {
if (state === "active" || state === "starting") return;
setState("starting");
setError(null);
try {
// 1. Start the audio bridge in the container (creates FIFO writer)
await commands.startAudioBridge(sessionId);
// 2. Get microphone access (use specific device if configured)
const audioConstraints: MediaTrackConstraints = {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
};
const stream = await getMicrophoneStream(audioConstraints, deviceId);
streamRef.current = stream;
// 3. Create AudioContext at 16kHz (browser handles resampling)
const audioContext = new AudioContext({ sampleRate: 16000 });
audioContextRef.current = audioContext;
// 4. Load AudioWorklet processor
await audioContext.audioWorklet.addModule("/audio-capture-processor.js");
// 5. Connect: mic → worklet → (silent) destination
const source = audioContext.createMediaStreamSource(stream);
const processor = new AudioWorkletNode(audioContext, "audio-capture-processor");
workletRef.current = processor;
// 6. Handle PCM chunks from the worklet
processor.port.onmessage = (event: MessageEvent<ArrayBuffer>) => {
const bytes = Array.from(new Uint8Array(event.data));
commands.sendAudioData(sessionId, bytes).catch(() => {
// Audio bridge may have been closed — ignore send errors
});
};
source.connect(processor);
processor.connect(audioContext.destination);
setState("active");
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
setError(msg);
setState("error");
// Clean up on failure
await commands.stopAudioBridge(sessionId).catch(() => {});
}
}, [sessionId, state, deviceId]);
const stop = useCallback(async () => {
// Tear down audio pipeline
workletRef.current?.disconnect();
workletRef.current = null;
if (audioContextRef.current) {
await audioContextRef.current.close().catch(() => {});
audioContextRef.current = null;
}
if (streamRef.current) {
streamRef.current.getTracks().forEach((t) => t.stop());
streamRef.current = null;
}
// Stop the container-side audio bridge
await commands.stopAudioBridge(sessionId).catch(() => {});
setState("inactive");
setError(null);
}, [sessionId]);
const toggle = useCallback(async () => {
if (state === "active") {
await stop();
} else {
await start();
}
}, [state, start, stop]);
return { state, error, start, stop, toggle };
}