Archived
First commit of local Closed Caption App
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
const axios = require('axios');
|
||||
const FormData = require('form-data');
|
||||
const settingsManager = require('../utils/settingsManager');
|
||||
|
||||
async function transcribe(audioBuffer) {
|
||||
// Check for API key in settings first, then env
|
||||
const apiKey = settingsManager.getSetting('elevenlabsApiKey') || process.env.ELEVENLABS_API_KEY;
|
||||
|
||||
if (!apiKey) {
|
||||
throw new Error('ElevenLabs API key not configured');
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('model_id', 'scribe_v1');
|
||||
formData.append('file', audioBuffer, {
|
||||
filename: 'audio.webm',
|
||||
contentType: 'audio/webm'
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await axios.post(
|
||||
'https://api.elevenlabs.io/v1/speech-to-text',
|
||||
formData,
|
||||
{
|
||||
headers: {
|
||||
...formData.getHeaders(),
|
||||
'xi-api-key': apiKey
|
||||
},
|
||||
maxContentLength: Infinity,
|
||||
maxBodyLength: Infinity
|
||||
}
|
||||
);
|
||||
|
||||
// Handle the response
|
||||
if (response.data && typeof response.data.text === 'string') {
|
||||
// Return the text even if it's empty (no speech detected)
|
||||
return response.data.text;
|
||||
} else if (response.data && response.data.transcription) {
|
||||
return response.data.transcription;
|
||||
} else if (typeof response.data === 'string') {
|
||||
return response.data;
|
||||
} else {
|
||||
console.log('ElevenLabs unexpected response:', JSON.stringify(response.data, null, 2));
|
||||
throw new Error('Unexpected response format from ElevenLabs API');
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.response) {
|
||||
console.error('ElevenLabs API error response:', error.response.data);
|
||||
throw new Error(`ElevenLabs API error: ${error.response.data.detail?.message || error.response.data.error || error.response.statusText}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
transcribe
|
||||
};
|
||||
@@ -0,0 +1,192 @@
|
||||
const { spawn } = require('child_process');
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const OpenAI = require('openai');
|
||||
const transcriptionQueue = require('../utils/transcriptionQueue');
|
||||
const { getAudioLevel } = require('../utils/audioUtils');
|
||||
const settingsManager = require('../utils/settingsManager');
|
||||
|
||||
const openai = process.env.OPENAI_API_KEY ? new OpenAI({
|
||||
apiKey: process.env.OPENAI_API_KEY
|
||||
}) : null;
|
||||
|
||||
// Get audio threshold dynamically from settings
|
||||
function getAudioThreshold() {
|
||||
return settingsManager.getSetting('audioThreshold') || parseFloat(process.env.AUDIO_THRESHOLD) || 0.01;
|
||||
}
|
||||
|
||||
async function transcribeLocal(audioBuffer) {
|
||||
// Check audio level before processing
|
||||
const audioLevel = getAudioLevel(audioBuffer);
|
||||
const audioThreshold = getAudioThreshold();
|
||||
|
||||
if (audioLevel < audioThreshold) {
|
||||
console.log(`Audio level ${audioLevel.toFixed(4)} below threshold ${audioThreshold}, skipping transcription`);
|
||||
return ''; // Return empty string for silence
|
||||
}
|
||||
|
||||
console.log(`Audio level ${audioLevel.toFixed(4)} above threshold, queuing for transcription`);
|
||||
|
||||
// Use queue to prevent multiple concurrent Whisper processes
|
||||
return transcriptionQueue.enqueue(async () => {
|
||||
const tempDir = os.tmpdir();
|
||||
const tempFile = path.join(tempDir, `audio_${Date.now()}.wav`);
|
||||
|
||||
try {
|
||||
await fs.writeFile(tempFile, audioBuffer);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
// Build Whisper command with current settings (not cached)
|
||||
const currentModel = settingsManager.getSetting('whisperModel') || process.env.WHISPER_MODEL || 'base';
|
||||
const whisperArgs = [
|
||||
tempFile,
|
||||
'--model', currentModel,
|
||||
'--language', 'en',
|
||||
'--task', 'transcribe',
|
||||
'--output_format', 'txt',
|
||||
'--output_dir', tempDir
|
||||
];
|
||||
|
||||
// Get current GPU settings (not cached)
|
||||
const gpuAcceleration = settingsManager.getSetting('gpuAcceleration') || 'none';
|
||||
|
||||
switch (gpuAcceleration) {
|
||||
case 'cuda':
|
||||
// NVIDIA CUDA
|
||||
whisperArgs.push('--device', 'cuda');
|
||||
whisperArgs.push('--fp16', 'True');
|
||||
break;
|
||||
|
||||
case 'rocm':
|
||||
// AMD ROCm (requires PyTorch built with ROCm)
|
||||
whisperArgs.push('--device', 'cuda'); // ROCm uses CUDA interface
|
||||
whisperArgs.push('--fp16', 'True');
|
||||
// Set environment variable for ROCm
|
||||
process.env.HSA_OVERRIDE_GFX_VERSION = '10.3.0'; // May need adjustment
|
||||
break;
|
||||
|
||||
case 'openvino':
|
||||
// Intel OpenVINO
|
||||
whisperArgs.push('--device', 'cpu'); // OpenVINO uses CPU device
|
||||
// Note: Requires whisper-openvino fork or custom implementation
|
||||
whisperArgs.push('--fp16', 'False');
|
||||
whisperArgs.push('--threads', '4');
|
||||
break;
|
||||
|
||||
default:
|
||||
// CPU optimizations
|
||||
whisperArgs.push('--device', 'cpu');
|
||||
whisperArgs.push('--fp16', 'False');
|
||||
whisperArgs.push('--threads', '2');
|
||||
}
|
||||
|
||||
console.log(`Starting Whisper with model: ${currentModel}, GPU: ${gpuAcceleration}`);
|
||||
console.log('Whisper args:', whisperArgs.join(' '));
|
||||
const whisperProcess = spawn('whisper', whisperArgs);
|
||||
|
||||
let output = '';
|
||||
let error = '';
|
||||
|
||||
whisperProcess.stdout.on('data', (data) => {
|
||||
output += data.toString();
|
||||
});
|
||||
|
||||
whisperProcess.stderr.on('data', (data) => {
|
||||
error += data.toString();
|
||||
});
|
||||
|
||||
whisperProcess.on('close', async (code) => {
|
||||
if (code !== 0) {
|
||||
reject(new Error(`Whisper process exited with code ${code}: ${error}`));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const outputFile = tempFile.replace('.wav', '.txt');
|
||||
const transcription = await fs.readFile(outputFile, 'utf8');
|
||||
|
||||
// Clean up files
|
||||
await fs.unlink(tempFile).catch(() => {});
|
||||
await fs.unlink(outputFile).catch(() => {});
|
||||
|
||||
resolve(transcription.trim());
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
|
||||
// Kill process if it takes too long (30 seconds timeout)
|
||||
const timeout = setTimeout(() => {
|
||||
whisperProcess.kill('SIGKILL');
|
||||
reject(new Error('Whisper process timeout'));
|
||||
}, 30000);
|
||||
|
||||
whisperProcess.on('close', () => {
|
||||
clearTimeout(timeout);
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
await fs.unlink(tempFile).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}, `whisper_${Date.now()}`);
|
||||
}
|
||||
|
||||
async function transcribeRemote(audioBuffer) {
|
||||
// Check for API key in settings first, then env
|
||||
const apiKey = settingsManager.getSetting('openaiApiKey') || process.env.OPENAI_API_KEY;
|
||||
|
||||
if (!apiKey) {
|
||||
throw new Error('OpenAI API key not configured');
|
||||
}
|
||||
|
||||
// Create OpenAI client with current API key
|
||||
const currentOpenAI = new OpenAI({ apiKey });
|
||||
|
||||
// Check audio level before processing
|
||||
const audioLevel = getAudioLevel(audioBuffer);
|
||||
const audioThreshold = getAudioThreshold();
|
||||
|
||||
if (audioLevel < audioThreshold) {
|
||||
console.log(`Audio level ${audioLevel.toFixed(4)} below threshold ${audioThreshold}, skipping remote transcription`);
|
||||
return ''; // Return empty string for silence
|
||||
}
|
||||
|
||||
console.log(`Audio level ${audioLevel.toFixed(4)} above threshold, sending to OpenAI`);
|
||||
|
||||
// Use queue for remote requests too to prevent overwhelming the API
|
||||
return transcriptionQueue.enqueue(async () => {
|
||||
const tempDir = os.tmpdir();
|
||||
const tempFile = path.join(tempDir, `audio_${Date.now()}.wav`);
|
||||
|
||||
try {
|
||||
await fs.writeFile(tempFile, audioBuffer);
|
||||
|
||||
const fileBuffer = await fs.readFile(tempFile);
|
||||
const transcription = await currentOpenAI.audio.transcriptions.create({
|
||||
file: new File([fileBuffer], 'audio.wav', { type: 'audio/wav' }),
|
||||
model: "whisper-1",
|
||||
language: "en"
|
||||
});
|
||||
|
||||
await fs.unlink(tempFile).catch(() => {});
|
||||
|
||||
return transcription.text;
|
||||
} catch (error) {
|
||||
await fs.unlink(tempFile).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}, `openai_${Date.now()}`);
|
||||
}
|
||||
|
||||
// Clean up old queued jobs periodically
|
||||
setInterval(() => {
|
||||
transcriptionQueue.clearOldJobs(30000); // Clear jobs older than 30 seconds
|
||||
}, 10000);
|
||||
|
||||
module.exports = {
|
||||
transcribeLocal,
|
||||
transcribeRemote,
|
||||
getQueueStatus: () => transcriptionQueue.getStatus()
|
||||
};
|
||||
Reference in New Issue
Block a user