ben/AndyTranscribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stream Whisper verbose metadata live without overflowing Reverb.

Browse Source 
Persist accumulated verbose_json on the recording and broadcast only transcript/whisper deltas so the show page can render language, segments, word timestamps, and confidence while a run is in progress.
This commit is contained in:
ben
2026-08-13 15:55:51 +02:00
parent 4c73620458
commit 187b6b5d12
16 changed files with 1646 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Services/TranscriptionService.php
+72 -62
View File
@@ -7,6 +7,7 @@ use Closure;
use Illuminate\Http\Client\PendingRequest;
use Illuminate\Http\Client\Response;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Laravel\Ai\Transcription;
use RuntimeException;
@@ -17,16 +18,15 @@ class TranscriptionService
/**
* Transcribe a recording with the local faster-whisper server.
*
* @param (Closure(string, int, ?string): void)|null $onProgress
* @param (Closure(string, int, ?string, ?array): void)|null $onProgress
* @param (Closure(): bool)|null $shouldContinue
*/
public function transcribe(Recording $recording, ?Closure $onProgress = null, ?Closure $shouldContinue = null): string
{
$report = $onProgress ?? static fn (string $message, int $percent, ?string $partial = null) => null;
$report = $onProgress ?? static fn (string $message, int $percent, ?string $partial = null, ?array $whisper = null) => null;
$model = config('ai.local_whisper_model', 'Systran/faster-whisper-base');
$report('Connecting to local faster-whisper server…', 30);
$report("Transcribing locally with {$model} (audio stays on this machine)…", 50);
$report("Transcribing locally with {$model} (audio stays on this machine)…", 0);
if (Transcription::isFaked()) {
$transcript = Transcription::fromStorage($recording->file_path)
@@ -36,15 +36,13 @@ class TranscriptionService
$transcript = $this->transcribeViaLocalWhisper($recording, $report, $model, $shouldContinue);
}
$report('Received transcript from local Whisper…', 85);
return (string) $transcript;
}
/**
* Call local Whisper, streaming SSE when the server supports it.
*
* @param Closure(string, int, ?string): void $report
* @param Closure(string, int, ?string, ?array): void $report
* @param (Closure(): bool)|null $shouldContinue
*/
private function transcribeViaLocalWhisper(Recording $recording, Closure $report, string $model, ?Closure $shouldContinue): string
@@ -62,9 +60,10 @@ class TranscriptionService
->withOptions(['stream' => true])
->post('audio/transcriptions', [
'model' => $model,
'response_format' => 'json',
'response_format' => 'verbose_json',
'stream' => 'true',
'without_timestamps' => 'false',
'timestamp_granularities[]' => 'word',
]);
if (! $response->successful()) {
@@ -76,30 +75,33 @@ class TranscriptionService
$contentType = strtolower((string) $response->header('Content-Type'));
if (! str_contains($contentType, 'text/event-stream')) {
return $this->extractTranscriptText($response);
Log::warning('Whisper did not stream SSE; falling back to a single JSON body', [
'recording_id' => $recording->id,
'content_type' => $contentType,
]);
return $this->extractTranscriptText($response, $report, $message);
}
return $this->consumeWhisperStream($response, $report, $message, $recording->duration_seconds, $shouldContinue);
return $this->consumeWhisperStream($response, $report, $message, $recording, $shouldContinue);
}
/**
* @param Closure(string, int, ?string): void $report
* @param Closure(string, int, ?string, ?array): void $report
* @param (Closure(): bool)|null $shouldContinue
*/
private function consumeWhisperStream(
Response $response,
Closure $report,
string $message,
?int $durationSeconds,
Recording $recording,
?Closure $shouldContinue,
): string {
$body = $response->toPsrResponse()->getBody();
$buffer = '';
$accumulated = '';
$lastPercent = 50;
$eventsWithoutTimestamp = 0;
$lastFlushAt = 0.0;
$pending = false;
$lastPercent = 0;
$idleReads = 0;
try {
while (! $body->eof()) {
@@ -112,43 +114,42 @@ class TranscriptionService
$chunk = $body->read(8192);
if ($chunk === '') {
break;
$idleReads++;
if ($idleReads >= 40) {
break;
}
usleep(50_000);
continue;
}
$idleReads = 0;
$buffer .= $chunk;
foreach ($this->stream->extractPayloads($buffer) as $payload) {
[$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending] = $this->ingestEvent(
[$accumulated, $lastPercent] = $this->ingestEvent(
$payload,
$accumulated,
$lastPercent,
$eventsWithoutTimestamp,
$durationSeconds,
$recording,
$report,
$message,
$lastFlushAt,
$pending,
);
}
}
foreach ($this->stream->flushBuffer($buffer) as $payload) {
[$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending] = $this->ingestEvent(
[$accumulated, $lastPercent] = $this->ingestEvent(
$payload,
$accumulated,
$lastPercent,
$eventsWithoutTimestamp,
$durationSeconds,
$recording,
$report,
$message,
$lastFlushAt,
$pending,
);
}
if ($pending && $accumulated !== '') {
$report($message, $lastPercent, $accumulated);
}
} finally {
$response->close();
}
@@ -161,61 +162,55 @@ class TranscriptionService
}
/**
* @param Closure(string, int, ?string): void $report
* @return array{0: string, 1: int, 2: int, 3: bool}
* @param Closure(string, int, ?string, ?array): void $report
* @return array{0: string, 1: int}
*/
private function ingestEvent(
string $payload,
string $accumulated,
int $lastPercent,
int $eventsWithoutTimestamp,
?int $durationSeconds,
Recording $recording,
Closure $report,
string $message,
float &$lastFlushAt,
bool $pending,
): array {
$event = $this->stream->parseEvent($payload);
if ($event === null) {
return [$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending];
return [$accumulated, $lastPercent];
}
$wasEmpty = $accumulated === '';
$accumulated = $this->stream->applyEvent($accumulated, $event);
$whisper = $event['whisper'] ?? [];
if ($accumulated === '') {
return [$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending];
if ($accumulated === '' && $whisper === []) {
return [$accumulated, $lastPercent];
}
$percent = $this->percentForEvent($event, $durationSeconds, $lastPercent, $eventsWithoutTimestamp);
$lastPercent = max($lastPercent, $percent);
$lastPercent = $this->percentForEvent($event, $recording, $lastPercent);
$now = microtime(true);
$shouldFlush = $wasEmpty || $event['done'] || ($now - $lastFlushAt) >= 1.0;
$report(
$message,
$lastPercent,
$accumulated === '' ? null : $accumulated,
$whisper === [] ? null : $whisper,
);
if ($shouldFlush) {
$report($message, $lastPercent, $accumulated);
$lastFlushAt = $now;
return [$accumulated, $lastPercent, $eventsWithoutTimestamp, false];
}
return [$accumulated, $lastPercent, $eventsWithoutTimestamp, true];
return [$accumulated, $lastPercent];
}
/**
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool} $event
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper?: array<string, mixed>} $event
*/
private function percentForEvent(array $event, ?int $durationSeconds, int $lastPercent, int &$eventsWithoutTimestamp): int
private function percentForEvent(array $event, Recording $recording, int $lastPercent): int
{
if ($event['end'] !== null && $durationSeconds !== null && $durationSeconds > 0) {
return (int) min(99, max(50, round(100 * $event['end'] / $durationSeconds)));
$duration = $recording->duration_seconds;
$end = $event['end'] ?? null;
if ($end === null || $duration === null || $duration <= 0) {
return $lastPercent;
}
$eventsWithoutTimestamp++;
return min(84, max($lastPercent, 50 + $eventsWithoutTimestamp));
return (int) min(99, max($lastPercent, round(100 * $end / $duration)));
}
private function localWhisperRequest(string $filename, string $path): PendingRequest
@@ -231,14 +226,29 @@ class TranscriptionService
->attach('file', fopen($path, 'r'), $filename);
}
private function extractTranscriptText(Response $response): string
/**
* @param Closure(string, int, ?string, ?array): void $report
*/
private function extractTranscriptText(Response $response, ?Closure $report = null, string $message = ''): string
{
$text = $response->json('text');
$json = $response->json();
if (! is_array($json)) {
throw new RuntimeException('Local transcription returned an empty transcript.');
}
$text = $json['text'] ?? null;
if (! is_string($text) || $text === '') {
throw new RuntimeException('Local transcription returned an empty transcript.');
}
$whisper = $this->stream->extractWhisperMeta($json);
if ($report !== null && $whisper !== []) {
$report($message, 99, $text, $whisper);
}
return $text;
}
}
app/Services/WhisperTranscriptionStream.php
+285 -9
View File
@@ -49,9 +49,9 @@ class WhisperTranscriptionStream
}
/**
* Parse one SSE JSON payload into append/replace/end/done fields.
* Parse one SSE JSON payload into append/replace/end/done/whisper fields.
*
* @return array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool}|null
* @return array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper: array<string, mixed>}|null
*/
public function parseEvent(string $json): ?array
{
@@ -62,20 +62,23 @@ class WhisperTranscriptionStream
}
$type = $data['type'] ?? null;
$whisper = $this->extractWhisperMeta($data);
$end = $this->latestAudioEnd($data, $whisper);
if ($type === 'transcript.text.delta') {
$delta = $data['delta'] ?? '';
if (! is_string($delta) || $delta === '') {
if ((! is_string($delta) || $delta === '') && $whisper === []) {
return null;
}
return [
'append' => $delta,
'append' => is_string($delta) && $delta !== '' ? $delta : null,
'replace' => null,
'end' => $this->nullableFloat($data['end'] ?? null),
'end' => $end,
'done' => false,
'legacy' => false,
'whisper' => $whisper,
];
}
@@ -85,19 +88,54 @@ class WhisperTranscriptionStream
return [
'append' => null,
'replace' => is_string($text) ? $text : '',
'end' => $this->nullableFloat($data['end'] ?? null),
'end' => $end,
'done' => true,
'legacy' => false,
'whisper' => $whisper,
];
}
if ($type === null && isset($data['text']) && is_string($data['text']) && $data['text'] !== '') {
if (isset($data['segments']) && is_array($data['segments'])) {
$text = $data['text'] ?? '';
if ((! is_string($text) || $text === '') && $whisper === []) {
return null;
}
return [
'append' => null,
'replace' => is_string($text) && $text !== '' ? $text : null,
'end' => $end,
'done' => true,
'legacy' => false,
'whisper' => $whisper,
];
}
if (
($type === null || $type === 'segment')
&& isset($data['text'])
&& is_string($data['text'])
&& $data['text'] !== ''
) {
return [
'append' => $data['text'],
'replace' => null,
'end' => $this->nullableFloat($data['end'] ?? null),
'end' => $end,
'done' => false,
'legacy' => true,
'whisper' => $whisper,
];
}
if ($whisper !== []) {
return [
'append' => null,
'replace' => null,
'end' => $end,
'done' => false,
'legacy' => false,
'whisper' => $whisper,
];
}
@@ -107,7 +145,7 @@ class WhisperTranscriptionStream
/**
* Apply a parsed event to the accumulated transcript.
*
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool} $event
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper?: array<string, mixed>} $event
*/
public function applyEvent(string $accumulated, array $event): string
{
@@ -134,6 +172,108 @@ class WhisperTranscriptionStream
return $accumulated.$chunk;
}
/**
* @param array<string, mixed> $data
* @return array<string, mixed>
*/
public function extractWhisperMeta(array $data): array
{
$meta = [];
if (isset($data['language']) && is_string($data['language']) && $data['language'] !== '') {
$meta['language'] = $data['language'];
}
if (is_numeric($data['duration'] ?? null)) {
$meta['duration'] = (float) $data['duration'];
}
if (isset($data['logprobs']) && is_array($data['logprobs'])) {
$logprobs = $this->normalizeLogprobs($data['logprobs']);
if ($logprobs !== []) {
$meta['logprobs'] = $logprobs;
}
}
$segment = $this->normalizeSegment($data);
if ($segment !== null && ($data['type'] ?? null) !== 'transcript.text.delta') {
$meta['segment'] = $segment;
}
if (isset($data['segments']) && is_array($data['segments'])) {
$segments = [];
foreach ($data['segments'] as $row) {
if (! is_array($row)) {
continue;
}
$normalized = $this->normalizeSegment($row);
if ($normalized !== null) {
$segments[] = $normalized;
}
}
if ($segments !== []) {
$meta['segments'] = $segments;
}
}
if (isset($data['words']) && is_array($data['words']) && ! isset($meta['segment']) && ! isset($meta['segments'])) {
$words = $this->normalizeWords($data['words']);
if ($words !== []) {
$meta['words'] = $words;
}
}
return $meta;
}
/**
* @param array<string, mixed> $data
* @return array<string, mixed>|null
*/
public function normalizeSegment(array $data): ?array
{
$hasDetail = isset($data['start'])
|| isset($data['end'])
|| isset($data['words'])
|| isset($data['avg_logprob'])
|| isset($data['tokens'])
|| isset($data['no_speech_prob'])
|| array_key_exists('id', $data);
if (! $hasDetail) {
return null;
}
$text = $data['text'] ?? null;
if (! is_string($text) || $text === '') {
return null;
}
$segment = [
'id' => is_numeric($data['id'] ?? null) ? (int) $data['id'] : null,
'seek' => is_numeric($data['seek'] ?? null) ? (int) $data['seek'] : null,
'start' => $this->nullableFloat($data['start'] ?? null),
'end' => $this->nullableFloat($data['end'] ?? null),
'text' => $text,
'tokens' => $this->normalizeTokens($data['tokens'] ?? null),
'temperature' => $this->nullableFloat($data['temperature'] ?? null),
'avg_logprob' => $this->nullableFloat($data['avg_logprob'] ?? null),
'compression_ratio' => $this->nullableFloat($data['compression_ratio'] ?? null),
'no_speech_prob' => $this->nullableFloat($data['no_speech_prob'] ?? null),
'words' => $this->normalizeWords($data['words'] ?? null),
];
return array_filter($segment, fn (mixed $value): bool => $value !== null && $value !== []);
}
/**
* @return list<string>
*/
@@ -150,6 +290,60 @@ class WhisperTranscriptionStream
return $payloads;
}
/**
* Latest media timestamp in this event (segment/word `end`). Never file `duration`.
*
* @param array<string, mixed> $data
* @param array<string, mixed> $whisper
*/
public function latestAudioEnd(array $data, array $whisper): ?float
{
$ends = [];
$direct = $this->nullableFloat($data['end'] ?? null);
if ($direct !== null) {
$ends[] = $direct;
}
$this->collectAudioEnds($ends, $whisper);
if (isset($data['words']) && is_array($data['words'])) {
$this->collectAudioEnds($ends, ['words' => $data['words']]);
}
return $ends === [] ? null : max($ends);
}
/**
* @param list<float> $ends
* @param array<string, mixed> $node
*/
private function collectAudioEnds(array &$ends, array $node): void
{
$end = $this->nullableFloat($node['end'] ?? null);
if ($end !== null) {
$ends[] = $end;
}
foreach (['words', 'segments'] as $key) {
if (! isset($node[$key]) || ! is_array($node[$key])) {
continue;
}
foreach ($node[$key] as $child) {
if (is_array($child)) {
$this->collectAudioEnds($ends, $child);
}
}
}
if (isset($node['segment']) && is_array($node['segment'])) {
$this->collectAudioEnds($ends, $node['segment']);
}
}
private function nullableFloat(mixed $value): ?float
{
if (! is_numeric($value)) {
@@ -158,4 +352,86 @@ class WhisperTranscriptionStream
return (float) $value;
}
/**
* @return list<int>|null
*/
private function normalizeTokens(mixed $tokens): ?array
{
if (! is_array($tokens) || $tokens === []) {
return null;
}
$normalized = [];
foreach ($tokens as $token) {
if (is_numeric($token)) {
$normalized[] = (int) $token;
}
}
return $normalized === [] ? null : $normalized;
}
/**
* @return list<array{word: string, start: ?float, end: ?float, probability: ?float}>|null
*/
private function normalizeWords(mixed $words): ?array
{
if (! is_array($words) || $words === []) {
return null;
}
$normalized = [];
foreach ($words as $word) {
if (! is_array($word)) {
continue;
}
$text = $word['word'] ?? $word['text'] ?? null;
if (! is_string($text) || $text === '') {
continue;
}
$normalized[] = array_filter([
'word' => $text,
'start' => $this->nullableFloat($word['start'] ?? null),
'end' => $this->nullableFloat($word['end'] ?? null),
'probability' => $this->nullableFloat($word['probability'] ?? $word['prob'] ?? null),
], fn (mixed $value): bool => $value !== null);
}
return $normalized === [] ? null : $normalized;
}
/**
* @return list<array{token: ?string, logprob: ?float}>
*/
private function normalizeLogprobs(array $logprobs): array
{
$normalized = [];
foreach ($logprobs as $row) {
if (! is_array($row)) {
continue;
}
$token = $row['token'] ?? $row['bytes'] ?? null;
$token = is_string($token) ? $token : null;
$logprob = $this->nullableFloat($row['logprob'] ?? $row['avg_logprob'] ?? null);
if ($token === null && $logprob === null) {
continue;
}
$normalized[] = array_filter([
'token' => $token,
'logprob' => $logprob,
], fn (mixed $value): bool => $value !== null);
}
return $normalized;
}
}