diff --git a/app/Events/RecordingTranscriptionUpdated.php b/app/Events/RecordingTranscriptionUpdated.php index d48bc79..1c8f83b 100644 --- a/app/Events/RecordingTranscriptionUpdated.php +++ b/app/Events/RecordingTranscriptionUpdated.php @@ -14,9 +14,21 @@ class RecordingTranscriptionUpdated implements ShouldBroadcastNow use Dispatchable, InteractsWithSockets, SerializesModels; /** - * Create a new event instance. + * Reverb's default max payload is 10KB. Keep streamed text well under that. */ - public function __construct(public Recording $recording) {} + public const MAX_DELTA_BYTES = 8_000; + + /** + * Create a new event instance. + * + * @param array|null $whisperDelta + */ + public function __construct( + public Recording $recording, + public ?string $transcriptDelta = null, + public bool $transcriptReplace = false, + public ?array $whisperDelta = null, + ) {} /** * Get the channels the event should broadcast on. @@ -41,11 +53,89 @@ class RecordingTranscriptionUpdated implements ShouldBroadcastNow */ public function broadcastWith(): array { - return array_merge( + $payload = array_merge( $this->recording->transcriptionStatusPayload(), [ 'word_count' => $this->recording->word_count, ], ); + + unset($payload['transcript'], $payload['whisper']); + + $delta = $this->transcriptDelta; + + if ($delta !== null && $delta !== '' && strlen($delta) <= self::MAX_DELTA_BYTES) { + $payload['transcript_delta'] = $delta; + $payload['transcript_replace'] = $this->transcriptReplace; + } + + $whisper = $this->compactWhisperDelta($this->whisperDelta); + + if ($whisper !== null) { + $payload['whisper_delta'] = $whisper; + } + + return $payload; + } + + /** + * @param array|null $whisper + * @return array|null + */ + private function compactWhisperDelta(?array $whisper): ?array + { + if ($whisper === null || $whisper === []) { + return null; + } + + // Accumulated segment lists belong in HTTP hydrate, not on Reverb. + unset($whisper['segments']); + + if ($whisper === []) { + return null; + } + + foreach (['tokens', 'logprobs', 'words'] as $drop) { + $encoded = json_encode($whisper); + + if (! is_string($encoded) || strlen($encoded) <= self::MAX_DELTA_BYTES) { + return $whisper; + } + + $whisper = $this->dropWhisperField($whisper, $drop); + } + + $encoded = json_encode($whisper); + + if (! is_string($encoded) || strlen($encoded) > self::MAX_DELTA_BYTES) { + return null; + } + + return $whisper; + } + + /** + * @param array $whisper + * @return array + */ + private function dropWhisperField(array $whisper, string $field): array + { + unset($whisper[$field]); + + if (isset($whisper['segment']) && is_array($whisper['segment'])) { + unset($whisper['segment'][$field]); + } + + if (isset($whisper['segments']) && is_array($whisper['segments'])) { + $whisper['segments'] = array_map(function (mixed $segment) use ($field): mixed { + if (is_array($segment)) { + unset($segment[$field]); + } + + return $segment; + }, $whisper['segments']); + } + + return $whisper; } } diff --git a/app/Jobs/TranscribeRecording.php b/app/Jobs/TranscribeRecording.php index d01fd59..4c0f867 100644 --- a/app/Jobs/TranscribeRecording.php +++ b/app/Jobs/TranscribeRecording.php @@ -67,17 +67,16 @@ class TranscribeRecording implements ShouldQueue 'transcription_status' => 'processing', 'transcription_started_at' => $this->recording->transcription_started_at ?? now(), 'transcription_error' => null, + 'transcription_verbose' => null, ])->save(); $this->runStartedAt ??= $this->recording->transcription_started_at?->toIso8601String(); - $this->reportIfOwned('Preparing audio file…', 15); - try { $text = $transcription->transcribe( $this->recording, - function (string $message, int $percent, ?string $partialTranscript = null): void { - $this->reportIfOwned($message, $percent, $partialTranscript); + function (string $message, int $percent, ?string $partialTranscript = null, ?array $whisper = null): void { + $this->reportIfOwned($message, $percent, $partialTranscript, $whisper); }, fn (): bool => Recording::query()->find($this->recording->id) ?->ownsTranscriptionRun($this->runStartedAt) ?? false, @@ -135,11 +134,6 @@ class TranscribeRecording implements ShouldQueue { $this->recording->refresh(); - if ($this->recording->ownsTranscriptionRun($this->runStartedAt)) { - $this->reportIfOwned('Saving transcript…', 90); - $this->recording->refresh(); - } - if ($this->recording->ownsTranscriptionRun($this->runStartedAt)) { $this->recording->markTranscriptionComplete($text); @@ -172,12 +166,12 @@ class TranscribeRecording implements ShouldQueue return false; } - private function reportIfOwned(string $message, int $percent, ?string $partialTranscript = null): void + private function reportIfOwned(string $message, int $percent, ?string $partialTranscript = null, ?array $whisper = null): void { if (! $this->recording->ownsTranscriptionRun($this->runStartedAt)) { return; } - $this->recording->reportProgress($message, $percent, partialTranscript: $partialTranscript); + $this->recording->reportProgress($message, $percent, partialTranscript: $partialTranscript, whisperDelta: $whisper); } } diff --git a/app/Livewire/Recordings/Index.php b/app/Livewire/Recordings/Index.php index 3d3a8f7..f9b10e6 100644 --- a/app/Livewire/Recordings/Index.php +++ b/app/Livewire/Recordings/Index.php @@ -10,7 +10,6 @@ use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Support\Facades\Auth; use Illuminate\Support\Facades\Gate; use Livewire\Attributes\Layout; -use Livewire\Attributes\On; use Livewire\Attributes\Title; use Livewire\Attributes\Url; use Livewire\Component; @@ -22,8 +21,6 @@ class Index extends Component { use WithPagination; - public int $userId; - #[Url(as: 'q', history: true)] public string $search = ''; @@ -45,7 +42,6 @@ class Index extends Component public function mount(): void { - $this->userId = (int) Auth::id(); $this->normalizeSort(); } @@ -70,15 +66,6 @@ class Index extends Component $this->resetPage(); } - /** - * Re-render when any of this user's recordings broadcast a status change. - */ - #[On('echo-private:user.{userId}.recordings,.RecordingTranscriptionUpdated')] - public function onTranscriptionUpdated(): void - { - // - } - public function queuePending(): void { $queued = 0; diff --git a/app/Livewire/Recordings/Show.php b/app/Livewire/Recordings/Show.php index d64507c..22edfb3 100644 --- a/app/Livewire/Recordings/Show.php +++ b/app/Livewire/Recordings/Show.php @@ -7,7 +7,6 @@ use Flux\Flux; use Illuminate\Contracts\View\View; use Illuminate\Support\Facades\Gate; use Livewire\Attributes\Layout; -use Livewire\Attributes\On; use Livewire\Component; #[Layout('layouts.app')] @@ -15,8 +14,6 @@ class Show extends Component { public Recording $recording; - public int $userId; - public function mount(Recording $recording): void { Gate::authorize('view', $recording); @@ -25,22 +22,6 @@ class Show extends Component $recording->refresh(); $this->recording = $recording; - $this->userId = (int) $recording->user_id; - } - - /** - * Re-render when this recording broadcasts a status change (same channel as the index). - * - * @param array $event - */ - #[On('echo-private:user.{userId}.recordings,.RecordingTranscriptionUpdated')] - public function onTranscriptionUpdated(array $event = []): void - { - if (isset($event['id']) && (int) $event['id'] !== (int) $this->recording->id) { - return; - } - - $this->recording->refresh(); } public function startTranscription(): void @@ -83,10 +64,6 @@ class Show extends Component public function render(): View { - if ($this->recording->isTranscribing()) { - $this->recording->refresh(); - } - return view('livewire.recordings.show') ->title($this->recording->title); } diff --git a/app/Models/Recording.php b/app/Models/Recording.php index 9c6fcbc..b3a51cb 100644 --- a/app/Models/Recording.php +++ b/app/Models/Recording.php @@ -12,6 +12,7 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use Throwable; @@ -33,6 +34,7 @@ class Recording extends Model 'file_size_bytes', 'content_hash', 'transcript', + 'transcription_verbose', 'transcription_status', 'transcription_progress', 'transcription_percent', @@ -57,6 +59,7 @@ class Recording extends Model 'transcription_duration_seconds' => 'integer', 'file_size_bytes' => 'integer', 'transcription_percent' => 'integer', + 'transcription_verbose' => 'array', 'user_id' => 'integer', ]; } @@ -146,7 +149,7 @@ class Recording extends Model ]); $recording = $this->fresh(); - RecordingTranscriptionUpdated::dispatch($recording); + $this->broadcastTranscriptionUpdated(); TranscribeRecording::dispatch($recording); } @@ -424,7 +427,7 @@ class Recording extends Model 'transcription_error' => 'Stopped by user', ])->save(); - RecordingTranscriptionUpdated::dispatch($this->fresh()); + $this->broadcastTranscriptionUpdated(); } /** @@ -458,7 +461,7 @@ class Recording extends Model 'transcription_duration_seconds' => $durationSeconds, ])->save(); - RecordingTranscriptionUpdated::dispatch($this->fresh()); + $this->broadcastTranscriptionUpdated(); } /** @@ -477,7 +480,7 @@ class Recording extends Model 'transcription_error' => $message, ])->save(); - RecordingTranscriptionUpdated::dispatch($this->fresh()); + $this->broadcastTranscriptionUpdated(); } /** @@ -501,9 +504,13 @@ class Recording extends Model /** * Update the live progress fields shown in the UI. + * + * @param array|null $whisperDelta */ - public function reportProgress(string $message, int $percent, string $status = 'processing', ?string $partialTranscript = null): void + public function reportProgress(string $message, int $percent, string $status = 'processing', ?string $partialTranscript = null, ?array $whisperDelta = null): void { + $diff = $this->transcriptBroadcastDiff($partialTranscript); + $attributes = [ 'transcription_status' => $status, 'transcription_progress' => $message, @@ -515,17 +522,158 @@ class Recording extends Model $attributes['transcript'] = $partialTranscript; } + if ($whisperDelta !== null) { + $attributes['transcription_verbose'] = $this->mergeWhisperVerbose($whisperDelta); + } + $this->forceFill($attributes)->save(); - RecordingTranscriptionUpdated::dispatch($this->fresh()); + $this->broadcastTranscriptionUpdated($diff['delta'], $diff['replace'], $whisperDelta); } /** * Broadcast the current transcription status to connected browsers. + * + * @param array|null $whisperDelta */ - public function broadcastTranscriptionUpdated(): void + public function broadcastTranscriptionUpdated(?string $transcriptDelta = null, bool $transcriptReplace = false, ?array $whisperDelta = null): void { - RecordingTranscriptionUpdated::dispatch($this->fresh() ?? $this); + try { + RecordingTranscriptionUpdated::dispatch( + $this->fresh() ?? $this, + $transcriptDelta, + $transcriptReplace, + $whisperDelta, + ); + } catch (Throwable $e) { + Log::warning('Failed to broadcast transcription status', [ + 'recording_id' => $this->id, + 'message' => $e->getMessage(), + ]); + } + } + + /** + * Incremental text to push over Reverb instead of the full transcript. + * + * @return array{delta: ?string, replace: bool} + */ + public function transcriptBroadcastDiff(?string $next): array + { + if ($next === null) { + return ['delta' => null, 'replace' => false]; + } + + $previous = (string) $this->transcript; + + if ($previous === '' || ! str_starts_with($next, $previous)) { + return ['delta' => $next, 'replace' => true]; + } + + $delta = substr($next, strlen($previous)); + + return ['delta' => $delta === '' ? null : $delta, 'replace' => false]; + } + + /** + * Fold a streamed Whisper chunk into the stored verbose_json snapshot. + * + * @param array $delta + * @return array + */ + public function mergeWhisperVerbose(array $delta): array + { + $verbose = is_array($this->transcription_verbose) ? $this->transcription_verbose : []; + + if (isset($delta['language']) && is_string($delta['language'])) { + $verbose['language'] = $delta['language']; + } + + if (isset($delta['duration']) && is_numeric($delta['duration'])) { + $verbose['duration'] = (float) $delta['duration']; + } + + $segments = is_array($verbose['segments'] ?? null) ? $verbose['segments'] : []; + + if (isset($delta['segments']) && is_array($delta['segments'])) { + $incoming = []; + + foreach ($delta['segments'] as $segment) { + if (is_array($segment)) { + $incoming[] = $segment; + } + } + + if ($incoming !== []) { + $segments = $incoming; + } + } elseif (isset($delta['segment']) && is_array($delta['segment'])) { + $segments = $this->appendVerboseSegment($segments, $delta['segment']); + } + + if (isset($delta['words']) && is_array($delta['words']) && $delta['words'] !== []) { + $last = $segments === [] ? null : count($segments) - 1; + + if ($last === null) { + $segments[] = ['words' => $delta['words']]; + } else { + $existing = is_array($segments[$last]['words'] ?? null) ? $segments[$last]['words'] : []; + $segments[$last]['words'] = array_merge($existing, $delta['words']); + } + } + + $logprobs = is_array($verbose['logprobs'] ?? null) ? $verbose['logprobs'] : []; + + if (isset($delta['logprobs']) && is_array($delta['logprobs'])) { + foreach ($delta['logprobs'] as $row) { + if (is_array($row)) { + $logprobs[] = $row; + } + } + } + + if ($segments !== []) { + $verbose['segments'] = $segments; + } + + if ($logprobs !== []) { + $verbose['logprobs'] = $logprobs; + } + + return $verbose; + } + + /** + * @param list> $segments + * @param array $segment + * @return list> + */ + private function appendVerboseSegment(array $segments, array $segment): array + { + $key = $this->verboseSegmentKey($segment); + + foreach ($segments as $existing) { + if ($this->verboseSegmentKey($existing) === $key) { + return $segments; + } + } + + $segments[] = $segment; + + return $segments; + } + + /** + * @param array $segment + */ + private function verboseSegmentKey(array $segment): string + { + return implode('|', [ + $segment['id'] ?? '', + $segment['start'] ?? '', + $segment['end'] ?? '', + $segment['text'] ?? '', + ]); } /** @@ -596,6 +744,7 @@ class Recording extends Model 'is_active' => $this->isTranscribing(), 'has_transcript' => filled($this->transcript), 'transcript' => $this->transcript, + 'whisper' => $this->transcription_verbose, 'transcribed_at' => $this->transcribed_at?->toIso8601String(), ]; } diff --git a/app/Services/TranscriptionService.php b/app/Services/TranscriptionService.php index bc98d1a..2dddf38 100644 --- a/app/Services/TranscriptionService.php +++ b/app/Services/TranscriptionService.php @@ -7,6 +7,7 @@ use Closure; use Illuminate\Http\Client\PendingRequest; use Illuminate\Http\Client\Response; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Log; use Laravel\Ai\Transcription; use RuntimeException; @@ -17,16 +18,15 @@ class TranscriptionService /** * Transcribe a recording with the local faster-whisper server. * - * @param (Closure(string, int, ?string): void)|null $onProgress + * @param (Closure(string, int, ?string, ?array): void)|null $onProgress * @param (Closure(): bool)|null $shouldContinue */ public function transcribe(Recording $recording, ?Closure $onProgress = null, ?Closure $shouldContinue = null): string { - $report = $onProgress ?? static fn (string $message, int $percent, ?string $partial = null) => null; + $report = $onProgress ?? static fn (string $message, int $percent, ?string $partial = null, ?array $whisper = null) => null; $model = config('ai.local_whisper_model', 'Systran/faster-whisper-base'); - $report('Connecting to local faster-whisper server…', 30); - $report("Transcribing locally with {$model} (audio stays on this machine)…", 50); + $report("Transcribing locally with {$model} (audio stays on this machine)…", 0); if (Transcription::isFaked()) { $transcript = Transcription::fromStorage($recording->file_path) @@ -36,15 +36,13 @@ class TranscriptionService $transcript = $this->transcribeViaLocalWhisper($recording, $report, $model, $shouldContinue); } - $report('Received transcript from local Whisper…', 85); - return (string) $transcript; } /** * Call local Whisper, streaming SSE when the server supports it. * - * @param Closure(string, int, ?string): void $report + * @param Closure(string, int, ?string, ?array): void $report * @param (Closure(): bool)|null $shouldContinue */ private function transcribeViaLocalWhisper(Recording $recording, Closure $report, string $model, ?Closure $shouldContinue): string @@ -62,9 +60,10 @@ class TranscriptionService ->withOptions(['stream' => true]) ->post('audio/transcriptions', [ 'model' => $model, - 'response_format' => 'json', + 'response_format' => 'verbose_json', 'stream' => 'true', 'without_timestamps' => 'false', + 'timestamp_granularities[]' => 'word', ]); if (! $response->successful()) { @@ -76,30 +75,33 @@ class TranscriptionService $contentType = strtolower((string) $response->header('Content-Type')); if (! str_contains($contentType, 'text/event-stream')) { - return $this->extractTranscriptText($response); + Log::warning('Whisper did not stream SSE; falling back to a single JSON body', [ + 'recording_id' => $recording->id, + 'content_type' => $contentType, + ]); + + return $this->extractTranscriptText($response, $report, $message); } - return $this->consumeWhisperStream($response, $report, $message, $recording->duration_seconds, $shouldContinue); + return $this->consumeWhisperStream($response, $report, $message, $recording, $shouldContinue); } /** - * @param Closure(string, int, ?string): void $report + * @param Closure(string, int, ?string, ?array): void $report * @param (Closure(): bool)|null $shouldContinue */ private function consumeWhisperStream( Response $response, Closure $report, string $message, - ?int $durationSeconds, + Recording $recording, ?Closure $shouldContinue, ): string { $body = $response->toPsrResponse()->getBody(); $buffer = ''; $accumulated = ''; - $lastPercent = 50; - $eventsWithoutTimestamp = 0; - $lastFlushAt = 0.0; - $pending = false; + $lastPercent = 0; + $idleReads = 0; try { while (! $body->eof()) { @@ -112,43 +114,42 @@ class TranscriptionService $chunk = $body->read(8192); if ($chunk === '') { - break; + $idleReads++; + + if ($idleReads >= 40) { + break; + } + + usleep(50_000); + + continue; } + $idleReads = 0; $buffer .= $chunk; foreach ($this->stream->extractPayloads($buffer) as $payload) { - [$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending] = $this->ingestEvent( + [$accumulated, $lastPercent] = $this->ingestEvent( $payload, $accumulated, $lastPercent, - $eventsWithoutTimestamp, - $durationSeconds, + $recording, $report, $message, - $lastFlushAt, - $pending, ); } } foreach ($this->stream->flushBuffer($buffer) as $payload) { - [$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending] = $this->ingestEvent( + [$accumulated, $lastPercent] = $this->ingestEvent( $payload, $accumulated, $lastPercent, - $eventsWithoutTimestamp, - $durationSeconds, + $recording, $report, $message, - $lastFlushAt, - $pending, ); } - - if ($pending && $accumulated !== '') { - $report($message, $lastPercent, $accumulated); - } } finally { $response->close(); } @@ -161,61 +162,55 @@ class TranscriptionService } /** - * @param Closure(string, int, ?string): void $report - * @return array{0: string, 1: int, 2: int, 3: bool} + * @param Closure(string, int, ?string, ?array): void $report + * @return array{0: string, 1: int} */ private function ingestEvent( string $payload, string $accumulated, int $lastPercent, - int $eventsWithoutTimestamp, - ?int $durationSeconds, + Recording $recording, Closure $report, string $message, - float &$lastFlushAt, - bool $pending, ): array { $event = $this->stream->parseEvent($payload); if ($event === null) { - return [$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending]; + return [$accumulated, $lastPercent]; } - $wasEmpty = $accumulated === ''; $accumulated = $this->stream->applyEvent($accumulated, $event); + $whisper = $event['whisper'] ?? []; - if ($accumulated === '') { - return [$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending]; + if ($accumulated === '' && $whisper === []) { + return [$accumulated, $lastPercent]; } - $percent = $this->percentForEvent($event, $durationSeconds, $lastPercent, $eventsWithoutTimestamp); - $lastPercent = max($lastPercent, $percent); + $lastPercent = $this->percentForEvent($event, $recording, $lastPercent); - $now = microtime(true); - $shouldFlush = $wasEmpty || $event['done'] || ($now - $lastFlushAt) >= 1.0; + $report( + $message, + $lastPercent, + $accumulated === '' ? null : $accumulated, + $whisper === [] ? null : $whisper, + ); - if ($shouldFlush) { - $report($message, $lastPercent, $accumulated); - $lastFlushAt = $now; - - return [$accumulated, $lastPercent, $eventsWithoutTimestamp, false]; - } - - return [$accumulated, $lastPercent, $eventsWithoutTimestamp, true]; + return [$accumulated, $lastPercent]; } /** - * @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool} $event + * @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper?: array} $event */ - private function percentForEvent(array $event, ?int $durationSeconds, int $lastPercent, int &$eventsWithoutTimestamp): int + private function percentForEvent(array $event, Recording $recording, int $lastPercent): int { - if ($event['end'] !== null && $durationSeconds !== null && $durationSeconds > 0) { - return (int) min(99, max(50, round(100 * $event['end'] / $durationSeconds))); + $duration = $recording->duration_seconds; + $end = $event['end'] ?? null; + + if ($end === null || $duration === null || $duration <= 0) { + return $lastPercent; } - $eventsWithoutTimestamp++; - - return min(84, max($lastPercent, 50 + $eventsWithoutTimestamp)); + return (int) min(99, max($lastPercent, round(100 * $end / $duration))); } private function localWhisperRequest(string $filename, string $path): PendingRequest @@ -231,14 +226,29 @@ class TranscriptionService ->attach('file', fopen($path, 'r'), $filename); } - private function extractTranscriptText(Response $response): string + /** + * @param Closure(string, int, ?string, ?array): void $report + */ + private function extractTranscriptText(Response $response, ?Closure $report = null, string $message = ''): string { - $text = $response->json('text'); + $json = $response->json(); + + if (! is_array($json)) { + throw new RuntimeException('Local transcription returned an empty transcript.'); + } + + $text = $json['text'] ?? null; if (! is_string($text) || $text === '') { throw new RuntimeException('Local transcription returned an empty transcript.'); } + $whisper = $this->stream->extractWhisperMeta($json); + + if ($report !== null && $whisper !== []) { + $report($message, 99, $text, $whisper); + } + return $text; } } diff --git a/app/Services/WhisperTranscriptionStream.php b/app/Services/WhisperTranscriptionStream.php index 4d2f0f0..d527645 100644 --- a/app/Services/WhisperTranscriptionStream.php +++ b/app/Services/WhisperTranscriptionStream.php @@ -49,9 +49,9 @@ class WhisperTranscriptionStream } /** - * Parse one SSE JSON payload into append/replace/end/done fields. + * Parse one SSE JSON payload into append/replace/end/done/whisper fields. * - * @return array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool}|null + * @return array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper: array}|null */ public function parseEvent(string $json): ?array { @@ -62,20 +62,23 @@ class WhisperTranscriptionStream } $type = $data['type'] ?? null; + $whisper = $this->extractWhisperMeta($data); + $end = $this->latestAudioEnd($data, $whisper); if ($type === 'transcript.text.delta') { $delta = $data['delta'] ?? ''; - if (! is_string($delta) || $delta === '') { + if ((! is_string($delta) || $delta === '') && $whisper === []) { return null; } return [ - 'append' => $delta, + 'append' => is_string($delta) && $delta !== '' ? $delta : null, 'replace' => null, - 'end' => $this->nullableFloat($data['end'] ?? null), + 'end' => $end, 'done' => false, 'legacy' => false, + 'whisper' => $whisper, ]; } @@ -85,19 +88,54 @@ class WhisperTranscriptionStream return [ 'append' => null, 'replace' => is_string($text) ? $text : '', - 'end' => $this->nullableFloat($data['end'] ?? null), + 'end' => $end, 'done' => true, 'legacy' => false, + 'whisper' => $whisper, ]; } - if ($type === null && isset($data['text']) && is_string($data['text']) && $data['text'] !== '') { + if (isset($data['segments']) && is_array($data['segments'])) { + $text = $data['text'] ?? ''; + + if ((! is_string($text) || $text === '') && $whisper === []) { + return null; + } + + return [ + 'append' => null, + 'replace' => is_string($text) && $text !== '' ? $text : null, + 'end' => $end, + 'done' => true, + 'legacy' => false, + 'whisper' => $whisper, + ]; + } + + if ( + ($type === null || $type === 'segment') + && isset($data['text']) + && is_string($data['text']) + && $data['text'] !== '' + ) { return [ 'append' => $data['text'], 'replace' => null, - 'end' => $this->nullableFloat($data['end'] ?? null), + 'end' => $end, 'done' => false, 'legacy' => true, + 'whisper' => $whisper, + ]; + } + + if ($whisper !== []) { + return [ + 'append' => null, + 'replace' => null, + 'end' => $end, + 'done' => false, + 'legacy' => false, + 'whisper' => $whisper, ]; } @@ -107,7 +145,7 @@ class WhisperTranscriptionStream /** * Apply a parsed event to the accumulated transcript. * - * @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool} $event + * @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper?: array} $event */ public function applyEvent(string $accumulated, array $event): string { @@ -134,6 +172,108 @@ class WhisperTranscriptionStream return $accumulated.$chunk; } + /** + * @param array $data + * @return array + */ + public function extractWhisperMeta(array $data): array + { + $meta = []; + + if (isset($data['language']) && is_string($data['language']) && $data['language'] !== '') { + $meta['language'] = $data['language']; + } + + if (is_numeric($data['duration'] ?? null)) { + $meta['duration'] = (float) $data['duration']; + } + + if (isset($data['logprobs']) && is_array($data['logprobs'])) { + $logprobs = $this->normalizeLogprobs($data['logprobs']); + + if ($logprobs !== []) { + $meta['logprobs'] = $logprobs; + } + } + + $segment = $this->normalizeSegment($data); + + if ($segment !== null && ($data['type'] ?? null) !== 'transcript.text.delta') { + $meta['segment'] = $segment; + } + + if (isset($data['segments']) && is_array($data['segments'])) { + $segments = []; + + foreach ($data['segments'] as $row) { + if (! is_array($row)) { + continue; + } + + $normalized = $this->normalizeSegment($row); + + if ($normalized !== null) { + $segments[] = $normalized; + } + } + + if ($segments !== []) { + $meta['segments'] = $segments; + } + } + + if (isset($data['words']) && is_array($data['words']) && ! isset($meta['segment']) && ! isset($meta['segments'])) { + $words = $this->normalizeWords($data['words']); + + if ($words !== []) { + $meta['words'] = $words; + } + } + + return $meta; + } + + /** + * @param array $data + * @return array|null + */ + public function normalizeSegment(array $data): ?array + { + $hasDetail = isset($data['start']) + || isset($data['end']) + || isset($data['words']) + || isset($data['avg_logprob']) + || isset($data['tokens']) + || isset($data['no_speech_prob']) + || array_key_exists('id', $data); + + if (! $hasDetail) { + return null; + } + + $text = $data['text'] ?? null; + + if (! is_string($text) || $text === '') { + return null; + } + + $segment = [ + 'id' => is_numeric($data['id'] ?? null) ? (int) $data['id'] : null, + 'seek' => is_numeric($data['seek'] ?? null) ? (int) $data['seek'] : null, + 'start' => $this->nullableFloat($data['start'] ?? null), + 'end' => $this->nullableFloat($data['end'] ?? null), + 'text' => $text, + 'tokens' => $this->normalizeTokens($data['tokens'] ?? null), + 'temperature' => $this->nullableFloat($data['temperature'] ?? null), + 'avg_logprob' => $this->nullableFloat($data['avg_logprob'] ?? null), + 'compression_ratio' => $this->nullableFloat($data['compression_ratio'] ?? null), + 'no_speech_prob' => $this->nullableFloat($data['no_speech_prob'] ?? null), + 'words' => $this->normalizeWords($data['words'] ?? null), + ]; + + return array_filter($segment, fn (mixed $value): bool => $value !== null && $value !== []); + } + /** * @return list */ @@ -150,6 +290,60 @@ class WhisperTranscriptionStream return $payloads; } + /** + * Latest media timestamp in this event (segment/word `end`). Never file `duration`. + * + * @param array $data + * @param array $whisper + */ + public function latestAudioEnd(array $data, array $whisper): ?float + { + $ends = []; + + $direct = $this->nullableFloat($data['end'] ?? null); + + if ($direct !== null) { + $ends[] = $direct; + } + + $this->collectAudioEnds($ends, $whisper); + + if (isset($data['words']) && is_array($data['words'])) { + $this->collectAudioEnds($ends, ['words' => $data['words']]); + } + + return $ends === [] ? null : max($ends); + } + + /** + * @param list $ends + * @param array $node + */ + private function collectAudioEnds(array &$ends, array $node): void + { + $end = $this->nullableFloat($node['end'] ?? null); + + if ($end !== null) { + $ends[] = $end; + } + + foreach (['words', 'segments'] as $key) { + if (! isset($node[$key]) || ! is_array($node[$key])) { + continue; + } + + foreach ($node[$key] as $child) { + if (is_array($child)) { + $this->collectAudioEnds($ends, $child); + } + } + } + + if (isset($node['segment']) && is_array($node['segment'])) { + $this->collectAudioEnds($ends, $node['segment']); + } + } + private function nullableFloat(mixed $value): ?float { if (! is_numeric($value)) { @@ -158,4 +352,86 @@ class WhisperTranscriptionStream return (float) $value; } + + /** + * @return list|null + */ + private function normalizeTokens(mixed $tokens): ?array + { + if (! is_array($tokens) || $tokens === []) { + return null; + } + + $normalized = []; + + foreach ($tokens as $token) { + if (is_numeric($token)) { + $normalized[] = (int) $token; + } + } + + return $normalized === [] ? null : $normalized; + } + + /** + * @return list|null + */ + private function normalizeWords(mixed $words): ?array + { + if (! is_array($words) || $words === []) { + return null; + } + + $normalized = []; + + foreach ($words as $word) { + if (! is_array($word)) { + continue; + } + + $text = $word['word'] ?? $word['text'] ?? null; + + if (! is_string($text) || $text === '') { + continue; + } + + $normalized[] = array_filter([ + 'word' => $text, + 'start' => $this->nullableFloat($word['start'] ?? null), + 'end' => $this->nullableFloat($word['end'] ?? null), + 'probability' => $this->nullableFloat($word['probability'] ?? $word['prob'] ?? null), + ], fn (mixed $value): bool => $value !== null); + } + + return $normalized === [] ? null : $normalized; + } + + /** + * @return list + */ + private function normalizeLogprobs(array $logprobs): array + { + $normalized = []; + + foreach ($logprobs as $row) { + if (! is_array($row)) { + continue; + } + + $token = $row['token'] ?? $row['bytes'] ?? null; + $token = is_string($token) ? $token : null; + $logprob = $this->nullableFloat($row['logprob'] ?? $row['avg_logprob'] ?? null); + + if ($token === null && $logprob === null) { + continue; + } + + $normalized[] = array_filter([ + 'token' => $token, + 'logprob' => $logprob, + ], fn (mixed $value): bool => $value !== null); + } + + return $normalized; + } } diff --git a/database/migrations/2026_08_13_124740_add_transcription_verbose_to_recordings_table.php b/database/migrations/2026_08_13_124740_add_transcription_verbose_to_recordings_table.php new file mode 100644 index 0000000..6941b1b --- /dev/null +++ b/database/migrations/2026_08_13_124740_add_transcription_verbose_to_recordings_table.php @@ -0,0 +1,30 @@ +json('transcription_verbose') + ->nullable() + ->after('transcript'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('recordings', function (Blueprint $table) { + $table->dropColumn('transcription_verbose'); + }); + } +}; diff --git a/resources/js/transcription.js b/resources/js/transcription.js index 77619bd..56eb859 100644 --- a/resources/js/transcription.js +++ b/resources/js/transcription.js @@ -55,39 +55,105 @@ export function formatTimestamp(value) { + ':' + pad(date.getMinutes()); } -function subscribeToRecording(recordingId, handler) { - if (!window.Echo) { +function unwrapBroadcast(event) { + if (! event || typeof event !== 'object') { + return {}; + } + + if ( + event.transcript_delta === undefined + && event.transcriptDelta === undefined + && event.data + && typeof event.data === 'object' + ) { + return event.data; + } + + return event; +} + +function subscribeToRecording({ recordingId, userId }, handler) { + if (! window.Echo) { return () => {}; } - const channelName = 'recording.' + recordingId; - const channel = window.Echo.private(channelName); + const channels = [window.Echo.private('recording.' + recordingId)]; - channel.listen('.RecordingTranscriptionUpdated', handler); + if (userId) { + channels.push(window.Echo.private('user.' + userId + '.recordings')); + } + + channels.forEach((channel) => { + channel.listen('.RecordingTranscriptionUpdated', handler); + }); return () => { - // Prefer stopListening over leave() so a remount does not drop other subscribers. - if (typeof channel.stopListening === 'function') { - channel.stopListening('.RecordingTranscriptionUpdated'); - } + channels.forEach((channel) => { + if (typeof channel.stopListening === 'function') { + channel.stopListening('.RecordingTranscriptionUpdated'); + } + }); }; } +function whisperSnapshot(snapshot) { + return { + language: snapshot?.language ?? null, + duration: snapshot?.duration ?? null, + segments: Array.isArray(snapshot?.segments) ? snapshot.segments : [], + logprobs: Array.isArray(snapshot?.logprobs) ? snapshot.logprobs : [], + }; +} + +function segmentKey(segment) { + return [segment?.id ?? '', segment?.start ?? '', segment?.end ?? '', segment?.text ?? ''].join('|'); +} + +export function formatClock(seconds) { + if (seconds == null || Number.isNaN(Number(seconds))) { + return ''; + } + + const value = Math.max(0, Number(seconds)); + const minutes = Math.floor(value / 60); + const rest = value - minutes * 60; + + return minutes + ':' + rest.toFixed(1).padStart(4, '0'); +} + +export function wordConfidenceClass(probability) { + if (probability == null) { + return 'bg-zinc-400/20 text-zinc-700 dark:text-zinc-200'; + } + + if (probability >= 0.85) { + return 'bg-teal-400/25 text-teal-800 dark:text-teal-200'; + } + + if (probability >= 0.6) { + return 'bg-amber-400/25 text-amber-800 dark:text-amber-200'; + } + + return 'bg-red-400/20 text-red-700 dark:text-red-300'; +} + /** - * Show-page Alpine component: Echo push + local elapsed tick + optional status hydrate. - * Livewire also listens on the user recordings channel and polls while active. + * Hydrate once at start and again when the run finishes; do not poll the full transcript. */ -export function transcriptionMonitor({ statusUrl, initial }) { +export function transcriptionMonitor({ statusUrl, initial, userId }) { return { statusUrl, + userId, status: { ...initial, badge_color: badgeColorFor(initial.status), }, pollError: null, tickTimer: null, - hydrateTimer: null, leaveChannel: null, + liveFromEcho: false, + lastDeltaStamp: null, + whisper: whisperSnapshot(initial?.whisper), get badgeColor() { return badgeColorFor(this.status.status); @@ -102,24 +168,27 @@ export function transcriptionMonitor({ statusUrl, initial }) { }, start() { - this.leaveChannel = subscribeToRecording(this.status.id, (event) => { - if (Number(event.id) !== Number(this.status.id)) { + this.leaveChannel = subscribeToRecording({ + recordingId: this.status.id, + userId: this.userId, + }, (event) => { + const payload = unwrapBroadcast(event); + + if (Number(payload.id) !== Number(this.status.id)) { return; } - this.applyPayload(event); + this.applyPayload(payload, { fromEcho: true }); }); if (this.status.is_active) { this.beginTick(); this.hydrateOnce(); - this.beginHydratePoll(); } }, destroy() { this.stopTick(); - this.stopHydratePoll(); if (this.leaveChannel) { this.leaveChannel(); @@ -127,21 +196,45 @@ export function transcriptionMonitor({ statusUrl, initial }) { } }, - applyPayload(payload) { + applyPayload(payload, { fromEcho = false } = {}) { const wasActive = this.status.is_active; + const transcript = this.mergeTranscript(payload, fromEcho); + const nextStatus = payload.status ?? this.status.status; + this.status = { ...this.status, ...payload, - badge_color: badgeColorFor(payload.status ?? this.status.status), + transcript, + has_transcript: Boolean(transcript), + percent: this.mergePercent(payload, nextStatus), + badge_color: badgeColorFor(nextStatus), }; + + if ( + payload.transcript_replace + || ( + fromEcho + && payload.status === 'processing' + && payload.percent === 0 + && ! payload.whisper_delta + && ! payload.transcript_delta + ) + ) { + this.whisper = whisperSnapshot(null); + } + + this.whisper = this.mergeWhisper(payload, fromEcho); this.pollError = null; if (this.status.is_active) { this.beginTick(); - this.beginHydratePoll(); } else { this.stopTick(); - this.stopHydratePoll(); + + if (wasActive) { + this.hydrateOnce(); + this.refreshLivewire(); + } } if (wasActive && ! this.status.is_active @@ -152,6 +245,139 @@ export function transcriptionMonitor({ statusUrl, initial }) { } }, + refreshLivewire() { + if (typeof this.$wire?.$refresh === 'function') { + this.$wire.$refresh(); + } + }, + + mergeTranscript(payload, fromEcho = false) { + const delta = payload.transcript_delta || payload.transcriptDelta; + const replace = payload.transcript_replace ?? payload.transcriptReplace ?? false; + + if (fromEcho && delta) { + const stamp = String(payload.percent ?? '') + ':' + delta; + + if (this.lastDeltaStamp === stamp) { + return this.status.transcript; + } + + this.lastDeltaStamp = stamp; + this.liveFromEcho = true; + } + + if (replace) { + return delta || ''; + } + + if (delta) { + this.liveFromEcho = this.liveFromEcho || fromEcho; + + return (this.status.transcript || '') + delta; + } + + if (this.liveFromEcho && (payload.status ?? this.status.status) === 'processing') { + return this.status.transcript; + } + + if (typeof payload.transcript === 'string') { + const current = this.status.transcript || ''; + + if (payload.transcript.length < current.length) { + return current; + } + + return payload.transcript; + } + + return this.status.transcript; + }, + + mergeWhisper(payload, fromEcho = false) { + const snapshot = payload.whisper; + const delta = payload.whisper_delta || payload.whisperDelta; + + if (snapshot && Array.isArray(snapshot.segments) && ! delta) { + if (fromEcho || (this.liveFromEcho && this.whisper.segments.length > snapshot.segments.length)) { + return this.whisper; + } + + return whisperSnapshot(snapshot); + } + + if (! delta) { + return this.whisper; + } + + if (fromEcho) { + this.liveFromEcho = true; + } + + const next = whisperSnapshot(this.whisper); + + if (delta.language) { + next.language = delta.language; + } + + if (delta.duration != null) { + next.duration = delta.duration; + } + + if (Array.isArray(delta.segments) && delta.segments.length) { + next.segments = delta.segments; + } else if (delta.segment) { + next.segments = this.appendSegment(next.segments, delta.segment); + } + + if (Array.isArray(delta.logprobs) && delta.logprobs.length) { + next.logprobs = next.logprobs.concat(delta.logprobs); + } + + if (Array.isArray(delta.words) && delta.words.length && next.segments.length) { + const last = { ...next.segments[next.segments.length - 1] }; + last.words = (last.words || []).concat(delta.words); + next.segments = next.segments.slice(0, -1).concat([last]); + } + + return next; + }, + + appendSegment(segments, segment) { + const key = segmentKey(segment); + + if (segments.some((row) => segmentKey(row) === key)) { + return segments; + } + + return segments.concat([segment]); + }, + + seekTo(seconds) { + const player = this.$refs.player; + + if (! player || seconds == null) { + return; + } + + player.currentTime = Number(seconds); + player.play().catch(() => {}); + }, + + mergePercent(payload, status) { + if (status !== 'processing') { + return payload.percent !== undefined ? payload.percent : this.status.percent; + } + + const incoming = payload.percent; + const current = Number(this.status.percent) || 0; + + if (incoming == null) { + return this.status.percent; + } + + return Math.max(current, Number(incoming) || 0); + }, + beginTick() { if (this.tickTimer) { return; @@ -167,21 +393,6 @@ export function transcriptionMonitor({ statusUrl, initial }) { } }, - beginHydratePoll() { - if (this.hydrateTimer || ! this.statusUrl) { - return; - } - - this.hydrateTimer = setInterval(() => this.hydrateOnce(), 2000); - }, - - stopHydratePoll() { - if (this.hydrateTimer) { - clearInterval(this.hydrateTimer); - this.hydrateTimer = null; - } - }, - tickElapsed() { if (! this.status.is_active || this.status.elapsed_seconds == null) { return; @@ -220,12 +431,14 @@ export function transcriptionMonitor({ statusUrl, initial }) { formatElapsed, formatDuration, formatTimestamp, + formatClock, + wordConfidenceClass, }; } /** * Index-page Alpine component: inline audio player only. - * Status/progress refresh via Livewire Echo + wire:poll. + * Status/progress refresh via wire:poll. Live transcript on the show page uses Echo. */ export function recordingsIndex() { return { diff --git a/resources/views/livewire/recordings/index.blade.php b/resources/views/livewire/recordings/index.blade.php index 90bdd2d..11946d6 100644 --- a/resources/views/livewire/recordings/index.blade.php +++ b/resources/views/livewire/recordings/index.blade.php @@ -187,7 +187,7 @@ :status="$recording->transcription_status" :label="$recording->transcriptionStatusLabel()" /> - @if ($recording->transcription_status === 'processing' && $recording->transcription_percent !== null) + @if ($recording->transcription_status === 'processing' && $recording->transcription_percent > 0) {{ $recording->transcription_percent }}% diff --git a/resources/views/livewire/recordings/show.blade.php b/resources/views/livewire/recordings/show.blade.php index bd6b54d..480a78c 100644 --- a/resources/views/livewire/recordings/show.blade.php +++ b/resources/views/livewire/recordings/show.blade.php @@ -1,12 +1,14 @@
isTranscribing()) + @if ($recording->transcription_status === 'pending') wire:poll.2s.visible @endif >
@@ -174,22 +176,30 @@ -
+
-

+ >
  • Engine:
    • +
  • + Detected language: + +
    • +
  • + Whisper duration: + +