diff --git a/app/Models/Recording.php b/app/Models/Recording.php index b3a51cb..2b686cf 100644 --- a/app/Models/Recording.php +++ b/app/Models/Recording.php @@ -595,6 +595,10 @@ class Recording extends Model $segments = is_array($verbose['segments'] ?? null) ? $verbose['segments'] : []; + if (isset($delta['segment']) && is_array($delta['segment'])) { + $segments = $this->appendVerboseSegment($segments, $delta['segment']); + } + if (isset($delta['segments']) && is_array($delta['segments'])) { $incoming = []; @@ -605,10 +609,14 @@ class Recording extends Model } if ($incoming !== []) { - $segments = $incoming; + if (isset($delta['segment'])) { + foreach ($incoming as $segment) { + $segments = $this->appendVerboseSegment($segments, $segment); + } + } else { + $segments = $incoming; + } } - } elseif (isset($delta['segment']) && is_array($delta['segment'])) { - $segments = $this->appendVerboseSegment($segments, $delta['segment']); } if (isset($delta['words']) && is_array($delta['words']) && $delta['words'] !== []) { diff --git a/app/Services/WhisperTranscriptionStream.php b/app/Services/WhisperTranscriptionStream.php index d527645..8e2de69 100644 --- a/app/Services/WhisperTranscriptionStream.php +++ b/app/Services/WhisperTranscriptionStream.php @@ -95,6 +95,17 @@ class WhisperTranscriptionStream ]; } + if ($this->isLiveSegmentEvent($type, $data)) { + return [ + 'append' => $data['text'], + 'replace' => null, + 'end' => $end, + 'done' => false, + 'legacy' => true, + 'whisper' => $whisper, + ]; + } + if (isset($data['segments']) && is_array($data['segments'])) { $text = $data['text'] ?? ''; @@ -112,22 +123,6 @@ class WhisperTranscriptionStream ]; } - if ( - ($type === null || $type === 'segment') - && isset($data['text']) - && is_string($data['text']) - && $data['text'] !== '' - ) { - return [ - 'append' => $data['text'], - 'replace' => null, - 'end' => $end, - 'done' => false, - 'legacy' => true, - 'whisper' => $whisper, - ]; - } - if ($whisper !== []) { return [ 'append' => null, @@ -217,7 +212,7 @@ class WhisperTranscriptionStream } } - if ($segments !== []) { + if ($segments !== [] && ! isset($meta['segment'])) { $meta['segments'] = $segments; } } @@ -274,6 +269,34 @@ class WhisperTranscriptionStream return array_filter($segment, fn (mixed $value): bool => $value !== null && $value !== []); } + /** + * One streamed Whisper chunk (not a finished verbose_json document). + * + * @param array $data + */ + private function isLiveSegmentEvent(mixed $type, array $data): bool + { + $text = $data['text'] ?? null; + + if (! is_string($text) || $text === '') { + return false; + } + + if ($type === 'segment') { + return true; + } + + if ($type !== null && $type !== '') { + return false; + } + + if (isset($data['start']) || isset($data['end'])) { + return true; + } + + return ! (isset($data['segments']) && is_array($data['segments'])); + } + /** * @return list */ diff --git a/resources/js/transcription.js b/resources/js/transcription.js index 56eb859..2e12631 100644 --- a/resources/js/transcription.js +++ b/resources/js/transcription.js @@ -63,6 +63,8 @@ function unwrapBroadcast(event) { if ( event.transcript_delta === undefined && event.transcriptDelta === undefined + && event.whisper_delta === undefined + && event.whisperDelta === undefined && event.data && typeof event.data === 'object' ) { @@ -100,13 +102,13 @@ function whisperSnapshot(snapshot) { return { language: snapshot?.language ?? null, duration: snapshot?.duration ?? null, - segments: Array.isArray(snapshot?.segments) ? snapshot.segments : [], - logprobs: Array.isArray(snapshot?.logprobs) ? snapshot.logprobs : [], + segments: Array.isArray(snapshot?.segments) ? snapshot.segments.slice() : [], + logprobs: Array.isArray(snapshot?.logprobs) ? snapshot.logprobs.slice() : [], }; } function segmentKey(segment) { - return [segment?.id ?? '', segment?.start ?? '', segment?.end ?? '', segment?.text ?? ''].join('|'); + return [segment?.start ?? '', segment?.end ?? '', segment?.text ?? ''].join('|'); } export function formatClock(seconds) { @@ -138,7 +140,8 @@ export function wordConfidenceClass(probability) { } /** - * Hydrate once at start and again when the run finishes; do not poll the full transcript. + * Live updates come from Echo. While a run is active we also hydrate from HTTP + * so segment cards keep appearing even if Livewire remorphs or an Echo frame is missed. */ export function transcriptionMonitor({ statusUrl, initial, userId }) { return { @@ -150,6 +153,7 @@ export function transcriptionMonitor({ statusUrl, initial, userId }) { }, pollError: null, tickTimer: null, + hydrateTimer: null, leaveChannel: null, liveFromEcho: false, lastDeltaStamp: null, @@ -183,12 +187,14 @@ export function transcriptionMonitor({ statusUrl, initial, userId }) { if (this.status.is_active) { this.beginTick(); + this.beginHydratePoll(); this.hydrateOnce(); } }, destroy() { this.stopTick(); + this.stopHydratePoll(); if (this.leaveChannel) { this.leaveChannel(); @@ -210,16 +216,7 @@ export function transcriptionMonitor({ statusUrl, initial, userId }) { badge_color: badgeColorFor(nextStatus), }; - if ( - payload.transcript_replace - || ( - fromEcho - && payload.status === 'processing' - && payload.percent === 0 - && ! payload.whisper_delta - && ! payload.transcript_delta - ) - ) { + if (this.shouldResetWhisper(payload, fromEcho)) { this.whisper = whisperSnapshot(null); } @@ -228,8 +225,10 @@ export function transcriptionMonitor({ statusUrl, initial, userId }) { if (this.status.is_active) { this.beginTick(); + this.beginHydratePoll(); } else { this.stopTick(); + this.stopHydratePoll(); if (wasActive) { this.hydrateOnce(); @@ -277,7 +276,14 @@ export function transcriptionMonitor({ statusUrl, initial, userId }) { } if (this.liveFromEcho && (payload.status ?? this.status.status) === 'processing') { - return this.status.transcript; + const incoming = typeof payload.transcript === 'string' ? payload.transcript : ''; + const current = this.status.transcript || ''; + + if (incoming.length > current.length) { + return incoming; + } + + return current; } if (typeof payload.transcript === 'string') { @@ -293,19 +299,74 @@ export function transcriptionMonitor({ statusUrl, initial, userId }) { return this.status.transcript; }, + shouldResetWhisper(payload, fromEcho = false) { + const delta = payload.whisper_delta || payload.whisperDelta; + const segment = delta?.segment; + + if ( + fromEcho + && payload.status === 'processing' + && payload.percent === 0 + && ! delta + && ! payload.transcript_delta + && ! payload.transcriptDelta + ) { + return true; + } + + if (segment && this.whisper.segments.length) { + const last = this.whisper.segments[this.whisper.segments.length - 1]; + + if (segment.start != null && last.start != null && Number(segment.start) + 0.05 < Number(last.start)) { + return true; + } + } + + return false; + }, + mergeWhisper(payload, fromEcho = false) { const snapshot = payload.whisper; const delta = payload.whisper_delta || payload.whisperDelta; if (snapshot && Array.isArray(snapshot.segments) && ! delta) { - if (fromEcho || (this.liveFromEcho && this.whisper.segments.length > snapshot.segments.length)) { + if (fromEcho) { return this.whisper; } - return whisperSnapshot(snapshot); + // HTTP hydrate is source of truth once it is ahead of (or equal to) local Echo state. + if (snapshot.segments.length >= this.whisper.segments.length) { + return whisperSnapshot(snapshot); + } + + const next = whisperSnapshot(this.whisper); + + if (! next.language && snapshot.language) { + next.language = snapshot.language; + } + + if (next.duration == null && snapshot.duration != null) { + next.duration = snapshot.duration; + } + + return next; } if (! delta) { + if (snapshot) { + const next = whisperSnapshot(this.whisper); + + if (snapshot.language) { + next.language = snapshot.language; + } + + if (snapshot.duration != null) { + next.duration = snapshot.duration; + } + + return next; + } + return this.whisper; } @@ -323,12 +384,20 @@ export function transcriptionMonitor({ statusUrl, initial, userId }) { next.duration = delta.duration; } - if (Array.isArray(delta.segments) && delta.segments.length) { - next.segments = delta.segments; - } else if (delta.segment) { + if (delta.segment) { next.segments = this.appendSegment(next.segments, delta.segment); } + if (Array.isArray(delta.segments) && delta.segments.length) { + if (! delta.segment) { + next.segments = delta.segments.slice(); + } else { + delta.segments.forEach((row) => { + next.segments = this.appendSegment(next.segments, row); + }); + } + } + if (Array.isArray(delta.logprobs) && delta.logprobs.length) { next.logprobs = next.logprobs.concat(delta.logprobs); } @@ -352,6 +421,10 @@ export function transcriptionMonitor({ statusUrl, initial, userId }) { return segments.concat([segment]); }, + segmentDomKey(segment, index) { + return segmentKey(segment) + '#' + index; + }, + seekTo(seconds) { const player = this.$refs.player; @@ -393,17 +466,49 @@ export function transcriptionMonitor({ statusUrl, initial, userId }) { } }, + beginHydratePoll() { + if (this.hydrateTimer || ! this.statusUrl) { + return; + } + + this.hydrateTimer = setInterval(() => { + if (! this.status.is_active) { + this.stopHydratePoll(); + + return; + } + + this.hydrateOnce(); + }, 1000); + }, + + stopHydratePoll() { + if (this.hydrateTimer) { + clearInterval(this.hydrateTimer); + this.hydrateTimer = null; + } + }, + tickElapsed() { if (! this.status.is_active || this.status.elapsed_seconds == null) { return; } const next = this.status.elapsed_seconds + 1; + const duration = Number(this.status.duration_seconds) || 0; + let percent = this.status.percent; + + if (this.status.status === 'processing' && duration > 0) { + const elapsedPercent = Math.min(99, Math.round((100 * next) / duration)); + + percent = Math.max(Number(percent) || 0, elapsedPercent); + } this.status = { ...this.status, elapsed_seconds: next, elapsed_human: formatElapsed(next), + percent, }; }, diff --git a/resources/views/livewire/recordings/show.blade.php b/resources/views/livewire/recordings/show.blade.php index 480a78c..28e8cd2 100644 --- a/resources/views/livewire/recordings/show.blade.php +++ b/resources/views/livewire/recordings/show.blade.php @@ -1,21 +1,20 @@ +{{-- + Full wire:ignore: Livewire must not remorph this tree while Echo/Alpine own the live UI. + Status + whisper segments hydrate over HTTP while active (no wire:poll here). +--}}
transcription_status === 'pending') - wire:poll.2s.visible - @endif + wire:ignore + wire:key="transcription-ui-{{ $recording->id }}-{{ $recording->transcription_status }}-{{ $recording->transcribed_at?->timestamp }}" + x-data="transcriptionMonitor(@js([ + 'statusUrl' => route('recordings.transcription-status', $recording), + 'userId' => (int) $recording->user_id, + 'initial' => $recording->transcriptionStatusPayload(), + ]))" + x-init=" + start(); + return () => destroy(); + " > -
← Recordings @@ -181,7 +180,6 @@
@@ -263,7 +260,7 @@
-