Stream Whisper verbose metadata live without overflowing Reverb.
Persist accumulated verbose_json on the recording and broadcast only transcript/whisper deltas so the show page can render language, segments, word timestamps, and confidence while a run is in progress.
This commit is contained in:
+251
-38
@@ -55,39 +55,105 @@ export function formatTimestamp(value) {
|
||||
+ ':' + pad(date.getMinutes());
|
||||
}
|
||||
|
||||
function subscribeToRecording(recordingId, handler) {
|
||||
if (!window.Echo) {
|
||||
function unwrapBroadcast(event) {
|
||||
if (! event || typeof event !== 'object') {
|
||||
return {};
|
||||
}
|
||||
|
||||
if (
|
||||
event.transcript_delta === undefined
|
||||
&& event.transcriptDelta === undefined
|
||||
&& event.data
|
||||
&& typeof event.data === 'object'
|
||||
) {
|
||||
return event.data;
|
||||
}
|
||||
|
||||
return event;
|
||||
}
|
||||
|
||||
function subscribeToRecording({ recordingId, userId }, handler) {
|
||||
if (! window.Echo) {
|
||||
return () => {};
|
||||
}
|
||||
|
||||
const channelName = 'recording.' + recordingId;
|
||||
const channel = window.Echo.private(channelName);
|
||||
const channels = [window.Echo.private('recording.' + recordingId)];
|
||||
|
||||
channel.listen('.RecordingTranscriptionUpdated', handler);
|
||||
if (userId) {
|
||||
channels.push(window.Echo.private('user.' + userId + '.recordings'));
|
||||
}
|
||||
|
||||
channels.forEach((channel) => {
|
||||
channel.listen('.RecordingTranscriptionUpdated', handler);
|
||||
});
|
||||
|
||||
return () => {
|
||||
// Prefer stopListening over leave() so a remount does not drop other subscribers.
|
||||
if (typeof channel.stopListening === 'function') {
|
||||
channel.stopListening('.RecordingTranscriptionUpdated');
|
||||
}
|
||||
channels.forEach((channel) => {
|
||||
if (typeof channel.stopListening === 'function') {
|
||||
channel.stopListening('.RecordingTranscriptionUpdated');
|
||||
}
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function whisperSnapshot(snapshot) {
|
||||
return {
|
||||
language: snapshot?.language ?? null,
|
||||
duration: snapshot?.duration ?? null,
|
||||
segments: Array.isArray(snapshot?.segments) ? snapshot.segments : [],
|
||||
logprobs: Array.isArray(snapshot?.logprobs) ? snapshot.logprobs : [],
|
||||
};
|
||||
}
|
||||
|
||||
function segmentKey(segment) {
|
||||
return [segment?.id ?? '', segment?.start ?? '', segment?.end ?? '', segment?.text ?? ''].join('|');
|
||||
}
|
||||
|
||||
export function formatClock(seconds) {
|
||||
if (seconds == null || Number.isNaN(Number(seconds))) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const value = Math.max(0, Number(seconds));
|
||||
const minutes = Math.floor(value / 60);
|
||||
const rest = value - minutes * 60;
|
||||
|
||||
return minutes + ':' + rest.toFixed(1).padStart(4, '0');
|
||||
}
|
||||
|
||||
export function wordConfidenceClass(probability) {
|
||||
if (probability == null) {
|
||||
return 'bg-zinc-400/20 text-zinc-700 dark:text-zinc-200';
|
||||
}
|
||||
|
||||
if (probability >= 0.85) {
|
||||
return 'bg-teal-400/25 text-teal-800 dark:text-teal-200';
|
||||
}
|
||||
|
||||
if (probability >= 0.6) {
|
||||
return 'bg-amber-400/25 text-amber-800 dark:text-amber-200';
|
||||
}
|
||||
|
||||
return 'bg-red-400/20 text-red-700 dark:text-red-300';
|
||||
}
|
||||
|
||||
/**
|
||||
* Show-page Alpine component: Echo push + local elapsed tick + optional status hydrate.
|
||||
* Livewire also listens on the user recordings channel and polls while active.
|
||||
* Hydrate once at start and again when the run finishes; do not poll the full transcript.
|
||||
*/
|
||||
export function transcriptionMonitor({ statusUrl, initial }) {
|
||||
export function transcriptionMonitor({ statusUrl, initial, userId }) {
|
||||
return {
|
||||
statusUrl,
|
||||
userId,
|
||||
status: {
|
||||
...initial,
|
||||
badge_color: badgeColorFor(initial.status),
|
||||
},
|
||||
pollError: null,
|
||||
tickTimer: null,
|
||||
hydrateTimer: null,
|
||||
leaveChannel: null,
|
||||
liveFromEcho: false,
|
||||
lastDeltaStamp: null,
|
||||
whisper: whisperSnapshot(initial?.whisper),
|
||||
|
||||
get badgeColor() {
|
||||
return badgeColorFor(this.status.status);
|
||||
@@ -102,24 +168,27 @@ export function transcriptionMonitor({ statusUrl, initial }) {
|
||||
},
|
||||
|
||||
start() {
|
||||
this.leaveChannel = subscribeToRecording(this.status.id, (event) => {
|
||||
if (Number(event.id) !== Number(this.status.id)) {
|
||||
this.leaveChannel = subscribeToRecording({
|
||||
recordingId: this.status.id,
|
||||
userId: this.userId,
|
||||
}, (event) => {
|
||||
const payload = unwrapBroadcast(event);
|
||||
|
||||
if (Number(payload.id) !== Number(this.status.id)) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.applyPayload(event);
|
||||
this.applyPayload(payload, { fromEcho: true });
|
||||
});
|
||||
|
||||
if (this.status.is_active) {
|
||||
this.beginTick();
|
||||
this.hydrateOnce();
|
||||
this.beginHydratePoll();
|
||||
}
|
||||
},
|
||||
|
||||
destroy() {
|
||||
this.stopTick();
|
||||
this.stopHydratePoll();
|
||||
|
||||
if (this.leaveChannel) {
|
||||
this.leaveChannel();
|
||||
@@ -127,21 +196,45 @@ export function transcriptionMonitor({ statusUrl, initial }) {
|
||||
}
|
||||
},
|
||||
|
||||
applyPayload(payload) {
|
||||
applyPayload(payload, { fromEcho = false } = {}) {
|
||||
const wasActive = this.status.is_active;
|
||||
const transcript = this.mergeTranscript(payload, fromEcho);
|
||||
const nextStatus = payload.status ?? this.status.status;
|
||||
|
||||
this.status = {
|
||||
...this.status,
|
||||
...payload,
|
||||
badge_color: badgeColorFor(payload.status ?? this.status.status),
|
||||
transcript,
|
||||
has_transcript: Boolean(transcript),
|
||||
percent: this.mergePercent(payload, nextStatus),
|
||||
badge_color: badgeColorFor(nextStatus),
|
||||
};
|
||||
|
||||
if (
|
||||
payload.transcript_replace
|
||||
|| (
|
||||
fromEcho
|
||||
&& payload.status === 'processing'
|
||||
&& payload.percent === 0
|
||||
&& ! payload.whisper_delta
|
||||
&& ! payload.transcript_delta
|
||||
)
|
||||
) {
|
||||
this.whisper = whisperSnapshot(null);
|
||||
}
|
||||
|
||||
this.whisper = this.mergeWhisper(payload, fromEcho);
|
||||
this.pollError = null;
|
||||
|
||||
if (this.status.is_active) {
|
||||
this.beginTick();
|
||||
this.beginHydratePoll();
|
||||
} else {
|
||||
this.stopTick();
|
||||
this.stopHydratePoll();
|
||||
|
||||
if (wasActive) {
|
||||
this.hydrateOnce();
|
||||
this.refreshLivewire();
|
||||
}
|
||||
}
|
||||
|
||||
if (wasActive && ! this.status.is_active
|
||||
@@ -152,6 +245,139 @@ export function transcriptionMonitor({ statusUrl, initial }) {
|
||||
}
|
||||
},
|
||||
|
||||
refreshLivewire() {
|
||||
if (typeof this.$wire?.$refresh === 'function') {
|
||||
this.$wire.$refresh();
|
||||
}
|
||||
},
|
||||
|
||||
mergeTranscript(payload, fromEcho = false) {
|
||||
const delta = payload.transcript_delta || payload.transcriptDelta;
|
||||
const replace = payload.transcript_replace ?? payload.transcriptReplace ?? false;
|
||||
|
||||
if (fromEcho && delta) {
|
||||
const stamp = String(payload.percent ?? '') + ':' + delta;
|
||||
|
||||
if (this.lastDeltaStamp === stamp) {
|
||||
return this.status.transcript;
|
||||
}
|
||||
|
||||
this.lastDeltaStamp = stamp;
|
||||
this.liveFromEcho = true;
|
||||
}
|
||||
|
||||
if (replace) {
|
||||
return delta || '';
|
||||
}
|
||||
|
||||
if (delta) {
|
||||
this.liveFromEcho = this.liveFromEcho || fromEcho;
|
||||
|
||||
return (this.status.transcript || '') + delta;
|
||||
}
|
||||
|
||||
if (this.liveFromEcho && (payload.status ?? this.status.status) === 'processing') {
|
||||
return this.status.transcript;
|
||||
}
|
||||
|
||||
if (typeof payload.transcript === 'string') {
|
||||
const current = this.status.transcript || '';
|
||||
|
||||
if (payload.transcript.length < current.length) {
|
||||
return current;
|
||||
}
|
||||
|
||||
return payload.transcript;
|
||||
}
|
||||
|
||||
return this.status.transcript;
|
||||
},
|
||||
|
||||
mergeWhisper(payload, fromEcho = false) {
|
||||
const snapshot = payload.whisper;
|
||||
const delta = payload.whisper_delta || payload.whisperDelta;
|
||||
|
||||
if (snapshot && Array.isArray(snapshot.segments) && ! delta) {
|
||||
if (fromEcho || (this.liveFromEcho && this.whisper.segments.length > snapshot.segments.length)) {
|
||||
return this.whisper;
|
||||
}
|
||||
|
||||
return whisperSnapshot(snapshot);
|
||||
}
|
||||
|
||||
if (! delta) {
|
||||
return this.whisper;
|
||||
}
|
||||
|
||||
if (fromEcho) {
|
||||
this.liveFromEcho = true;
|
||||
}
|
||||
|
||||
const next = whisperSnapshot(this.whisper);
|
||||
|
||||
if (delta.language) {
|
||||
next.language = delta.language;
|
||||
}
|
||||
|
||||
if (delta.duration != null) {
|
||||
next.duration = delta.duration;
|
||||
}
|
||||
|
||||
if (Array.isArray(delta.segments) && delta.segments.length) {
|
||||
next.segments = delta.segments;
|
||||
} else if (delta.segment) {
|
||||
next.segments = this.appendSegment(next.segments, delta.segment);
|
||||
}
|
||||
|
||||
if (Array.isArray(delta.logprobs) && delta.logprobs.length) {
|
||||
next.logprobs = next.logprobs.concat(delta.logprobs);
|
||||
}
|
||||
|
||||
if (Array.isArray(delta.words) && delta.words.length && next.segments.length) {
|
||||
const last = { ...next.segments[next.segments.length - 1] };
|
||||
last.words = (last.words || []).concat(delta.words);
|
||||
next.segments = next.segments.slice(0, -1).concat([last]);
|
||||
}
|
||||
|
||||
return next;
|
||||
},
|
||||
|
||||
appendSegment(segments, segment) {
|
||||
const key = segmentKey(segment);
|
||||
|
||||
if (segments.some((row) => segmentKey(row) === key)) {
|
||||
return segments;
|
||||
}
|
||||
|
||||
return segments.concat([segment]);
|
||||
},
|
||||
|
||||
seekTo(seconds) {
|
||||
const player = this.$refs.player;
|
||||
|
||||
if (! player || seconds == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
player.currentTime = Number(seconds);
|
||||
player.play().catch(() => {});
|
||||
},
|
||||
|
||||
mergePercent(payload, status) {
|
||||
if (status !== 'processing') {
|
||||
return payload.percent !== undefined ? payload.percent : this.status.percent;
|
||||
}
|
||||
|
||||
const incoming = payload.percent;
|
||||
const current = Number(this.status.percent) || 0;
|
||||
|
||||
if (incoming == null) {
|
||||
return this.status.percent;
|
||||
}
|
||||
|
||||
return Math.max(current, Number(incoming) || 0);
|
||||
},
|
||||
|
||||
beginTick() {
|
||||
if (this.tickTimer) {
|
||||
return;
|
||||
@@ -167,21 +393,6 @@ export function transcriptionMonitor({ statusUrl, initial }) {
|
||||
}
|
||||
},
|
||||
|
||||
beginHydratePoll() {
|
||||
if (this.hydrateTimer || ! this.statusUrl) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.hydrateTimer = setInterval(() => this.hydrateOnce(), 2000);
|
||||
},
|
||||
|
||||
stopHydratePoll() {
|
||||
if (this.hydrateTimer) {
|
||||
clearInterval(this.hydrateTimer);
|
||||
this.hydrateTimer = null;
|
||||
}
|
||||
},
|
||||
|
||||
tickElapsed() {
|
||||
if (! this.status.is_active || this.status.elapsed_seconds == null) {
|
||||
return;
|
||||
@@ -220,12 +431,14 @@ export function transcriptionMonitor({ statusUrl, initial }) {
|
||||
formatElapsed,
|
||||
formatDuration,
|
||||
formatTimestamp,
|
||||
formatClock,
|
||||
wordConfidenceClass,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Index-page Alpine component: inline audio player only.
|
||||
* Status/progress refresh via Livewire Echo + wire:poll.
|
||||
* Status/progress refresh via wire:poll. Live transcript on the show page uses Echo.
|
||||
*/
|
||||
export function recordingsIndex() {
|
||||
return {
|
||||
|
||||
@@ -187,7 +187,7 @@
|
||||
:status="$recording->transcription_status"
|
||||
:label="$recording->transcriptionStatusLabel()"
|
||||
/>
|
||||
@if ($recording->transcription_status === 'processing' && $recording->transcription_percent !== null)
|
||||
@if ($recording->transcription_status === 'processing' && $recording->transcription_percent > 0)
|
||||
<span class="tabular-nums text-xs text-zinc-500 dark:text-zinc-400">
|
||||
{{ $recording->transcription_percent }}%
|
||||
</span>
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
<div
|
||||
@if ($recording->isTranscribing())
|
||||
@if ($recording->transcription_status === 'pending')
|
||||
wire:poll.2s.visible
|
||||
@endif
|
||||
>
|
||||
<div
|
||||
wire:key="transcription-ui-{{ $recording->id }}-{{ $recording->transcription_status }}-{{ $recording->transcription_percent }}-{{ md5((string) $recording->transcription_progress) }}-{{ $recording->transcribed_at?->timestamp }}"
|
||||
wire:ignore.self
|
||||
wire:key="transcription-ui-{{ $recording->id }}-{{ $recording->transcription_status }}-{{ $recording->transcribed_at?->timestamp }}"
|
||||
x-data="transcriptionMonitor(@js([
|
||||
'statusUrl' => route('recordings.transcription-status', $recording),
|
||||
'userId' => (int) $recording->user_id,
|
||||
'initial' => $recording->transcriptionStatusPayload(),
|
||||
]))"
|
||||
x-init="
|
||||
@@ -164,7 +166,7 @@
|
||||
<flux:callout.text>
|
||||
<span
|
||||
class="tabular-nums"
|
||||
x-show="status.status === 'processing' && status.percent != null"
|
||||
x-show="status.status === 'processing' && Number(status.percent) > 0"
|
||||
x-cloak
|
||||
>
|
||||
<span x-text="status.percent + '%'"></span>
|
||||
@@ -174,22 +176,30 @@
|
||||
</flux:callout.text>
|
||||
</flux:callout>
|
||||
|
||||
<div class="mt-3" x-show="status.status === 'processing'" x-cloak>
|
||||
<div class="mt-3" x-show="status.status === 'processing' && Number(status.percent) > 0" x-cloak>
|
||||
<flux:progress color="amber" x-bind:value="status.percent || 0" />
|
||||
</div>
|
||||
|
||||
<p
|
||||
<div
|
||||
wire:ignore
|
||||
class="mt-4 whitespace-pre-wrap text-sm leading-relaxed text-zinc-800 dark:text-zinc-100"
|
||||
x-show="status.transcript"
|
||||
x-cloak
|
||||
x-show="status.transcript && !whisper.segments.length"
|
||||
x-text="status.transcript"
|
||||
></p>
|
||||
></div>
|
||||
|
||||
<ul class="mt-3 space-y-1 text-xs text-zinc-600 dark:text-zinc-400">
|
||||
<li>
|
||||
Engine:
|
||||
<span class="font-medium" x-text="status.driver_label || '—'"></span>
|
||||
</li>
|
||||
<li x-show="whisper.language">
|
||||
Detected language:
|
||||
<span class="font-medium uppercase" x-text="whisper.language"></span>
|
||||
</li>
|
||||
<li x-show="whisper.duration">
|
||||
Whisper duration:
|
||||
<span class="font-medium" x-text="formatClock(whisper.duration)"></span>
|
||||
</li>
|
||||
<template x-if="status.duration_seconds">
|
||||
<li>
|
||||
Audio length:
|
||||
@@ -219,7 +229,11 @@
|
||||
</div>
|
||||
|
||||
<div x-show="!status.is_active && status.has_transcript" x-cloak>
|
||||
<p class="mt-4 whitespace-pre-wrap text-sm leading-relaxed text-zinc-800 dark:text-zinc-100" x-text="status.transcript"></p>
|
||||
<p
|
||||
class="mt-4 whitespace-pre-wrap text-sm leading-relaxed text-zinc-800 dark:text-zinc-100"
|
||||
x-show="!whisper.segments.length"
|
||||
x-text="status.transcript"
|
||||
></p>
|
||||
<flux:text
|
||||
class="mt-4 text-xs"
|
||||
x-show="status.transcribed_at"
|
||||
@@ -230,6 +244,83 @@
|
||||
></flux:text>
|
||||
</div>
|
||||
|
||||
<div
|
||||
wire:ignore
|
||||
class="mt-4 space-y-4"
|
||||
x-show="whisper.language || whisper.segments.length || whisper.logprobs.length"
|
||||
>
|
||||
<div
|
||||
class="flex flex-wrap gap-2 text-xs text-zinc-600 dark:text-zinc-400"
|
||||
x-show="!status.is_active && (whisper.language || whisper.duration)"
|
||||
>
|
||||
<span x-show="whisper.language">
|
||||
Language
|
||||
<span class="font-medium uppercase text-zinc-800 dark:text-zinc-100" x-text="whisper.language"></span>
|
||||
</span>
|
||||
<span x-show="whisper.duration">
|
||||
· Duration
|
||||
<span class="font-medium text-zinc-800 dark:text-zinc-100" x-text="formatClock(whisper.duration)"></span>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<template x-for="(segment, index) in whisper.segments" :key="segment.id ?? (segment.start + '-' + index)">
|
||||
<div class="rounded-lg border border-zinc-200 p-3 dark:border-zinc-700">
|
||||
<div class="flex flex-wrap items-baseline justify-between gap-2 text-xs text-zinc-500 dark:text-zinc-400">
|
||||
<span class="tabular-nums">
|
||||
<span x-text="formatClock(segment.start)"></span>
|
||||
–
|
||||
<span x-text="formatClock(segment.end)"></span>
|
||||
</span>
|
||||
<span x-show="segment.avg_logprob != null">
|
||||
avg logprob
|
||||
<span class="font-medium text-zinc-700 dark:text-zinc-200" x-text="Number(segment.avg_logprob).toFixed(3)"></span>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<p class="mt-2 text-sm leading-relaxed text-zinc-800 dark:text-zinc-100" x-text="segment.text"></p>
|
||||
|
||||
<div class="mt-2 flex flex-wrap gap-1" x-show="segment.words && segment.words.length">
|
||||
<template x-for="(word, wordIndex) in (segment.words || [])" :key="wordIndex">
|
||||
<button
|
||||
type="button"
|
||||
class="rounded px-1.5 py-0.5 text-xs tabular-nums"
|
||||
:class="wordConfidenceClass(word.probability)"
|
||||
:title="(word.start != null ? formatClock(word.start) : '') + (word.probability != null ? (' · ' + Math.round(word.probability * 100) + '%') : '')"
|
||||
@click="seekTo(word.start)"
|
||||
x-text="word.word"
|
||||
></button>
|
||||
</template>
|
||||
</div>
|
||||
|
||||
<details class="mt-2 text-xs text-zinc-500 dark:text-zinc-400" x-show="segment.tokens && segment.tokens.length">
|
||||
<summary class="cursor-pointer select-none">Tokens</summary>
|
||||
<p class="mt-1 break-all font-mono" x-text="(segment.tokens || []).join(' ')"></p>
|
||||
</details>
|
||||
|
||||
<p class="mt-1 text-xs text-zinc-500 dark:text-zinc-400" x-show="segment.no_speech_prob != null">
|
||||
no-speech
|
||||
<span class="font-medium" x-text="Number(segment.no_speech_prob).toFixed(3)"></span>
|
||||
<span x-show="segment.compression_ratio != null">
|
||||
· compression
|
||||
<span class="font-medium" x-text="Number(segment.compression_ratio).toFixed(2)"></span>
|
||||
</span>
|
||||
</p>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<details class="text-xs text-zinc-500 dark:text-zinc-400" x-show="whisper.logprobs.length">
|
||||
<summary class="cursor-pointer select-none">Token logprobs</summary>
|
||||
<ul class="mt-2 space-y-1 font-mono">
|
||||
<template x-for="(row, index) in whisper.logprobs" :key="index">
|
||||
<li>
|
||||
<span x-text="row.token || '—'"></span>
|
||||
<span x-show="row.logprob != null" x-text="' ' + Number(row.logprob).toFixed(3)"></span>
|
||||
</li>
|
||||
</template>
|
||||
</ul>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
<flux:text
|
||||
x-show="!status.is_active && status.status !== 'failed' && status.status !== 'cancelled' && !status.has_transcript"
|
||||
x-cloak
|
||||
|
||||
Reference in New Issue
Block a user