ben/AndyTranscribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stream Whisper verbose metadata live without overflowing Reverb.

Browse Source 
Persist accumulated verbose_json on the recording and broadcast only transcript/whisper deltas so the show page can render language, segments, word timestamps, and confidence while a run is in progress.
This commit is contained in:
ben
2026-08-13 15:55:51 +02:00
parent 4c73620458
commit 187b6b5d12
16 changed files with 1646 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Services/WhisperTranscriptionStream.php
+285 -9
View File
@@ -49,9 +49,9 @@ class WhisperTranscriptionStream
}
/**
* Parse one SSE JSON payload into append/replace/end/done fields.
* Parse one SSE JSON payload into append/replace/end/done/whisper fields.
*
* @return array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool}|null
* @return array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper: array<string, mixed>}|null
*/
public function parseEvent(string $json): ?array
{
@@ -62,20 +62,23 @@ class WhisperTranscriptionStream
}
$type = $data['type'] ?? null;
$whisper = $this->extractWhisperMeta($data);
$end = $this->latestAudioEnd($data, $whisper);
if ($type === 'transcript.text.delta') {
$delta = $data['delta'] ?? '';
if (! is_string($delta) || $delta === '') {
if ((! is_string($delta) || $delta === '') && $whisper === []) {
return null;
}
return [
'append' => $delta,
'append' => is_string($delta) && $delta !== '' ? $delta : null,
'replace' => null,
'end' => $this->nullableFloat($data['end'] ?? null),
'end' => $end,
'done' => false,
'legacy' => false,
'whisper' => $whisper,
];
}
@@ -85,19 +88,54 @@ class WhisperTranscriptionStream
return [
'append' => null,
'replace' => is_string($text) ? $text : '',
'end' => $this->nullableFloat($data['end'] ?? null),
'end' => $end,
'done' => true,
'legacy' => false,
'whisper' => $whisper,
];
}
if ($type === null && isset($data['text']) && is_string($data['text']) && $data['text'] !== '') {
if (isset($data['segments']) && is_array($data['segments'])) {
$text = $data['text'] ?? '';
if ((! is_string($text) || $text === '') && $whisper === []) {
return null;
}
return [
'append' => null,
'replace' => is_string($text) && $text !== '' ? $text : null,
'end' => $end,
'done' => true,
'legacy' => false,
'whisper' => $whisper,
];
}
if (
($type === null || $type === 'segment')
&& isset($data['text'])
&& is_string($data['text'])
&& $data['text'] !== ''
) {
return [
'append' => $data['text'],
'replace' => null,
'end' => $this->nullableFloat($data['end'] ?? null),
'end' => $end,
'done' => false,
'legacy' => true,
'whisper' => $whisper,
];
}
if ($whisper !== []) {
return [
'append' => null,
'replace' => null,
'end' => $end,
'done' => false,
'legacy' => false,
'whisper' => $whisper,
];
}
@@ -107,7 +145,7 @@ class WhisperTranscriptionStream
/**
* Apply a parsed event to the accumulated transcript.
*
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool} $event
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper?: array<string, mixed>} $event
*/
public function applyEvent(string $accumulated, array $event): string
{
@@ -134,6 +172,108 @@ class WhisperTranscriptionStream
return $accumulated.$chunk;
}
/**
* @param array<string, mixed> $data
* @return array<string, mixed>
*/
public function extractWhisperMeta(array $data): array
{
$meta = [];
if (isset($data['language']) && is_string($data['language']) && $data['language'] !== '') {
$meta['language'] = $data['language'];
}
if (is_numeric($data['duration'] ?? null)) {
$meta['duration'] = (float) $data['duration'];
}
if (isset($data['logprobs']) && is_array($data['logprobs'])) {
$logprobs = $this->normalizeLogprobs($data['logprobs']);
if ($logprobs !== []) {
$meta['logprobs'] = $logprobs;
}
}
$segment = $this->normalizeSegment($data);
if ($segment !== null && ($data['type'] ?? null) !== 'transcript.text.delta') {
$meta['segment'] = $segment;
}
if (isset($data['segments']) && is_array($data['segments'])) {
$segments = [];
foreach ($data['segments'] as $row) {
if (! is_array($row)) {
continue;
}
$normalized = $this->normalizeSegment($row);
if ($normalized !== null) {
$segments[] = $normalized;
}
}
if ($segments !== []) {
$meta['segments'] = $segments;
}
}
if (isset($data['words']) && is_array($data['words']) && ! isset($meta['segment']) && ! isset($meta['segments'])) {
$words = $this->normalizeWords($data['words']);
if ($words !== []) {
$meta['words'] = $words;
}
}
return $meta;
}
/**
* @param array<string, mixed> $data
* @return array<string, mixed>|null
*/
public function normalizeSegment(array $data): ?array
{
$hasDetail = isset($data['start'])
|| isset($data['end'])
|| isset($data['words'])
|| isset($data['avg_logprob'])
|| isset($data['tokens'])
|| isset($data['no_speech_prob'])
|| array_key_exists('id', $data);
if (! $hasDetail) {
return null;
}
$text = $data['text'] ?? null;
if (! is_string($text) || $text === '') {
return null;
}
$segment = [
'id' => is_numeric($data['id'] ?? null) ? (int) $data['id'] : null,
'seek' => is_numeric($data['seek'] ?? null) ? (int) $data['seek'] : null,
'start' => $this->nullableFloat($data['start'] ?? null),
'end' => $this->nullableFloat($data['end'] ?? null),
'text' => $text,
'tokens' => $this->normalizeTokens($data['tokens'] ?? null),
'temperature' => $this->nullableFloat($data['temperature'] ?? null),
'avg_logprob' => $this->nullableFloat($data['avg_logprob'] ?? null),
'compression_ratio' => $this->nullableFloat($data['compression_ratio'] ?? null),
'no_speech_prob' => $this->nullableFloat($data['no_speech_prob'] ?? null),
'words' => $this->normalizeWords($data['words'] ?? null),
];
return array_filter($segment, fn (mixed $value): bool => $value !== null && $value !== []);
}
/**
* @return list<string>
*/
@@ -150,6 +290,60 @@ class WhisperTranscriptionStream
return $payloads;
}
/**
* Latest media timestamp in this event (segment/word `end`). Never file `duration`.
*
* @param array<string, mixed> $data
* @param array<string, mixed> $whisper
*/
public function latestAudioEnd(array $data, array $whisper): ?float
{
$ends = [];
$direct = $this->nullableFloat($data['end'] ?? null);
if ($direct !== null) {
$ends[] = $direct;
}
$this->collectAudioEnds($ends, $whisper);
if (isset($data['words']) && is_array($data['words'])) {
$this->collectAudioEnds($ends, ['words' => $data['words']]);
}
return $ends === [] ? null : max($ends);
}
/**
* @param list<float> $ends
* @param array<string, mixed> $node
*/
private function collectAudioEnds(array &$ends, array $node): void
{
$end = $this->nullableFloat($node['end'] ?? null);
if ($end !== null) {
$ends[] = $end;
}
foreach (['words', 'segments'] as $key) {
if (! isset($node[$key]) || ! is_array($node[$key])) {
continue;
}
foreach ($node[$key] as $child) {
if (is_array($child)) {
$this->collectAudioEnds($ends, $child);
}
}
}
if (isset($node['segment']) && is_array($node['segment'])) {
$this->collectAudioEnds($ends, $node['segment']);
}
}
private function nullableFloat(mixed $value): ?float
{
if (! is_numeric($value)) {
@@ -158,4 +352,86 @@ class WhisperTranscriptionStream
return (float) $value;
}
/**
* @return list<int>|null
*/
private function normalizeTokens(mixed $tokens): ?array
{
if (! is_array($tokens) || $tokens === []) {
return null;
}
$normalized = [];
foreach ($tokens as $token) {
if (is_numeric($token)) {
$normalized[] = (int) $token;
}
}
return $normalized === [] ? null : $normalized;
}
/**
* @return list<array{word: string, start: ?float, end: ?float, probability: ?float}>|null
*/
private function normalizeWords(mixed $words): ?array
{
if (! is_array($words) || $words === []) {
return null;
}
$normalized = [];
foreach ($words as $word) {
if (! is_array($word)) {
continue;
}
$text = $word['word'] ?? $word['text'] ?? null;
if (! is_string($text) || $text === '') {
continue;
}
$normalized[] = array_filter([
'word' => $text,
'start' => $this->nullableFloat($word['start'] ?? null),
'end' => $this->nullableFloat($word['end'] ?? null),
'probability' => $this->nullableFloat($word['probability'] ?? $word['prob'] ?? null),
], fn (mixed $value): bool => $value !== null);
}
return $normalized === [] ? null : $normalized;
}
/**
* @return list<array{token: ?string, logprob: ?float}>
*/
private function normalizeLogprobs(array $logprobs): array
{
$normalized = [];
foreach ($logprobs as $row) {
if (! is_array($row)) {
continue;
}
$token = $row['token'] ?? $row['bytes'] ?? null;
$token = is_string($token) ? $token : null;
$logprob = $this->nullableFloat($row['logprob'] ?? $row['avg_logprob'] ?? null);
if ($token === null && $logprob === null) {
continue;
}
$normalized[] = array_filter([
'token' => $token,
'logprob' => $logprob,
], fn (mixed $value): bool => $value !== null);
}
return $normalized;
}
}