ben/AndyTranscribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stream Whisper verbose metadata live without overflowing Reverb.

Browse Source 
Persist accumulated verbose_json on the recording and broadcast only transcript/whisper deltas so the show page can render language, segments, word timestamps, and confidence while a run is in progress.
This commit is contained in:
ben
2026-08-13 15:55:51 +02:00
parent 4c73620458
commit 187b6b5d12
16 changed files with 1646 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Events/RecordingTranscriptionUpdated.php
+93 -3
View File
@@ -14,9 +14,21 @@ class RecordingTranscriptionUpdated implements ShouldBroadcastNow
use Dispatchable, InteractsWithSockets, SerializesModels;
/**
* Create a new event instance.
* Reverb's default max payload is 10KB. Keep streamed text well under that.
*/
public function __construct(public Recording $recording) {}
public const MAX_DELTA_BYTES = 8_000;
/**
* Create a new event instance.
*
* @param array<string, mixed>|null $whisperDelta
*/
public function __construct(
public Recording $recording,
public ?string $transcriptDelta = null,
public bool $transcriptReplace = false,
public ?array $whisperDelta = null,
) {}
/**
* Get the channels the event should broadcast on.
@@ -41,11 +53,89 @@ class RecordingTranscriptionUpdated implements ShouldBroadcastNow
*/
public function broadcastWith(): array
{
return array_merge(
$payload = array_merge(
$this->recording->transcriptionStatusPayload(),
[
'word_count' => $this->recording->word_count,
],
);
unset($payload['transcript'], $payload['whisper']);
$delta = $this->transcriptDelta;
if ($delta !== null && $delta !== '' && strlen($delta) <= self::MAX_DELTA_BYTES) {
$payload['transcript_delta'] = $delta;
$payload['transcript_replace'] = $this->transcriptReplace;
}
$whisper = $this->compactWhisperDelta($this->whisperDelta);
if ($whisper !== null) {
$payload['whisper_delta'] = $whisper;
}
return $payload;
}
/**
* @param array<string, mixed>|null $whisper
* @return array<string, mixed>|null
*/
private function compactWhisperDelta(?array $whisper): ?array
{
if ($whisper === null || $whisper === []) {
return null;
}
// Accumulated segment lists belong in HTTP hydrate, not on Reverb.
unset($whisper['segments']);
if ($whisper === []) {
return null;
}
foreach (['tokens', 'logprobs', 'words'] as $drop) {
$encoded = json_encode($whisper);
if (! is_string($encoded) || strlen($encoded) <= self::MAX_DELTA_BYTES) {
return $whisper;
}
$whisper = $this->dropWhisperField($whisper, $drop);
}
$encoded = json_encode($whisper);
if (! is_string($encoded) || strlen($encoded) > self::MAX_DELTA_BYTES) {
return null;
}
return $whisper;
}
/**
* @param array<string, mixed> $whisper
* @return array<string, mixed>
*/
private function dropWhisperField(array $whisper, string $field): array
{
unset($whisper[$field]);
if (isset($whisper['segment']) && is_array($whisper['segment'])) {
unset($whisper['segment'][$field]);
}
if (isset($whisper['segments']) && is_array($whisper['segments'])) {
$whisper['segments'] = array_map(function (mixed $segment) use ($field): mixed {
if (is_array($segment)) {
unset($segment[$field]);
}
return $segment;
}, $whisper['segments']);
}
return $whisper;
}
}
app/Jobs/TranscribeRecording.php
+5 -11
View File
@@ -67,17 +67,16 @@ class TranscribeRecording implements ShouldQueue
'transcription_status' => 'processing',
'transcription_started_at' => $this->recording->transcription_started_at ?? now(),
'transcription_error' => null,
'transcription_verbose' => null,
])->save();
$this->runStartedAt ??= $this->recording->transcription_started_at?->toIso8601String();
$this->reportIfOwned('Preparing audio file…', 15);
try {
$text = $transcription->transcribe(
$this->recording,
function (string $message, int $percent, ?string $partialTranscript = null): void {
$this->reportIfOwned($message, $percent, $partialTranscript);
function (string $message, int $percent, ?string $partialTranscript = null, ?array $whisper = null): void {
$this->reportIfOwned($message, $percent, $partialTranscript, $whisper);
},
fn (): bool => Recording::query()->find($this->recording->id)
?->ownsTranscriptionRun($this->runStartedAt) ?? false,
@@ -135,11 +134,6 @@ class TranscribeRecording implements ShouldQueue
{
$this->recording->refresh();
if ($this->recording->ownsTranscriptionRun($this->runStartedAt)) {
$this->reportIfOwned('Saving transcript…', 90);
$this->recording->refresh();
}
if ($this->recording->ownsTranscriptionRun($this->runStartedAt)) {
$this->recording->markTranscriptionComplete($text);
@@ -172,12 +166,12 @@ class TranscribeRecording implements ShouldQueue
return false;
}
private function reportIfOwned(string $message, int $percent, ?string $partialTranscript = null): void
private function reportIfOwned(string $message, int $percent, ?string $partialTranscript = null, ?array $whisper = null): void
{
if (! $this->recording->ownsTranscriptionRun($this->runStartedAt)) {
return;
}
$this->recording->reportProgress($message, $percent, partialTranscript: $partialTranscript);
$this->recording->reportProgress($message, $percent, partialTranscript: $partialTranscript, whisperDelta: $whisper);
}
}
app/Livewire/Recordings/Index.php
-13
View File
@@ -10,7 +10,6 @@ use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\Gate;
use Livewire\Attributes\Layout;
use Livewire\Attributes\On;
use Livewire\Attributes\Title;
use Livewire\Attributes\Url;
use Livewire\Component;
@@ -22,8 +21,6 @@ class Index extends Component
{
use WithPagination;
public int $userId;
#[Url(as: 'q', history: true)]
public string $search = '';
@@ -45,7 +42,6 @@ class Index extends Component
public function mount(): void
{
$this->userId = (int) Auth::id();
$this->normalizeSort();
}
@@ -70,15 +66,6 @@ class Index extends Component
$this->resetPage();
}
/**
* Re-render when any of this user's recordings broadcast a status change.
*/
#[On('echo-private:user.{userId}.recordings,.RecordingTranscriptionUpdated')]
public function onTranscriptionUpdated(): void
{
//
}
public function queuePending(): void
{
$queued = 0;
app/Livewire/Recordings/Show.php
-23
View File
@@ -7,7 +7,6 @@ use Flux\Flux;
use Illuminate\Contracts\View\View;
use Illuminate\Support\Facades\Gate;
use Livewire\Attributes\Layout;
use Livewire\Attributes\On;
use Livewire\Component;
#[Layout('layouts.app')]
@@ -15,8 +14,6 @@ class Show extends Component
{
public Recording $recording;
public int $userId;
public function mount(Recording $recording): void
{
Gate::authorize('view', $recording);
@@ -25,22 +22,6 @@ class Show extends Component
$recording->refresh();
$this->recording = $recording;
$this->userId = (int) $recording->user_id;
}
/**
* Re-render when this recording broadcasts a status change (same channel as the index).
*
* @param array<string, mixed> $event
*/
#[On('echo-private:user.{userId}.recordings,.RecordingTranscriptionUpdated')]
public function onTranscriptionUpdated(array $event = []): void
{
if (isset($event['id']) && (int) $event['id'] !== (int) $this->recording->id) {
return;
}
$this->recording->refresh();
}
public function startTranscription(): void
@@ -83,10 +64,6 @@ class Show extends Component
public function render(): View
{
if ($this->recording->isTranscribing()) {
$this->recording->refresh();
}
return view('livewire.recordings.show')
->title($this->recording->title);
}
app/Models/Recording.php
+157 -8
View File
@@ -12,6 +12,7 @@ use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Throwable;
@@ -33,6 +34,7 @@ class Recording extends Model
'file_size_bytes',
'content_hash',
'transcript',
'transcription_verbose',
'transcription_status',
'transcription_progress',
'transcription_percent',
@@ -57,6 +59,7 @@ class Recording extends Model
'transcription_duration_seconds' => 'integer',
'file_size_bytes' => 'integer',
'transcription_percent' => 'integer',
'transcription_verbose' => 'array',
'user_id' => 'integer',
];
}
@@ -146,7 +149,7 @@ class Recording extends Model
]);
$recording = $this->fresh();
RecordingTranscriptionUpdated::dispatch($recording);
$this->broadcastTranscriptionUpdated();
TranscribeRecording::dispatch($recording);
}
@@ -424,7 +427,7 @@ class Recording extends Model
'transcription_error' => 'Stopped by user',
])->save();
RecordingTranscriptionUpdated::dispatch($this->fresh());
$this->broadcastTranscriptionUpdated();
}
/**
@@ -458,7 +461,7 @@ class Recording extends Model
'transcription_duration_seconds' => $durationSeconds,
])->save();
RecordingTranscriptionUpdated::dispatch($this->fresh());
$this->broadcastTranscriptionUpdated();
}
/**
@@ -477,7 +480,7 @@ class Recording extends Model
'transcription_error' => $message,
])->save();
RecordingTranscriptionUpdated::dispatch($this->fresh());
$this->broadcastTranscriptionUpdated();
}
/**
@@ -501,9 +504,13 @@ class Recording extends Model
/**
* Update the live progress fields shown in the UI.
*
* @param array<string, mixed>|null $whisperDelta
*/
public function reportProgress(string $message, int $percent, string $status = 'processing', ?string $partialTranscript = null): void
public function reportProgress(string $message, int $percent, string $status = 'processing', ?string $partialTranscript = null, ?array $whisperDelta = null): void
{
$diff = $this->transcriptBroadcastDiff($partialTranscript);
$attributes = [
'transcription_status' => $status,
'transcription_progress' => $message,
@@ -515,17 +522,158 @@ class Recording extends Model
$attributes['transcript'] = $partialTranscript;
}
if ($whisperDelta !== null) {
$attributes['transcription_verbose'] = $this->mergeWhisperVerbose($whisperDelta);
}
$this->forceFill($attributes)->save();
RecordingTranscriptionUpdated::dispatch($this->fresh());
$this->broadcastTranscriptionUpdated($diff['delta'], $diff['replace'], $whisperDelta);
}
/**
* Broadcast the current transcription status to connected browsers.
*
* @param array<string, mixed>|null $whisperDelta
*/
public function broadcastTranscriptionUpdated(): void
public function broadcastTranscriptionUpdated(?string $transcriptDelta = null, bool $transcriptReplace = false, ?array $whisperDelta = null): void
{
RecordingTranscriptionUpdated::dispatch($this->fresh() ?? $this);
try {
RecordingTranscriptionUpdated::dispatch(
$this->fresh() ?? $this,
$transcriptDelta,
$transcriptReplace,
$whisperDelta,
);
} catch (Throwable $e) {
Log::warning('Failed to broadcast transcription status', [
'recording_id' => $this->id,
'message' => $e->getMessage(),
]);
}
}
/**
* Incremental text to push over Reverb instead of the full transcript.
*
* @return array{delta: ?string, replace: bool}
*/
public function transcriptBroadcastDiff(?string $next): array
{
if ($next === null) {
return ['delta' => null, 'replace' => false];
}
$previous = (string) $this->transcript;
if ($previous === '' || ! str_starts_with($next, $previous)) {
return ['delta' => $next, 'replace' => true];
}
$delta = substr($next, strlen($previous));
return ['delta' => $delta === '' ? null : $delta, 'replace' => false];
}
/**
* Fold a streamed Whisper chunk into the stored verbose_json snapshot.
*
* @param array<string, mixed> $delta
* @return array<string, mixed>
*/
public function mergeWhisperVerbose(array $delta): array
{
$verbose = is_array($this->transcription_verbose) ? $this->transcription_verbose : [];
if (isset($delta['language']) && is_string($delta['language'])) {
$verbose['language'] = $delta['language'];
}
if (isset($delta['duration']) && is_numeric($delta['duration'])) {
$verbose['duration'] = (float) $delta['duration'];
}
$segments = is_array($verbose['segments'] ?? null) ? $verbose['segments'] : [];
if (isset($delta['segments']) && is_array($delta['segments'])) {
$incoming = [];
foreach ($delta['segments'] as $segment) {
if (is_array($segment)) {
$incoming[] = $segment;
}
}
if ($incoming !== []) {
$segments = $incoming;
}
} elseif (isset($delta['segment']) && is_array($delta['segment'])) {
$segments = $this->appendVerboseSegment($segments, $delta['segment']);
}
if (isset($delta['words']) && is_array($delta['words']) && $delta['words'] !== []) {
$last = $segments === [] ? null : count($segments) - 1;
if ($last === null) {
$segments[] = ['words' => $delta['words']];
} else {
$existing = is_array($segments[$last]['words'] ?? null) ? $segments[$last]['words'] : [];
$segments[$last]['words'] = array_merge($existing, $delta['words']);
}
}
$logprobs = is_array($verbose['logprobs'] ?? null) ? $verbose['logprobs'] : [];
if (isset($delta['logprobs']) && is_array($delta['logprobs'])) {
foreach ($delta['logprobs'] as $row) {
if (is_array($row)) {
$logprobs[] = $row;
}
}
}
if ($segments !== []) {
$verbose['segments'] = $segments;
}
if ($logprobs !== []) {
$verbose['logprobs'] = $logprobs;
}
return $verbose;
}
/**
* @param list<array<string, mixed>> $segments
* @param array<string, mixed> $segment
* @return list<array<string, mixed>>
*/
private function appendVerboseSegment(array $segments, array $segment): array
{
$key = $this->verboseSegmentKey($segment);
foreach ($segments as $existing) {
if ($this->verboseSegmentKey($existing) === $key) {
return $segments;
}
}
$segments[] = $segment;
return $segments;
}
/**
* @param array<string, mixed> $segment
*/
private function verboseSegmentKey(array $segment): string
{
return implode('|', [
$segment['id'] ?? '',
$segment['start'] ?? '',
$segment['end'] ?? '',
$segment['text'] ?? '',
]);
}
/**
@@ -596,6 +744,7 @@ class Recording extends Model
'is_active' => $this->isTranscribing(),
'has_transcript' => filled($this->transcript),
'transcript' => $this->transcript,
'whisper' => $this->transcription_verbose,
'transcribed_at' => $this->transcribed_at?->toIso8601String(),
];
}
app/Services/TranscriptionService.php
+72 -62
View File
@@ -7,6 +7,7 @@ use Closure;
use Illuminate\Http\Client\PendingRequest;
use Illuminate\Http\Client\Response;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Laravel\Ai\Transcription;
use RuntimeException;
@@ -17,16 +18,15 @@ class TranscriptionService
/**
* Transcribe a recording with the local faster-whisper server.
*
* @param (Closure(string, int, ?string): void)|null $onProgress
* @param (Closure(string, int, ?string, ?array): void)|null $onProgress
* @param (Closure(): bool)|null $shouldContinue
*/
public function transcribe(Recording $recording, ?Closure $onProgress = null, ?Closure $shouldContinue = null): string
{
$report = $onProgress ?? static fn (string $message, int $percent, ?string $partial = null) => null;
$report = $onProgress ?? static fn (string $message, int $percent, ?string $partial = null, ?array $whisper = null) => null;
$model = config('ai.local_whisper_model', 'Systran/faster-whisper-base');
$report('Connecting to local faster-whisper server…', 30);
$report("Transcribing locally with {$model} (audio stays on this machine)…", 50);
$report("Transcribing locally with {$model} (audio stays on this machine)…", 0);
if (Transcription::isFaked()) {
$transcript = Transcription::fromStorage($recording->file_path)
@@ -36,15 +36,13 @@ class TranscriptionService
$transcript = $this->transcribeViaLocalWhisper($recording, $report, $model, $shouldContinue);
}
$report('Received transcript from local Whisper…', 85);
return (string) $transcript;
}
/**
* Call local Whisper, streaming SSE when the server supports it.
*
* @param Closure(string, int, ?string): void $report
* @param Closure(string, int, ?string, ?array): void $report
* @param (Closure(): bool)|null $shouldContinue
*/
private function transcribeViaLocalWhisper(Recording $recording, Closure $report, string $model, ?Closure $shouldContinue): string
@@ -62,9 +60,10 @@ class TranscriptionService
->withOptions(['stream' => true])
->post('audio/transcriptions', [
'model' => $model,
'response_format' => 'json',
'response_format' => 'verbose_json',
'stream' => 'true',
'without_timestamps' => 'false',
'timestamp_granularities[]' => 'word',
]);
if (! $response->successful()) {
@@ -76,30 +75,33 @@ class TranscriptionService
$contentType = strtolower((string) $response->header('Content-Type'));
if (! str_contains($contentType, 'text/event-stream')) {
return $this->extractTranscriptText($response);
Log::warning('Whisper did not stream SSE; falling back to a single JSON body', [
'recording_id' => $recording->id,
'content_type' => $contentType,
]);
return $this->extractTranscriptText($response, $report, $message);
}
return $this->consumeWhisperStream($response, $report, $message, $recording->duration_seconds, $shouldContinue);
return $this->consumeWhisperStream($response, $report, $message, $recording, $shouldContinue);
}
/**
* @param Closure(string, int, ?string): void $report
* @param Closure(string, int, ?string, ?array): void $report
* @param (Closure(): bool)|null $shouldContinue
*/
private function consumeWhisperStream(
Response $response,
Closure $report,
string $message,
?int $durationSeconds,
Recording $recording,
?Closure $shouldContinue,
): string {
$body = $response->toPsrResponse()->getBody();
$buffer = '';
$accumulated = '';
$lastPercent = 50;
$eventsWithoutTimestamp = 0;
$lastFlushAt = 0.0;
$pending = false;
$lastPercent = 0;
$idleReads = 0;
try {
while (! $body->eof()) {
@@ -112,43 +114,42 @@ class TranscriptionService
$chunk = $body->read(8192);
if ($chunk === '') {
break;
$idleReads++;
if ($idleReads >= 40) {
break;
}
usleep(50_000);
continue;
}
$idleReads = 0;
$buffer .= $chunk;
foreach ($this->stream->extractPayloads($buffer) as $payload) {
[$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending] = $this->ingestEvent(
[$accumulated, $lastPercent] = $this->ingestEvent(
$payload,
$accumulated,
$lastPercent,
$eventsWithoutTimestamp,
$durationSeconds,
$recording,
$report,
$message,
$lastFlushAt,
$pending,
);
}
}
foreach ($this->stream->flushBuffer($buffer) as $payload) {
[$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending] = $this->ingestEvent(
[$accumulated, $lastPercent] = $this->ingestEvent(
$payload,
$accumulated,
$lastPercent,
$eventsWithoutTimestamp,
$durationSeconds,
$recording,
$report,
$message,
$lastFlushAt,
$pending,
);
}
if ($pending && $accumulated !== '') {
$report($message, $lastPercent, $accumulated);
}
} finally {
$response->close();
}
@@ -161,61 +162,55 @@ class TranscriptionService
}
/**
* @param Closure(string, int, ?string): void $report
* @return array{0: string, 1: int, 2: int, 3: bool}
* @param Closure(string, int, ?string, ?array): void $report
* @return array{0: string, 1: int}
*/
private function ingestEvent(
string $payload,
string $accumulated,
int $lastPercent,
int $eventsWithoutTimestamp,
?int $durationSeconds,
Recording $recording,
Closure $report,
string $message,
float &$lastFlushAt,
bool $pending,
): array {
$event = $this->stream->parseEvent($payload);
if ($event === null) {
return [$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending];
return [$accumulated, $lastPercent];
}
$wasEmpty = $accumulated === '';
$accumulated = $this->stream->applyEvent($accumulated, $event);
$whisper = $event['whisper'] ?? [];
if ($accumulated === '') {
return [$accumulated, $lastPercent, $eventsWithoutTimestamp, $pending];
if ($accumulated === '' && $whisper === []) {
return [$accumulated, $lastPercent];
}
$percent = $this->percentForEvent($event, $durationSeconds, $lastPercent, $eventsWithoutTimestamp);
$lastPercent = max($lastPercent, $percent);
$lastPercent = $this->percentForEvent($event, $recording, $lastPercent);
$now = microtime(true);
$shouldFlush = $wasEmpty || $event['done'] || ($now - $lastFlushAt) >= 1.0;
$report(
$message,
$lastPercent,
$accumulated === '' ? null : $accumulated,
$whisper === [] ? null : $whisper,
);
if ($shouldFlush) {
$report($message, $lastPercent, $accumulated);
$lastFlushAt = $now;
return [$accumulated, $lastPercent, $eventsWithoutTimestamp, false];
}
return [$accumulated, $lastPercent, $eventsWithoutTimestamp, true];
return [$accumulated, $lastPercent];
}
/**
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool} $event
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper?: array<string, mixed>} $event
*/
private function percentForEvent(array $event, ?int $durationSeconds, int $lastPercent, int &$eventsWithoutTimestamp): int
private function percentForEvent(array $event, Recording $recording, int $lastPercent): int
{
if ($event['end'] !== null && $durationSeconds !== null && $durationSeconds > 0) {
return (int) min(99, max(50, round(100 * $event['end'] / $durationSeconds)));
$duration = $recording->duration_seconds;
$end = $event['end'] ?? null;
if ($end === null || $duration === null || $duration <= 0) {
return $lastPercent;
}
$eventsWithoutTimestamp++;
return min(84, max($lastPercent, 50 + $eventsWithoutTimestamp));
return (int) min(99, max($lastPercent, round(100 * $end / $duration)));
}
private function localWhisperRequest(string $filename, string $path): PendingRequest
@@ -231,14 +226,29 @@ class TranscriptionService
->attach('file', fopen($path, 'r'), $filename);
}
private function extractTranscriptText(Response $response): string
/**
* @param Closure(string, int, ?string, ?array): void $report
*/
private function extractTranscriptText(Response $response, ?Closure $report = null, string $message = ''): string
{
$text = $response->json('text');
$json = $response->json();
if (! is_array($json)) {
throw new RuntimeException('Local transcription returned an empty transcript.');
}
$text = $json['text'] ?? null;
if (! is_string($text) || $text === '') {
throw new RuntimeException('Local transcription returned an empty transcript.');
}
$whisper = $this->stream->extractWhisperMeta($json);
if ($report !== null && $whisper !== []) {
$report($message, 99, $text, $whisper);
}
return $text;
}
}
app/Services/WhisperTranscriptionStream.php
+285 -9
View File
@@ -49,9 +49,9 @@ class WhisperTranscriptionStream
}
/**
* Parse one SSE JSON payload into append/replace/end/done fields.
* Parse one SSE JSON payload into append/replace/end/done/whisper fields.
*
* @return array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool}|null
* @return array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper: array<string, mixed>}|null
*/
public function parseEvent(string $json): ?array
{
@@ -62,20 +62,23 @@ class WhisperTranscriptionStream
}
$type = $data['type'] ?? null;
$whisper = $this->extractWhisperMeta($data);
$end = $this->latestAudioEnd($data, $whisper);
if ($type === 'transcript.text.delta') {
$delta = $data['delta'] ?? '';
if (! is_string($delta) || $delta === '') {
if ((! is_string($delta) || $delta === '') && $whisper === []) {
return null;
}
return [
'append' => $delta,
'append' => is_string($delta) && $delta !== '' ? $delta : null,
'replace' => null,
'end' => $this->nullableFloat($data['end'] ?? null),
'end' => $end,
'done' => false,
'legacy' => false,
'whisper' => $whisper,
];
}
@@ -85,19 +88,54 @@ class WhisperTranscriptionStream
return [
'append' => null,
'replace' => is_string($text) ? $text : '',
'end' => $this->nullableFloat($data['end'] ?? null),
'end' => $end,
'done' => true,
'legacy' => false,
'whisper' => $whisper,
];
}
if ($type === null && isset($data['text']) && is_string($data['text']) && $data['text'] !== '') {
if (isset($data['segments']) && is_array($data['segments'])) {
$text = $data['text'] ?? '';
if ((! is_string($text) || $text === '') && $whisper === []) {
return null;
}
return [
'append' => null,
'replace' => is_string($text) && $text !== '' ? $text : null,
'end' => $end,
'done' => true,
'legacy' => false,
'whisper' => $whisper,
];
}
if (
($type === null || $type === 'segment')
&& isset($data['text'])
&& is_string($data['text'])
&& $data['text'] !== ''
) {
return [
'append' => $data['text'],
'replace' => null,
'end' => $this->nullableFloat($data['end'] ?? null),
'end' => $end,
'done' => false,
'legacy' => true,
'whisper' => $whisper,
];
}
if ($whisper !== []) {
return [
'append' => null,
'replace' => null,
'end' => $end,
'done' => false,
'legacy' => false,
'whisper' => $whisper,
];
}
@@ -107,7 +145,7 @@ class WhisperTranscriptionStream
/**
* Apply a parsed event to the accumulated transcript.
*
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool} $event
* @param array{append: ?string, replace: ?string, end: ?float, done: bool, legacy: bool, whisper?: array<string, mixed>} $event
*/
public function applyEvent(string $accumulated, array $event): string
{
@@ -134,6 +172,108 @@ class WhisperTranscriptionStream
return $accumulated.$chunk;
}
/**
* @param array<string, mixed> $data
* @return array<string, mixed>
*/
public function extractWhisperMeta(array $data): array
{
$meta = [];
if (isset($data['language']) && is_string($data['language']) && $data['language'] !== '') {
$meta['language'] = $data['language'];
}
if (is_numeric($data['duration'] ?? null)) {
$meta['duration'] = (float) $data['duration'];
}
if (isset($data['logprobs']) && is_array($data['logprobs'])) {
$logprobs = $this->normalizeLogprobs($data['logprobs']);
if ($logprobs !== []) {
$meta['logprobs'] = $logprobs;
}
}
$segment = $this->normalizeSegment($data);
if ($segment !== null && ($data['type'] ?? null) !== 'transcript.text.delta') {
$meta['segment'] = $segment;
}
if (isset($data['segments']) && is_array($data['segments'])) {
$segments = [];
foreach ($data['segments'] as $row) {
if (! is_array($row)) {
continue;
}
$normalized = $this->normalizeSegment($row);
if ($normalized !== null) {
$segments[] = $normalized;
}
}
if ($segments !== []) {
$meta['segments'] = $segments;
}
}
if (isset($data['words']) && is_array($data['words']) && ! isset($meta['segment']) && ! isset($meta['segments'])) {
$words = $this->normalizeWords($data['words']);
if ($words !== []) {
$meta['words'] = $words;
}
}
return $meta;
}
/**
* @param array<string, mixed> $data
* @return array<string, mixed>|null
*/
public function normalizeSegment(array $data): ?array
{
$hasDetail = isset($data['start'])
|| isset($data['end'])
|| isset($data['words'])
|| isset($data['avg_logprob'])
|| isset($data['tokens'])
|| isset($data['no_speech_prob'])
|| array_key_exists('id', $data);
if (! $hasDetail) {
return null;
}
$text = $data['text'] ?? null;
if (! is_string($text) || $text === '') {
return null;
}
$segment = [
'id' => is_numeric($data['id'] ?? null) ? (int) $data['id'] : null,
'seek' => is_numeric($data['seek'] ?? null) ? (int) $data['seek'] : null,
'start' => $this->nullableFloat($data['start'] ?? null),
'end' => $this->nullableFloat($data['end'] ?? null),
'text' => $text,
'tokens' => $this->normalizeTokens($data['tokens'] ?? null),
'temperature' => $this->nullableFloat($data['temperature'] ?? null),
'avg_logprob' => $this->nullableFloat($data['avg_logprob'] ?? null),
'compression_ratio' => $this->nullableFloat($data['compression_ratio'] ?? null),
'no_speech_prob' => $this->nullableFloat($data['no_speech_prob'] ?? null),
'words' => $this->normalizeWords($data['words'] ?? null),
];
return array_filter($segment, fn (mixed $value): bool => $value !== null && $value !== []);
}
/**
* @return list<string>
*/
@@ -150,6 +290,60 @@ class WhisperTranscriptionStream
return $payloads;
}
/**
* Latest media timestamp in this event (segment/word `end`). Never file `duration`.
*
* @param array<string, mixed> $data
* @param array<string, mixed> $whisper
*/
public function latestAudioEnd(array $data, array $whisper): ?float
{
$ends = [];
$direct = $this->nullableFloat($data['end'] ?? null);
if ($direct !== null) {
$ends[] = $direct;
}
$this->collectAudioEnds($ends, $whisper);
if (isset($data['words']) && is_array($data['words'])) {
$this->collectAudioEnds($ends, ['words' => $data['words']]);
}
return $ends === [] ? null : max($ends);
}
/**
* @param list<float> $ends
* @param array<string, mixed> $node
*/
private function collectAudioEnds(array &$ends, array $node): void
{
$end = $this->nullableFloat($node['end'] ?? null);
if ($end !== null) {
$ends[] = $end;
}
foreach (['words', 'segments'] as $key) {
if (! isset($node[$key]) || ! is_array($node[$key])) {
continue;
}
foreach ($node[$key] as $child) {
if (is_array($child)) {
$this->collectAudioEnds($ends, $child);
}
}
}
if (isset($node['segment']) && is_array($node['segment'])) {
$this->collectAudioEnds($ends, $node['segment']);
}
}
private function nullableFloat(mixed $value): ?float
{
if (! is_numeric($value)) {
@@ -158,4 +352,86 @@ class WhisperTranscriptionStream
return (float) $value;
}
/**
* @return list<int>|null
*/
private function normalizeTokens(mixed $tokens): ?array
{
if (! is_array($tokens) || $tokens === []) {
return null;
}
$normalized = [];
foreach ($tokens as $token) {
if (is_numeric($token)) {
$normalized[] = (int) $token;
}
}
return $normalized === [] ? null : $normalized;
}
/**
* @return list<array{word: string, start: ?float, end: ?float, probability: ?float}>|null
*/
private function normalizeWords(mixed $words): ?array
{
if (! is_array($words) || $words === []) {
return null;
}
$normalized = [];
foreach ($words as $word) {
if (! is_array($word)) {
continue;
}
$text = $word['word'] ?? $word['text'] ?? null;
if (! is_string($text) || $text === '') {
continue;
}
$normalized[] = array_filter([
'word' => $text,
'start' => $this->nullableFloat($word['start'] ?? null),
'end' => $this->nullableFloat($word['end'] ?? null),
'probability' => $this->nullableFloat($word['probability'] ?? $word['prob'] ?? null),
], fn (mixed $value): bool => $value !== null);
}
return $normalized === [] ? null : $normalized;
}
/**
* @return list<array{token: ?string, logprob: ?float}>
*/
private function normalizeLogprobs(array $logprobs): array
{
$normalized = [];
foreach ($logprobs as $row) {
if (! is_array($row)) {
continue;
}
$token = $row['token'] ?? $row['bytes'] ?? null;
$token = is_string($token) ? $token : null;
$logprob = $this->nullableFloat($row['logprob'] ?? $row['avg_logprob'] ?? null);
if ($token === null && $logprob === null) {
continue;
}
$normalized[] = array_filter([
'token' => $token,
'logprob' => $logprob,
], fn (mixed $value): bool => $value !== null);
}
return $normalized;
}
}