Compare commits

..
2 Commits
Author SHA1 Message Date
ben b3fb74fb1b Make the disk space meter visible and show used percent.
CI / test (push) Failing after 31s
CI / build-and-push (push) Skipped
CI / deploy (push) Skipped
Use an inline-colored full-width progress bar so the fill renders without relying on purged Tailwind classes.
2026-08-13 00:37:20 +02:00
ben f42d124593 Harden stuck Whisper recovery and record transcription duration.
Fail jobs on timeout, treat stale reserved queue rows as orphans, skip migrate/seed on queue/reverb boot, and store how long each successful run took.
2026-08-13 00:12:06 +02:00
10 changed files with 258 additions and 97 deletions
+1 -1
View File
@@ -210,7 +210,7 @@ Edit `.env` before `docker compose up` when you need different ports or models:
| `REVERB_HOST_PORT` | Host port for WebSockets | `8081` | | `REVERB_HOST_PORT` | Host port for WebSockets | `8081` |
| `WHISPER_HOST_PORT` | Host port for Whisper | `8090` | | `WHISPER_HOST_PORT` | Host port for Whisper | `8090` |
| `LOCAL_WHISPER_MODEL` | Whisper model id | `Systran/faster-whisper-base` | | `LOCAL_WHISPER_MODEL` | Whisper model id | `Systran/faster-whisper-base` |
| `TRANSCRIPTION_TIMEOUT` | Job/HTTP timeout (seconds) | `600` | | `TRANSCRIPTION_TIMEOUT` | Job/HTTP timeout (seconds). Hung Whisper calls fail the job; UI can restart. | `600` |
| `DB_QUEUE_RETRY_AFTER` | Must exceed `TRANSCRIPTION_TIMEOUT` | `660` | | `DB_QUEUE_RETRY_AFTER` | Must exceed `TRANSCRIPTION_TIMEOUT` | `660` |
Inside Compose, Laravel talks to Whisper at `http://whisper:8000/v1` and publishes broadcasts to the `reverb` service. The browser connects to Reverb on `localhost:8081`. Inside Compose, Laravel talks to Whisper at `http://whisper:8000/v1` and publishes broadcasts to the `reverb` service. The browser connects to Reverb on `localhost:8081`.
+11 -21
View File
@@ -6,9 +6,11 @@ use App\Models\Recording;
use App\Services\TranscriptionService; use App\Services\TranscriptionService;
use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable; use Illuminate\Foundation\Queue\Queueable;
use Illuminate\Queue\Attributes\FailOnTimeout;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use Throwable; use Throwable;
#[FailOnTimeout]
class TranscribeRecording implements ShouldQueue class TranscribeRecording implements ShouldQueue
{ {
use Queueable; use Queueable;
@@ -20,6 +22,9 @@ class TranscribeRecording implements ShouldQueue
/** /**
* The number of seconds the job can run before timing out. * The number of seconds the job can run before timing out.
*
* Covers a hung Whisper HTTP call: the worker is killed, failed() runs,
* and the recording is marked failed so the UI can restart.
*/ */
public int $timeout; public int $timeout;
@@ -134,16 +139,7 @@ class TranscribeRecording implements ShouldQueue
} }
if ($this->recording->ownsTranscriptionRun($this->runStartedAt)) { if ($this->recording->ownsTranscriptionRun($this->runStartedAt)) {
$this->recording->forceFill([ $this->recording->markTranscriptionComplete($text);
'transcript' => $text,
'transcription_status' => 'done',
'transcription_progress' => 'Transcription complete',
'transcription_percent' => 100,
'transcription_error' => null,
'transcribed_at' => now(),
])->save();
$this->recording->broadcastTranscriptionUpdated();
return true; return true;
} }
@@ -152,22 +148,16 @@ class TranscribeRecording implements ShouldQueue
if ( if (
$this->recording->transcription_status === 'failed' $this->recording->transcription_status === 'failed'
&& $this->recording->matchesTranscriptionRun($this->runStartedAt) && $this->recording->matchesTranscriptionRun($this->runStartedAt)
&& str_contains((string) $this->recording->transcription_error, 'worker stopped') && (
str_contains((string) $this->recording->transcription_error, 'worker stopped')
|| str_contains((string) $this->recording->transcription_error, 'timed out')
)
) { ) {
Log::warning('Recovering transcript after false orphan failure', [ Log::warning('Recovering transcript after false orphan failure', [
'recording_id' => $this->recording->id, 'recording_id' => $this->recording->id,
]); ]);
$this->recording->forceFill([ $this->recording->markTranscriptionComplete($text);
'transcript' => $text,
'transcription_status' => 'done',
'transcription_progress' => 'Transcription complete',
'transcription_percent' => 100,
'transcription_error' => null,
'transcribed_at' => now(),
])->save();
$this->recording->broadcastTranscriptionUpdated();
return true; return true;
} }
+97 -41
View File
@@ -41,6 +41,7 @@ class Recording extends Model
'transcription_driver', 'transcription_driver',
'ollama_url', 'ollama_url',
'transcribed_at', 'transcribed_at',
'transcription_duration_seconds',
]; ];
/** /**
@@ -53,6 +54,7 @@ class Recording extends Model
'transcribed_at' => 'datetime', 'transcribed_at' => 'datetime',
'transcription_started_at' => 'datetime', 'transcription_started_at' => 'datetime',
'duration_seconds' => 'integer', 'duration_seconds' => 'integer',
'transcription_duration_seconds' => 'integer',
'file_size_bytes' => 'integer', 'file_size_bytes' => 'integer',
'transcription_percent' => 'integer', 'transcription_percent' => 'integer',
'user_id' => 'integer', 'user_id' => 'integer',
@@ -138,6 +140,7 @@ class Recording extends Model
'transcription_percent' => null, 'transcription_percent' => null,
'transcription_started_at' => now(), 'transcription_started_at' => now(),
'transcription_error' => null, 'transcription_error' => null,
'transcription_duration_seconds' => null,
// Keep the previous transcript until a new run succeeds. // Keep the previous transcript until a new run succeeds.
'transcribed_at' => $this->transcribed_at, 'transcribed_at' => $this->transcribed_at,
]); ]);
@@ -228,33 +231,40 @@ class Recording extends Model
return ($start > 0 ? '…' : '').$excerpt.(mb_strlen($transcript) > $start + mb_strlen($excerpt) ? '…' : ''); return ($start > 0 ? '…' : '').$excerpt.(mb_strlen($transcript) > $start + mb_strlen($excerpt) ? '…' : '');
} }
/**
* Seconds after which a reserved queue row is considered abandoned
* (worker died mid-Whisper without releasing the job).
*/
public function transcriptionJobStaleAfterSeconds(): int
{
return max(120, (int) config('ai.transcription_timeout', 600) + 90);
}
/** /**
* Whether a TranscribeRecording job for this recording is still on the queue. * Whether a TranscribeRecording job for this recording is still on the queue.
*
* Reserved jobs older than the transcription timeout (+ grace) are ignored so
* orphan recovery can unblock the UI when Whisper/the worker is wedged.
*/ */
public function hasActiveTranscriptionJob(): bool public function hasActiveTranscriptionJob(): bool
{ {
$staleBefore = now()->timestamp - $this->transcriptionJobStaleAfterSeconds();
return DB::table('jobs') return DB::table('jobs')
->pluck('payload') ->orderBy('id')
->contains(function (string $payload): bool { ->get(['id', 'payload', 'reserved_at'])
if (! str_contains($payload, TranscribeRecording::class)) { ->contains(function (object $job) use ($staleBefore): bool {
$payload = (string) $job->payload;
if (! str_contains($payload, 'TranscribeRecording')) {
return false; return false;
} }
$data = json_decode($payload, true); if ($job->reserved_at !== null && (int) $job->reserved_at <= $staleBefore) {
$command = $data['data']['command'] ?? null;
if (! is_string($command)) {
return false; return false;
} }
try { return $this->jobPayloadBelongsToRecording($payload);
$job = unserialize($command);
} catch (Throwable) {
return $this->payloadMentionsRecording($payload);
}
return $job instanceof TranscribeRecording
&& (int) $job->recording->getKey() === (int) $this->id;
}); });
} }
@@ -263,8 +273,11 @@ class Recording extends Model
*/ */
private function payloadMentionsRecording(string $payload): bool private function payloadMentionsRecording(string $payload): bool
{ {
return (bool) preg_match('/id";i:'.$this->id.';/', $payload) // Jobs table stores JSON; the serialized command inside escapes quotes as \".
|| str_contains($payload, 'id";s:'.strlen((string) $this->id).':"'.$this->id.'"'); return (bool) preg_match('/id\\\\";i:'.$this->id.';/', $payload)
|| (bool) preg_match('/id";i:'.$this->id.';/', $payload)
|| str_contains($payload, 'id";s:'.strlen((string) $this->id).':"'.$this->id.'"')
|| str_contains($payload, 'id\\\\";s:'.strlen((string) $this->id).':\\"'.$this->id.'\\"');
} }
/** /**
@@ -332,34 +345,16 @@ class Recording extends Model
->each(function (object $job) use (&$deleted): void { ->each(function (object $job) use (&$deleted): void {
$payload = (string) $job->payload; $payload = (string) $job->payload;
if (! str_contains($payload, TranscribeRecording::class)) { if (! str_contains($payload, 'TranscribeRecording')) {
return; return;
} }
$data = json_decode($payload, true); if (! $this->jobPayloadBelongsToRecording($payload)) {
$command = $data['data']['command'] ?? null;
if (! is_string($command)) {
return; return;
} }
try { DB::table('jobs')->where('id', $job->id)->delete();
$queued = unserialize($command); $deleted++;
} catch (Throwable) {
if (! preg_match('/id";i:'.$this->id.';/', $payload)) {
return;
}
DB::table('jobs')->where('id', $job->id)->delete();
$deleted++;
return;
}
if ($queued instanceof TranscribeRecording && (int) $queued->recording->getKey() === (int) $this->id) {
DB::table('jobs')->where('id', $job->id)->delete();
$deleted++;
}
}); });
$this->releaseTranscriptionUniqueLock(); $this->releaseTranscriptionUniqueLock();
@@ -367,6 +362,36 @@ class Recording extends Model
return $deleted; return $deleted;
} }
/**
* Whether a jobs.payload row targets this recording.
*/
private function jobPayloadBelongsToRecording(string $payload): bool
{
$data = json_decode($payload, true);
$command = $data['data']['command'] ?? null;
if (is_string($command)) {
if (
preg_match('/id";i:'.$this->id.';/', $command)
|| str_contains($command, 'id";s:'.strlen((string) $this->id).':"'.$this->id.'"')
) {
return true;
}
try {
$queued = unserialize($command);
if ($queued instanceof TranscribeRecording) {
return (int) $queued->recording->getKey() === (int) $this->id;
}
} catch (Throwable) {
// Fall through to escaped JSON heuristics.
}
}
return $this->payloadMentionsRecording($payload);
}
/** /**
* Stop transcription: drop queued jobs and mark the run cancelled. * Stop transcription: drop queued jobs and mark the run cancelled.
* *
@@ -400,6 +425,30 @@ class Recording extends Model
)->forceRelease(); )->forceRelease();
} }
/**
* Persist a successful transcript and how long the run took.
*/
public function markTranscriptionComplete(string $text): void
{
$finishedAt = now();
$startedAt = $this->transcription_started_at;
$durationSeconds = $startedAt === null
? null
: max(0, (int) round($startedAt->diffInSeconds($finishedAt)));
$this->forceFill([
'transcript' => $text,
'transcription_status' => 'done',
'transcription_progress' => 'Transcription complete',
'transcription_percent' => 100,
'transcription_error' => null,
'transcribed_at' => $finishedAt,
'transcription_duration_seconds' => $durationSeconds,
])->save();
RecordingTranscriptionUpdated::dispatch($this->fresh());
}
/** /**
* Mark transcription as failed and unblock the UI. * Mark transcription as failed and unblock the UI.
*/ */
@@ -420,7 +469,7 @@ class Recording extends Model
} }
/** /**
* Recover a stuck transcription if the queue job is gone. * Recover a stuck transcription if the queue job is gone or a reservation is stale.
*/ */
public function recoverOrphanedTranscription(): bool public function recoverOrphanedTranscription(): bool
{ {
@@ -428,8 +477,11 @@ class Recording extends Model
return false; return false;
} }
// Drop abandoned reserved rows so a restart can enqueue cleanly.
$this->discardQueuedTranscriptionJobs();
$this->markTranscriptionFailed( $this->markTranscriptionFailed(
'Transcription worker stopped before finishing. Start transcription again.', 'Transcription timed out or the worker stopped before finishing. Start transcription again.',
); );
return true; return true;
@@ -519,6 +571,10 @@ class Recording extends Model
'elapsed_seconds' => $elapsed, 'elapsed_seconds' => $elapsed,
'elapsed_human' => $elapsed === null ? null : $this->formatElapsed($elapsed), 'elapsed_human' => $elapsed === null ? null : $this->formatElapsed($elapsed),
'duration_seconds' => $this->duration_seconds, 'duration_seconds' => $this->duration_seconds,
'transcription_duration_seconds' => $this->transcription_duration_seconds,
'transcription_duration_human' => $this->transcription_duration_seconds === null
? null
: $this->formatElapsed($this->transcription_duration_seconds),
'is_active' => $this->isTranscribing(), 'is_active' => $this->isTranscribing(),
'has_transcript' => filled($this->transcript), 'has_transcript' => filled($this->transcript),
'transcript' => $this->transcript, 'transcript' => $this->transcript,
-18
View File
@@ -33,24 +33,6 @@ class DiskSpaceBar extends Component
return $this->disk !== null; return $this->disk !== null;
} }
/**
* Progress fill color based on remaining free space.
*/
public function barColor(): string
{
$freePercent = $this->disk['free_percent'] ?? 100;
if ($freePercent <= 5) {
return 'bg-red-600';
}
if ($freePercent <= 15) {
return 'bg-amber-500';
}
return 'bg-teal-600';
}
/** /**
* Get the view / contents that represent the component. * Get the view / contents that represent the component.
*/ */
@@ -0,0 +1,30 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('recordings', function (Blueprint $table) {
$table->unsignedInteger('transcription_duration_seconds')
->nullable()
->after('transcribed_at');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('recordings', function (Blueprint $table) {
$table->dropColumn('transcription_duration_seconds');
});
}
};
+17 -2
View File
@@ -36,7 +36,22 @@ if [ ! -f vendor/autoload.php ]; then
composer install --prefer-dist --no-interaction composer install --prefer-dist --no-interaction
fi fi
php artisan migrate --force --no-interaction # Only the web app should migrate/seed. Queue and Reverb share the DB and must
php artisan db:seed --force --no-interaction # not race on sqlite (locks) or re-seed on every restart/deploy.
should_bootstrap_db() {
case " $* " in
*" queue:work "*|*" queue:listen "*|*" reverb:start "*)
return 1
;;
*)
return 0
;;
esac
}
if should_bootstrap_db "$@"; then
php artisan migrate --force --no-interaction
php artisan db:seed --force --no-interaction
fi
exec "$@" exec "$@"
@@ -1,24 +1,33 @@
@php @php
/** @var array{used_percent: float, free_percent: float, free_human: string, total_human: string, used_human: string} $disk */ /** @var array{used_percent: float, free_percent: float, free_human: string, total_human: string, used_human: string} $disk */
$usedPercent = min(100, max(0, (float) $disk['used_percent']));
$usedPercentLabel = rtrim(rtrim(number_format($usedPercent, 1, '.', ''), '0'), '.') ?: '0';
$freePercent = (float) ($disk['free_percent'] ?? 100);
// Inline colors so the fill is visible even before / without a Tailwind rebuild.
$fillColor = match (true) {
$freePercent <= 5 => '#dc2626',
$freePercent <= 15 => '#f59e0b',
default => '#0d9488',
};
@endphp @endphp
<div <div
class="flex items-center gap-2 rounded-lg border border-zinc-200 bg-zinc-50 px-2.5 py-1.5 dark:border-zinc-600 dark:bg-zinc-900/50" class="flex min-w-36 flex-col gap-1 rounded-lg border border-zinc-200 bg-zinc-50 px-2.5 py-1.5 dark:border-zinc-600 dark:bg-zinc-900/50"
title="{{ $disk['free_human'] }} free of {{ $disk['total_human'] }} · {{ $disk['used_percent'] }}% used" title="{{ $disk['free_human'] }} free of {{ $disk['total_human'] }} · {{ $usedPercentLabel }}% used"
> >
<div <div
class="h-1.5 w-14 shrink-0 overflow-hidden rounded-full bg-zinc-200 dark:bg-zinc-700" class="h-2 w-full overflow-hidden rounded-full bg-zinc-200 dark:bg-zinc-700"
role="progressbar" role="progressbar"
aria-valuemin="0" aria-valuemin="0"
aria-valuemax="100" aria-valuemax="100"
aria-valuenow="{{ (int) round($disk['used_percent']) }}" aria-valuenow="{{ (int) round($usedPercent) }}"
aria-label="Disk space used" aria-label="Disk space {{ $usedPercentLabel }}% used"
> >
<div <div
class="h-full rounded-full transition-[width] duration-300 {{ $barColor }}" class="h-full rounded-full transition-[width] duration-300"
style="width: {{ min(100, max(0, $disk['used_percent'])) }}%" style="width: {{ $usedPercent }}%; background-color: {{ $fillColor }};"
></div> ></div>
</div> </div>
<span class="hidden text-xs font-medium tabular-nums text-zinc-600 sm:inline dark:text-zinc-300"> <span class="text-xs font-medium tabular-nums text-zinc-600 dark:text-zinc-300">
{{ $disk['free_human'] }} free {{ $usedPercentLabel }}% used · {{ $disk['free_human'] }} free
</span> </span>
</div> </div>
@@ -216,7 +216,10 @@
<flux:text <flux:text
class="mt-4 text-xs" class="mt-4 text-xs"
x-show="status.transcribed_at" x-show="status.transcribed_at"
x-text="status.transcribed_at ? ('Transcribed ' + formatTimestamp(status.transcribed_at)) : ''" x-text="status.transcribed_at
? ('Transcribed ' + formatTimestamp(status.transcribed_at)
+ (status.transcription_duration_human ? (' · took ' + status.transcription_duration_human) : ''))
: ''"
></flux:text> ></flux:text>
</div> </div>
+10 -3
View File
@@ -50,10 +50,17 @@ class DiskSpaceTest extends TestCase
{ {
Cache::flush(); Cache::flush();
$this->get(route('recordings.index')) $response = $this->get(route('recordings.index'))
->assertOk() ->assertOk()
->assertSee('Disk space used', false) ->assertSee('Disk space', false)
->assertSee('% used', false)
->assertSee('free') ->assertSee('free')
->assertSee('role="progressbar"', false); ->assertSee('role="progressbar"', false)
->assertSee('background-color:', false);
$this->assertMatchesRegularExpression(
'/width:\s*[\d.]+%;\s*background-color:\s*#(0d9488|f59e0b|dc2626)/',
$response->getContent(),
);
} }
} }
+70 -1
View File
@@ -13,6 +13,7 @@ use App\Services\TranscriptionService;
use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Http\UploadedFile; use Illuminate\Http\UploadedFile;
use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Storage;
use Laravel\Ai\Transcription; use Laravel\Ai\Transcription;
use Livewire\Livewire; use Livewire\Livewire;
@@ -209,6 +210,8 @@ class RecordingUploadTest extends TestCase
Transcription::fake(['Hello from the recorder.']); Transcription::fake(['Hello from the recorder.']);
$this->freezeTime();
$recording = Recording::query()->create([ $recording = Recording::query()->create([
'user_id' => $this->user->id, 'user_id' => $this->user->id,
'title' => 'Sample', 'title' => 'Sample',
@@ -217,6 +220,7 @@ class RecordingUploadTest extends TestCase
'file_size_bytes' => 12, 'file_size_bytes' => 12,
'transcription_status' => 'pending', 'transcription_status' => 'pending',
'transcription_driver' => 'local', 'transcription_driver' => 'local',
'transcription_started_at' => now()->subSeconds(42),
]); ]);
(new TranscribeRecording($recording))->handle(app(TranscriptionService::class)); (new TranscribeRecording($recording))->handle(app(TranscriptionService::class));
@@ -227,6 +231,8 @@ class RecordingUploadTest extends TestCase
$this->assertSame(100, $recording->transcription_percent); $this->assertSame(100, $recording->transcription_percent);
$this->assertSame('Transcription complete', $recording->transcription_progress); $this->assertSame('Transcription complete', $recording->transcription_progress);
$this->assertNotNull($recording->transcribed_at); $this->assertNotNull($recording->transcribed_at);
$this->assertSame(42, $recording->transcription_duration_seconds);
$this->assertSame('42s', $recording->transcriptionStatusPayload()['transcription_duration_human']);
} }
public function test_transcription_job_stores_error_on_failure(): void public function test_transcription_job_stores_error_on_failure(): void
@@ -283,7 +289,70 @@ class RecordingUploadTest extends TestCase
$recording->refresh(); $recording->refresh();
$this->assertSame('failed', $recording->transcription_status); $this->assertSame('failed', $recording->transcription_status);
$this->assertStringContainsString('worker stopped', $recording->transcription_error); $this->assertStringContainsString('timed out', $recording->transcription_error);
}
public function test_stale_reserved_job_is_treated_as_orphaned(): void
{
$recording = Recording::query()->create([
'user_id' => $this->user->id,
'title' => 'Wedged whisper',
'original_filename' => 'wedged.mp3',
'file_path' => 'recordings/wedged.mp3',
'file_size_bytes' => 100,
'transcription_status' => 'processing',
'transcription_progress' => 'Transcribing locally…',
'transcription_percent' => 50,
'transcription_driver' => 'local',
'transcription_started_at' => now()->subMinutes(20),
'updated_at' => now()->subMinutes(20),
]);
$job = new TranscribeRecording($recording);
$payload = json_encode([
'displayName' => TranscribeRecording::class,
'data' => [
'command' => serialize($job),
],
], JSON_THROW_ON_ERROR);
DB::table('jobs')->insert([
'queue' => 'default',
'payload' => $payload,
'attempts' => 1,
'reserved_at' => now()->subMinutes(15)->timestamp,
'available_at' => now()->subMinutes(20)->timestamp,
'created_at' => now()->subMinutes(20)->timestamp,
]);
$this->assertFalse($recording->hasActiveTranscriptionJob());
$this->assertTrue($recording->isOrphanedTranscription());
$deleted = $recording->discardQueuedTranscriptionJobs();
$this->assertSame(1, $deleted, 'stale reserved job should be discarded');
$this->assertDatabaseCount('jobs', 0);
// Put the stale job back to exercise status-poll recovery.
DB::table('jobs')->insert([
'queue' => 'default',
'payload' => $payload,
'attempts' => 1,
'reserved_at' => now()->subMinutes(15)->timestamp,
'available_at' => now()->subMinutes(20)->timestamp,
'created_at' => now()->subMinutes(20)->timestamp,
]);
$recording->forceFill([
'transcription_status' => 'processing',
'transcription_error' => null,
])->save();
$this->getJson(route('recordings.transcription-status', $recording))
->assertOk()
->assertJsonPath('status', 'failed');
$this->assertDatabaseCount('jobs', 0);
$recording->refresh();
$this->assertSame('failed', $recording->transcription_status);
} }
public function test_recent_processing_is_not_marked_orphaned(): void public function test_recent_processing_is_not_marked_orphaned(): void