Expand Watchtower reporting with queue and scheduled task hooks.

Co-authored-by: Cursor <cursoragent@cursor.com>

app/Providers/AppServiceProvider.php

@@ -4,9 +4,6 @@ namespace App\Providers;
 
 use App\Auth\Http\Responses\LoginResponse as AppLoginResponse;
 use Filament\Auth\Http\Responses\Contracts\LoginResponse;
-use Illuminate\Queue\Events\JobFailed;
-use Illuminate\Support\Facades\Log;
-use Illuminate\Support\Facades\Queue;
 use Illuminate\Support\ServiceProvider;
 
 class AppServiceProvider extends ServiceProvider
@@ -24,17 +21,6 @@ class AppServiceProvider extends ServiceProvider
      */
     public function boot(): void
     {
-        Queue::failing(function (JobFailed $event): void {
-            if (! str_contains($event->job->getName(), 'SendQueuedNotifications')) {
-                return;
-            }
-
-            Log::error('Queued timesheet notification job failed', [
-                'job' => $event->job->getName(),
-                'connection' => $event->connectionName,
-                'queue' => $event->job->getQueue(),
-                'exception' => $event->exception->getMessage(),
-            ]);
-        });
+        //
     }
 }

app/Providers/WatchtowerServiceProvider.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace App\Providers;
+
+use App\Support\WatchtowerReporter;
+use Illuminate\Console\Events\ScheduledTaskFailed;
+use Illuminate\Queue\Events\JobFailed;
+use Illuminate\Support\Facades\Event;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\ServiceProvider;
+
+class WatchtowerServiceProvider extends ServiceProvider
+{
+    public function register(): void
+    {
+        $this->app->singleton(WatchtowerReporter::class);
+    }
+
+    public function boot(): void
+    {
+        Queue::failing(function (JobFailed $event): void {
+            app(WatchtowerReporter::class)->report($event->exception, [
+                'source' => 'queue',
+                'job' => $event->job->resolveName(),
+                'connection' => $event->connectionName,
+                'queue' => $event->job->getQueue(),
+            ]);
+
+            if (str_contains($event->job->getName(), 'SendQueuedNotifications')) {
+                Log::error('Queued timesheet notification job failed', [
+                    'job' => $event->job->getName(),
+                    'connection' => $event->connectionName,
+                    'queue' => $event->job->getQueue(),
+                    'exception' => $event->exception->getMessage(),
+                ]);
+            }
+        });
+
+        Event::listen(ScheduledTaskFailed::class, function (ScheduledTaskFailed $event): void {
+            app(WatchtowerReporter::class)->report($event->exception, [
+                'source' => 'scheduled',
+                'task' => $event->task->getSummaryForDisplay(),
+            ]);
+        });
+    }
+}

app/Support/WatchtowerReporter.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace App\Support;
+
+use Illuminate\Support\Facades\Http;
+use Symfony\Component\HttpKernel\Exception\HttpExceptionInterface;
+use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
+use Throwable;
+
+class WatchtowerReporter
+{
+    public function isEnabled(): bool
+    {
+        return filled(config('services.watchtower.url'))
+            && filled(config('services.watchtower.token'));
+    }
+
+    /**
+     * Mirror Flare noise filters — skip expected client errors.
+     */
+    public function shouldReport(Throwable $throwable): bool
+    {
+        if ($throwable instanceof NotFoundHttpException) {
+            return false;
+        }
+
+        if ($throwable instanceof HttpExceptionInterface && $throwable->getStatusCode() < 500) {
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * @param  array<string, mixed>  $context
+     */
+    public function report(Throwable $throwable, array $context = []): void
+    {
+        if (! $this->isEnabled() || ! $this->shouldReport($throwable)) {
+            return;
+        }
+
+        try {
+            $request = app()->runningInConsole() ? null : request();
+
+            Http::withToken((string) config('services.watchtower.token'))
+                ->timeout(3)
+                ->post(rtrim((string) config('services.watchtower.url'), '/').'/api/errors', [
+                    'app_name' => config('services.watchtower.app_name'),
+                    'level' => 'error',
+                    'message' => $throwable->getMessage(),
+                    'trace' => $throwable->getTraceAsString(),
+                    'context' => array_filter(array_merge([
+                        'exception_class' => $throwable::class,
+                        'file' => $throwable->getFile(),
+                        'line' => $throwable->getLine(),
+                        'url' => $request?->fullUrl(),
+                    ], $context), fn ($value) => $value !== null),
+                ]);
+        } catch (Throwable) {
+            // Never let a Watchtower outage break the app.
+        }
+    }
+}

bootstrap/app.php

@@ -3,11 +3,11 @@
 use App\Http\Middleware\AttachFlareContext;
 use App\Http\Middleware\RestrictHealthCheck;
 use App\Http\Middleware\SecurityHeaders;
+use App\Support\WatchtowerReporter;
 use Illuminate\Foundation\Application;
 use Illuminate\Foundation\Configuration\Exceptions;
 use Illuminate\Foundation\Configuration\Middleware;
 use Illuminate\Http\Request;
-use Illuminate\Support\Facades\Http;
 use Illuminate\Support\Facades\Route;
 use Spatie\LaravelFlare\Facades\Flare;
 use Symfony\Component\HttpKernel\Exception\HttpExceptionInterface;
@@ -57,32 +57,8 @@ return Application::configure(basePath: dirname(__DIR__))
         );
 
         $exceptions->report(function (\Throwable $e): void {
-            $watchtowerUrl = config('services.watchtower.url');
-            $watchtowerToken = config('services.watchtower.token');
-
-            if (! filled($watchtowerUrl) || ! filled($watchtowerToken)) {
-                return;
-            }
-
-            try {
-                $request = app()->runningInConsole() ? null : request();
-
-                Http::withToken($watchtowerToken)
-                    ->timeout(3)
-                    ->post(rtrim($watchtowerUrl, '/').'/api/errors', [
-                        'app_name' => config('services.watchtower.app_name'),
-                        'level' => 'error',
-                        'message' => $e->getMessage(),
-                        'trace' => $e->getTraceAsString(),
-                        'context' => array_filter([
-                            'exception_class' => $e::class,
-                            'file' => $e->getFile(),
-                            'line' => $e->getLine(),
-                            'url' => $request?->fullUrl(),
-                        ], fn ($value) => $value !== null),
-                    ]);
-            } catch (\Throwable) {
-                // Never let a Watchtower outage break the app.
-            }
+            app(WatchtowerReporter::class)->report($e, [
+                'source' => app()->runningInConsole() ? 'console' : 'http',
+            ]);
         });
     })->create();

bootstrap/providers.php

@@ -2,5 +2,6 @@
 
 return [
     App\Providers\AppServiceProvider::class,
+    App\Providers\WatchtowerServiceProvider::class,
     App\Providers\Filament\AdminPanelProvider::class,
 ];

docs/OBSERVABILITY.md

@@ -1,9 +1,10 @@
 # Observability Runbook — Quatriz TimeSheet
 
-Production admin timesheet app with three integrated tools:
+Production admin timesheet app with four integrated tools:
 
 | Tool | Package / service | Purpose |
 |------|-------------------|---------|
+| **Watchtower** | Self-hosted at log.skysyaz.my | Exception and background-job error reporting |
 | **Flare** | `spatie/laravel-flare` | Exception and error reporting |
 | **Activity log** | `spatie/laravel-activitylog` | Admin/user audit trail |
 | **Better Uptime** | External monitors + signed heartbeats | Uptime and cron/queue liveness |
@@ -29,6 +30,9 @@ Production admin timesheet app with three integrated tools:
 | `UPTIME_SCHEDULER_STALE_MINUTES` | No | `5` | Scheduler heartbeat TTL |
 | `UPTIME_QUEUE_STALE_MINUTES` | No | `5` | Queue worker heartbeat TTL |
 | `HEALTH_CHECK_ALLOWED_IPS` | Recommended | *(empty)* | Comma-separated IPs for `/up` |
+| `WATCHTOWER_URL` | For Watchtower | *(empty)* | Watchtower base URL (e.g. `https://log.skysyaz.my`) |
+| `WATCHTOWER_TOKEN` | For Watchtower | *(empty)* | Bearer token from `php artisan watchtower:token timesheet` on Watchtower |
+| `WATCHTOWER_APP_NAME` | No | `timesheet` | Source app name shown in Watchtower panel |
 
 Generate a heartbeat token:
 
@@ -49,6 +53,7 @@ php -r "echo bin2hex(random_bytes(32)), PHP_EOL;"
 2. **Environment**
    - Copy new vars from `.env.example` into production `.env`
    - Set `FLARE_KEY` and `FLARE_REPORT=true` when Flare credentials are ready
+   - Set `WATCHTOWER_URL` and `WATCHTOWER_TOKEN` on production (deploy does not sync `.env`)
    - Set `BETTER_UPTIME_ENABLED=true` and `UPTIME_HEARTBEAT_TOKEN` when monitors are ready
    - Add Better Uptime probe IPs to `HEALTH_CHECK_ALLOWED_IPS` for `/up`
 
@@ -79,6 +84,7 @@ php -r "echo bin2hex(random_bytes(32)), PHP_EOL;"
    php artisan uptime:signal-heartbeat
    curl -s -H "X-Uptime-Token: YOUR_TOKEN" "https://timesheet.quatriz-sd.my/uptime/heartbeat"
    php artisan flare:test --errors   # after FLARE_KEY is set
+   php artisan test --filter=WatchtowerIntegrationTest
    ```
 
 ### Rollback plan
@@ -140,6 +146,55 @@ Create these monitors in [Better Uptime / Better Stack](https://betteruptime.com
 
 ---
 
+## Watchtower
+
+Self-hosted error ingest at **https://log.skysyaz.my**. Alerts via Telegram when new unresolved issues appear.
+
+### Setup
+
+1. On Watchtower: `php artisan watchtower:token timesheet`
+2. Add `WATCHTOWER_URL` and `WATCHTOWER_TOKEN` to this app's `.env`
+3. `php artisan config:cache` on production after env changes
+4. `php artisan test --filter=WatchtowerIntegrationTest`
+
+### Reporting hooks
+
+All hooks call `App\Support\WatchtowerReporter` — never throws if Watchtower is down (3s timeout + try/catch).
+
+| Hook | File | `context.source` | When it fires |
+|------|------|------------------|---------------|
+| Exception reporter | `bootstrap/app.php` | `http` or `console` | Unhandled/reportable exceptions (HTTP requests, Artisan commands) |
+| Queue failure | `app/Providers/WatchtowerServiceProvider.php` | `queue` | Any job exhausts retries (`Queue::failing`) — notifications, heartbeats, future jobs |
+| Scheduled task failure | `app/Providers/WatchtowerServiceProvider.php` | `scheduled` | Cron task throws (`ScheduledTaskFailed`) — heartbeat, activitylog:clean, future schedules |
+
+### Payload sent to `POST /api/errors`
+
+- `app_name`, `level`, `message`, `trace`
+- `context`: `exception_class`, `file`, `line`, `url` (HTTP only), plus hook-specific fields (`job`, `task`, `source`, …)
+
+### Ignored exceptions (same noise policy as Flare)
+
+- `404 Not Found`
+- HTTP exceptions with status < 500
+
+Laravel also skips reporting `ValidationException`, `AuthorizationException`, and similar expected errors before the Watchtower hook runs.
+
+### Not automatically reported
+
+- Errors only logged with `Log::warning` / `Log::info` (no throw)
+- Validation and auth failures handled in Filament/forms
+- Duplicate queue report: `QueuesTimesheetNotification::failed()` logs locally; Watchtower is notified once via `Queue::failing`
+
+### Ops: triaging a Watchtower alert
+
+1. Open https://log.skysyaz.my → filter by app **timesheet**
+2. Check `context.source` — `http` vs `queue` vs `scheduled` vs `console`
+3. For `queue`: inspect `job` and `queue` in context; restart `timesheet-queue` if worker died
+4. For `scheduled`: verify cron (`schedule:run`) and the failing command in `routes/console.php`
+5. Resolve in panel when fixed; dedup suppresses repeat Telegram alerts within 1 hour
+
+---
+
 ## Activity log
 
 ### What is logged
@@ -197,6 +252,9 @@ curl -H "X-Uptime-Token: YOUR_TOKEN" "http://localhost:8000/uptime/heartbeat"
 # Flare (requires key)
 php artisan flare:test --errors
 
+# Watchtower (requires token)
+php artisan test --filter=WatchtowerIntegrationTest
+
 # Audit log migration
 php artisan migrate:status | grep activity_log
 ```
@@ -217,6 +275,8 @@ php artisan migrate:status | grep activity_log
 | `app/Console/Commands/SignalUptimeHeartbeat.php` | Scheduler signal |
 | `app/Jobs/RecordQueueHeartbeat.php` | Queue liveness |
 | `app/Http/Middleware/AttachFlareContext.php` | Per-request Flare context |
-| `bootstrap/app.php` | Flare exception handler, `/up` health |
+| `app/Support/WatchtowerReporter.php` | Watchtower HTTP client + noise filters |
+| `app/Providers/WatchtowerServiceProvider.php` | Queue + scheduled task failure hooks |
+| `bootstrap/app.php` | Flare + Watchtower exception hooks, `/up` health |
 | `routes/console.php` | Schedule definitions |
 | `routes/web.php` | Uptime routes |

docs/generate-test-catalog.php

@@ -34,6 +34,7 @@ $moduleMap = [
     'BackfillActivityLogTest' => ['Audit Log', 'Medium'],
     'ObservabilityAccessTest' => ['Observability', 'High'],
     'FlareIntegrationTest' => ['Observability', 'Medium'],
+    'WatchtowerIntegrationTest' => ['Observability', 'Medium'],
     'UptimeHeartbeatTest' => ['Observability', 'High'],
     'SiteTrafficTest' => ['Observability', 'Medium'],
     'LocalAvatarProviderTest' => ['UI', 'Low'],

tests/Feature/WatchtowerIntegrationTest.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace Tests\Feature;
+
+use App\Support\WatchtowerReporter;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Http;
+use RuntimeException;
+use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
+use Tests\TestCase;
+
+class WatchtowerIntegrationTest extends TestCase
+{
+    use RefreshDatabase;
+
+    public function test_watchtower_reporting_defaults_off_without_credentials(): void
+    {
+        config([
+            'services.watchtower.url' => null,
+            'services.watchtower.token' => null,
+        ]);
+
+        Http::fake();
+
+        app(WatchtowerReporter::class)->report(new RuntimeException('should not send'));
+
+        Http::assertNothingSent();
+    }
+
+    public function test_watchtower_should_report_filters_noise_like_flare(): void
+    {
+        $reporter = app(WatchtowerReporter::class);
+
+        $this->assertFalse($reporter->shouldReport(new NotFoundHttpException()));
+        $this->assertFalse($reporter->shouldReport(new \Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException()));
+        $this->assertTrue($reporter->shouldReport(new RuntimeException('server error')));
+    }
+
+    public function test_watchtower_reporter_posts_errors_when_configured(): void
+    {
+        Http::fake([
+            'https://log.skysyaz.my/api/errors' => Http::response(['ok' => true], 201),
+        ]);
+
+        config([
+            'services.watchtower.url' => 'https://log.skysyaz.my',
+            'services.watchtower.token' => 'test-watchtower-token',
+            'services.watchtower.app_name' => 'timesheet',
+        ]);
+
+        app(WatchtowerReporter::class)->report(new RuntimeException('Watchtower integration test'), [
+            'source' => 'test',
+        ]);
+
+        Http::assertSent(function ($request): bool {
+            return $request->url() === 'https://log.skysyaz.my/api/errors'
+                && $request->hasHeader('Authorization', 'Bearer test-watchtower-token')
+                && $request['app_name'] === 'timesheet'
+                && $request['message'] === 'Watchtower integration test'
+                && $request['context']['source'] === 'test'
+                && $request['context']['exception_class'] === RuntimeException::class;
+        });
+    }
+
+    public function test_watchtower_reporter_never_throws_when_watchtower_is_down(): void
+    {
+        Http::fake([
+            'https://log.skysyaz.my/api/errors' => Http::response(null, 503),
+        ]);
+
+        config([
+            'services.watchtower.url' => 'https://log.skysyaz.my',
+            'services.watchtower.token' => 'test-watchtower-token',
+        ]);
+
+        app(WatchtowerReporter::class)->report(new RuntimeException('outage test'));
+
+        $this->assertTrue(true);
+    }
+}