also looking at this
Add BunPress guide and pre-E2E quality gates
#4
246 files
+1744
-64381
| @@ -1,497 +0,0 @@ | ||
| 1 | # Jobs & Queues | |
| 2 | ||
| 3 | Stacks provides a powerful job queue system for handling background tasks, built on [bun-queue](https://github.com/stacksjs/bun-queue). Process emails, notifications, reports, and other time-consuming tasks asynchronously. | |
| 4 | ||
| 5 | ## Overview | |
| 6 | ||
| 7 | The job system helps you: | |
| 8 | ||
| 9 | - **Defer work** - Process tasks in the background | |
| 10 | - **Handle failures** - Automatic retries with backoff | |
| 11 | - **Scale processing** - Run multiple workers | |
| 12 | - **Monitor progress** - Track job status and failures | |
| 13 | ||
| 14 | ## Quick Start | |
| 15 | ||
| 16 | ### Dispatching Jobs | |
| 17 | ||
| 18 | ```typescript | |
| 19 | import { dispatch } from '@stacksjs/queue' | |
| 20 | ||
| 21 | // Dispatch a job | |
| 22 | await dispatch('send-email', { | |
| 23 | to: 'user@example.com', | |
| 24 | subject: 'Welcome!', | |
| 25 | template: 'welcome', | |
| 26 | }) | |
| 27 | ||
| 28 | // Using job classes | |
| 29 | await job('SendWelcomeEmail').dispatch({ userId: 1 }) | |
| 30 | ``` | |
| 31 | ||
| 32 | ### Creating Jobs | |
| 33 | ||
| 34 | Create job files in `app/Jobs/`: | |
| 35 | ||
| 36 | ```typescript | |
| 37 | // app/Jobs/SendWelcomeEmail.ts | |
| 38 | import { Job } from '@stacksjs/queue' | |
| 39 | import { mail } from '@stacksjs/email' | |
| 40 | ||
| 41 | export default class SendWelcomeEmail extends Job { | |
| 42 | // Queue name | |
| 43 | queue = 'emails' | |
| 44 | ||
| 45 | // Retry attempts | |
| 46 | tries = 3 | |
| 47 | ||
| 48 | // Seconds between retries | |
| 49 | backoff = 60 | |
| 50 | ||
| 51 | // Maximum execution time (seconds) | |
| 52 | timeout = 120 | |
| 53 | ||
| 54 | async handle(data: { userId: number }) { | |
| 55 | const user = await User.find(data.userId) | |
| 56 | ||
| 57 | await mail.send({ | |
| 58 | to: user.email, | |
| 59 | subject: 'Welcome to our app!', | |
| 60 | template: 'emails/welcome', | |
| 61 | data: { name: user.name }, | |
| 62 | }) | |
| 63 | } | |
| 64 | ||
| 65 | // Called when all retries fail | |
| 66 | async failed(error: Error, data: { userId: number }) { | |
| 67 | console.error(`Failed to send welcome email to user ${data.userId}:`, error) | |
| 68 | await notifyAdmin('Welcome email failed', { userId: data.userId, error }) | |
| 69 | } | |
| 70 | } | |
| 71 | ``` | |
| 72 | ||
| 73 | ## Dispatch Methods | |
| 74 | ||
| 75 | ### Basic Dispatch | |
| 76 | ||
| 77 | ```typescript | |
| 78 | import { dispatch, dispatchSync } from '@stacksjs/queue' | |
| 79 | ||
| 80 | // Queue the job (async processing) | |
| 81 | await dispatch('process-order', { orderId: 123 }) | |
| 82 | ||
| 83 | // Execute immediately (sync) | |
| 84 | await dispatchSync('process-order', { orderId: 123 }) | |
| 85 | ``` | |
| 86 | ||
| 87 | ### Conditional Dispatch | |
| 88 | ||
| 89 | ```typescript | |
| 90 | import { dispatchIf, dispatchUnless } from '@stacksjs/queue' | |
| 91 | ||
| 92 | // Dispatch only if condition is true | |
| 93 | await dispatchIf( | |
| 94 | user.wantsNotifications, | |
| 95 | 'send-notification', | |
| 96 | { userId: user.id } | |
| 97 | ) | |
| 98 | ||
| 99 | // Dispatch unless condition is true | |
| 100 | await dispatchUnless( | |
| 101 | user.optedOut, | |
| 102 | 'send-marketing', | |
| 103 | { userId: user.id } | |
| 104 | ) | |
| 105 | ``` | |
| 106 | ||
| 107 | ### Delayed Dispatch | |
| 108 | ||
| 109 | ```typescript | |
| 110 | import { dispatchAfter } from '@stacksjs/queue' | |
| 111 | ||
| 112 | // Dispatch after 60 seconds | |
| 113 | await dispatchAfter(60, 'send-reminder', { userId: 1 }) | |
| 114 | ||
| 115 | // Dispatch at specific time | |
| 116 | await dispatchAfter( | |
| 117 | new Date('2024-12-25T00:00:00Z'), | |
| 118 | 'send-christmas-promo', | |
| 119 | { userId: 1 } | |
| 120 | ) | |
| 121 | ``` | |
| 122 | ||
| 123 | ### Job Chains | |
| 124 | ||
| 125 | Execute jobs in sequence: | |
| 126 | ||
| 127 | ```typescript | |
| 128 | import { chain } from '@stacksjs/queue' | |
| 129 | ||
| 130 | await chain() | |
| 131 | .add('validate-order', { orderId: 1 }) | |
| 132 | .add('process-payment', { orderId: 1 }) | |
| 133 | .add('send-confirmation', { orderId: 1 }) | |
| 134 | .add('notify-warehouse', { orderId: 1 }) | |
| 135 | .dispatch() | |
| 136 | ``` | |
| 137 | ||
| 138 | If any job fails, the chain stops. | |
| 139 | ||
| 140 | ### Batch Processing | |
| 141 | ||
| 142 | Process jobs in parallel: | |
| 143 | ||
| 144 | ```typescript | |
| 145 | import { batch } from '@stacksjs/queue' | |
| 146 | ||
| 147 | const result = await batch([ | |
| 148 | { job: 'process-user', data: { userId: 1 } }, | |
| 149 | { job: 'process-user', data: { userId: 2 } }, | |
| 150 | { job: 'process-user', data: { userId: 3 } }, | |
| 151 | ]) | |
| 152 | .allowFailures() // Continue even if some fail | |
| 153 | .onSuccess(() => console.log('All completed')) | |
| 154 | .onFailure((failed) => console.log(`${failed.length} failed`)) | |
| 155 | .dispatch() | |
| 156 | ||
| 157 | console.log(`Successful: ${result.successful}`) | |
| 158 | console.log(`Failed: ${result.failed}`) | |
| 159 | ``` | |
| 160 | ||
| 161 | ## Job Configuration | |
| 162 | ||
| 163 | ### Retry Settings | |
| 164 | ||
| 165 | ```typescript | |
| 166 | export default class UnreliableJob extends Job { | |
| 167 | // Number of retry attempts | |
| 168 | tries = 5 | |
| 169 | ||
| 170 | // Seconds between retries (fixed) | |
| 171 | backoff = 60 | |
| 172 | ||
| 173 | // Or exponential backoff (array of delays) | |
| 174 | backoff = [60, 300, 900, 3600] // 1min, 5min, 15min, 1hr | |
| 175 | ||
| 176 | // Or use backoff strategy | |
| 177 | backoffStrategy = 'exponential' // 'linear' | 'exponential' | |
| 178 | } | |
| 179 | ``` | |
| 180 | ||
| 181 | ### Timeouts | |
| 182 | ||
| 183 | ```typescript | |
| 184 | export default class LongRunningJob extends Job { | |
| 185 | // Maximum execution time in seconds | |
| 186 | timeout = 300 // 5 minutes | |
| 187 | ||
| 188 | async handle(data: any) { | |
| 189 | await veryLongProcess() | |
| 190 | } | |
| 191 | ||
| 192 | // Called if job times out | |
| 193 | async timedOut(data: any) { | |
| 194 | await cleanup(data) | |
| 195 | await notifyTimeout(data) | |
| 196 | } | |
| 197 | } | |
| 198 | ``` | |
| 199 | ||
| 200 | ### Unique Jobs | |
| 201 | ||
| 202 | Prevent duplicate jobs: | |
| 203 | ||
| 204 | ```typescript | |
| 205 | import { UniqueJobMiddleware } from '@stacksjs/queue' | |
| 206 | ||
| 207 | export default class ProcessOrder extends Job { | |
| 208 | middleware = [ | |
| 209 | new UniqueJobMiddleware({ | |
| 210 | key: (data) => `order:${data.orderId}`, | |
| 211 | ttl: 3600, // Unique for 1 hour | |
| 212 | }) | |
| 213 | ] | |
| 214 | } | |
| 215 | ``` | |
| 216 | ||
| 217 | ### Rate Limiting | |
| 218 | ||
| 219 | ```typescript | |
| 220 | import { RateLimitMiddleware } from '@stacksjs/queue' | |
| 221 | ||
| 222 | export default class SendNotification extends Job { | |
| 223 | middleware = [ | |
| 224 | new RateLimitMiddleware({ | |
| 225 | key: 'notifications', | |
| 226 | maxAttempts: 100, | |
| 227 | decayMinutes: 1, // 100 per minute | |
| 228 | }) | |
| 229 | ] | |
| 230 | } | |
| 231 | ``` | |
| 232 | ||
| 233 | ## Running Workers | |
| 234 | ||
| 235 | ### CLI Commands | |
| 236 | ||
| 237 | ```bash | |
| 238 | # Start queue worker | |
| 239 | buddy queue:work | |
| 240 | ||
| 241 | # Specify queue | |
| 242 | buddy queue:work --queue=emails | |
| 243 | ||
| 244 | # Multiple queues with priority (left = highest) | |
| 245 | buddy queue:work --queue=high,default,low | |
| 246 | ||
| 247 | # Limit jobs processed | |
| 248 | buddy queue:work --max-jobs=100 | |
| 249 | ||
| 250 | # Stop when empty | |
| 251 | buddy queue:work --stop-when-empty | |
| 252 | ``` | |
| 253 | ||
| 254 | ### Programmatic Workers | |
| 255 | ||
| 256 | ```typescript | |
| 257 | import { QueueWorker, WorkerManager } from '@stacksjs/queue' | |
| 258 | ||
| 259 | // Single worker | |
| 260 | const worker = new QueueWorker({ | |
| 261 | queues: ['default', 'emails'], | |
| 262 | concurrency: 5, | |
| 263 | maxJobs: 1000, | |
| 264 | }) | |
| 265 | ||
| 266 | await worker.start() | |
| 267 | ||
| 268 | // Multiple workers with manager | |
| 269 | const manager = new WorkerManager() | |
| 270 | manager.addWorker('default', { concurrency: 3 }) | |
| 271 | manager.addWorker('emails', { concurrency: 2 }) | |
| 272 | manager.addWorker('reports', { concurrency: 1 }) | |
| 273 | ||
| 274 | await manager.start() | |
| 275 | ||
| 276 | // Graceful shutdown | |
| 277 | process.on('SIGTERM', async () => { | |
| 278 | await manager.stop() | |
| 279 | process.exit(0) | |
| 280 | }) | |
| 281 | ``` | |
| 282 | ||
| 283 | ## Failed Jobs | |
| 284 | ||
| 285 | ### Handling Failures | |
| 286 | ||
| 287 | ```typescript | |
| 288 | export default class ImportData extends Job { | |
| 289 | tries = 3 | |
| 290 | ||
| 291 | async handle(data: { fileId: number }) { | |
| 292 | const file = await Storage.get(data.fileId) | |
| 293 | await processFile(file) | |
| 294 | } | |
| 295 | ||
| 296 | async failed(error: Error, data: { fileId: number }) { | |
| 297 | // Log the failure | |
| 298 | console.error('Import failed:', error) | |
| 299 | ||
| 300 | // Notify admins | |
| 301 | await slack.send({ | |
| 302 | channel: '#alerts', | |
| 303 | text: `Data import failed for file ${data.fileId}: ${error.message}`, | |
| 304 | }) | |
| 305 | ||
| 306 | // Mark file as failed | |
| 307 | await File.update(data.fileId, { status: 'failed' }) | |
| 308 | } | |
| 309 | } | |
| 310 | ``` | |
| 311 | ||
| 312 | ### Managing Failed Jobs | |
| 313 | ||
| 314 | ```typescript | |
| 315 | import { FailedJobManager, retryFailedJob, executeFailedJobs } from '@stacksjs/queue' | |
| 316 | ||
| 317 | const manager = new FailedJobManager() | |
| 318 | ||
| 319 | // List failed jobs | |
| 320 | const failedJobs = await manager.all() | |
| 321 | ||
| 322 | // Retry a specific job | |
| 323 | await retryFailedJob(failedJobId) | |
| 324 | ||
| 325 | // Retry all failed jobs | |
| 326 | await executeFailedJobs() | |
| 327 | ||
| 328 | // Delete a failed job | |
| 329 | await manager.delete(failedJobId) | |
| 330 | ||
| 331 | // Clear all failed jobs | |
| 332 | await manager.flush() | |
| 333 | ``` | |
| 334 | ||
| 335 | ### Failed Job Notifications | |
| 336 | ||
| 337 | ```typescript | |
| 338 | import { configureFailedJobNotifications } from '@stacksjs/queue' | |
| 339 | ||
| 340 | configureFailedJobNotifications({ | |
| 341 | channels: ['email', 'slack'], | |
| 342 | email: { | |
| 343 | to: 'admin@example.com', | |
| 344 | }, | |
| 345 | slack: { | |
| 346 | webhook: process.env.SLACK_WEBHOOK_URL, | |
| 347 | }, | |
| 348 | }) | |
| 349 | ``` | |
| 350 | ||
| 351 | ## Scheduling Jobs | |
| 352 | ||
| 353 | Schedule recurring jobs with cron expressions: | |
| 354 | ||
| 355 | ```typescript | |
| 356 | // app/Jobs/DailyReport.ts | |
| 357 | export default class DailyReport extends Job { | |
| 358 | // Run at 8am every day | |
| 359 | schedule = '0 8 _ _ *' | |
| 360 | ||
| 361 | async handle() { | |
| 362 | const report = await generateDailyReport() | |
| 363 | await sendReportEmail(report) | |
| 364 | } | |
| 365 | } | |
| 366 | ``` | |
| 367 | ||
| 368 | ### Starting the Scheduler | |
| 369 | ||
| 370 | ```bash | |
| 371 | buddy queue:schedule | |
| 372 | ``` | |
| 373 | ||
| 374 | Or programmatically: | |
| 375 | ||
| 376 | ```typescript | |
| 377 | import { startScheduler, stopScheduler, getSchedulerStatus } from '@stacksjs/queue' | |
| 378 | ||
| 379 | await startScheduler() | |
| 380 | ||
| 381 | // Check status | |
| 382 | const status = getSchedulerStatus() | |
| 383 | console.log(status.running) | |
| 384 | console.log(status.nextRunTimes) | |
| 385 | ||
| 386 | // Manually trigger a scheduled job | |
| 387 | await triggerJob('daily-report') | |
| 388 | ||
| 389 | // Stop scheduler | |
| 390 | await stopScheduler() | |
| 391 | ``` | |
| 392 | ||
| 393 | ## Queue Events | |
| 394 | ||
| 395 | Listen for job events: | |
| 396 | ||
| 397 | ```typescript | |
| 398 | import { onQueueEvent } from '@stacksjs/queue' | |
| 399 | ||
| 400 | onQueueEvent('job:dispatched', (payload) => { | |
| 401 | console.log(`Job dispatched: ${payload.jobName}`) | |
| 402 | }) | |
| 403 | ||
| 404 | onQueueEvent('job:started', (payload) => { | |
| 405 | console.log(`Job started: ${payload.jobId}`) | |
| 406 | }) | |
| 407 | ||
| 408 | onQueueEvent('job:completed', (payload) => { | |
| 409 | console.log(`Job completed: ${payload.jobId} in ${payload.duration}ms`) | |
| 410 | }) | |
| 411 | ||
| 412 | onQueueEvent('job:failed', (payload) => { | |
| 413 | console.error(`Job failed: ${payload.jobId}`, payload.error) | |
| 414 | }) | |
| 415 | ||
| 416 | onQueueEvent('job:retrying', (payload) => { | |
| 417 | console.log(`Retrying job: ${payload.jobId}, attempt ${payload.attempt}`) | |
| 418 | }) | |
| 419 | ``` | |
| 420 | ||
| 421 | ## Health Checks | |
| 422 | ||
| 423 | Monitor queue health: | |
| 424 | ||
| 425 | ```typescript | |
| 426 | import { checkQueueHealth, isQueueHealthy } from '@stacksjs/queue' | |
| 427 | ||
| 428 | // Detailed health check | |
| 429 | const health = await checkQueueHealth() | |
| 430 | console.log(health.status) // 'healthy' | 'degraded' | 'unhealthy' | |
| 431 | console.log(health.queues) // Per-queue status | |
| 432 | console.log(health.workers) // Worker status | |
| 433 | ||
| 434 | // Simple boolean check | |
| 435 | if (!await isQueueHealthy()) { | |
| 436 | await alertOps('Queue system unhealthy') | |
| 437 | } | |
| 438 | ``` | |
| 439 | ||
| 440 | ## Testing Jobs | |
| 441 | ||
| 442 | ```typescript | |
| 443 | import { describe, expect, it } from 'bun:test' | |
| 444 | import { fake, getFakeQueue, restore, runTestJob } from '@stacksjs/queue' | |
| 445 | ||
| 446 | describe('SendWelcomeEmail', () => { | |
| 447 | it('dispatches email job', async () => { | |
| 448 | // Fake the queue (jobs won't actually process) | |
| 449 | fake() | |
| 450 | ||
| 451 | // Dispatch job | |
| 452 | await dispatch('SendWelcomeEmail', { userId: 1 }) | |
| 453 | ||
| 454 | // Assert job was dispatched | |
| 455 | const fakeQueue = getFakeQueue() | |
| 456 | expect(fakeQueue.hasDispatched('SendWelcomeEmail')).toBe(true) | |
| 457 | ||
| 458 | // Check job data | |
| 459 | const jobs = fakeQueue.dispatched('SendWelcomeEmail') | |
| 460 | expect(jobs[0].data).toEqual({ userId: 1 }) | |
| 461 | ||
| 462 | // Restore real queue | |
| 463 | restore() | |
| 464 | }) | |
| 465 | ||
| 466 | it('handles job execution', async () => { | |
| 467 | // Actually run the job in test | |
| 468 | const result = await runTestJob('SendWelcomeEmail', { userId: 1 }) | |
| 469 | ||
| 470 | expect(result.success).toBe(true) | |
| 471 | }) | |
| 472 | }) | |
| 473 | ``` | |
| 474 | ||
| 475 | ## Best Practices | |
| 476 | ||
| 477 | ### DO | |
| 478 | ||
| 479 | - **Keep jobs small** - One responsibility per job | |
| 480 | - **Make jobs idempotent** - Safe to retry multiple times | |
| 481 | - **Set appropriate timeouts** - Prevent hung jobs | |
| 482 | - **Handle failures gracefully** - Implement `failed()` method | |
| 483 | - **Use unique constraints** - Prevent duplicate processing | |
| 484 | ||
| 485 | ### DON'T | |
| 486 | ||
| 487 | - **Don't process user input directly** - Validate in the web request | |
| 488 | - **Don't store sensitive data** - Use IDs and fetch fresh data | |
| 489 | - **Don't use too many retries** - 3-5 is usually sufficient | |
| 490 | - **Don't ignore failures** - Set up notifications | |
| 491 | ||
| 492 | ## Related Resources | |
| 493 | ||
| 494 | - **[Queue Package](/packages/queue)** - Full API reference | |
| 495 | - **[Scheduler Package](/packages/scheduler)** - Cron scheduling | |
| 496 | - **[Error Handling](/basics/error-handling)** - Handling job failures | |
| 497 | - **[Logging](/basics/logging)** - Logging job activity | |