diff --git a/CHANGELOG.md b/CHANGELOG.md index c8ffe64..084aeff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ All notable changes to this project will be documented in this file. +## [1.3.0] - 2026-01-15 +### Added +- **Configurable retry strategies**: Choose between `exponential` (default), `linear`, or `fixed` backoff strategies +- **Retry configuration**: Set custom `delayMs` (base delay) and `maxDelayMs` (maximum delay cap) for retry attempts +- **Webhook notifications on retry**: Webhooks are now called on each retry attempt with detailed information (attemptNumber, retriesLeft, nextRetryAt) +- **Enhanced webhook payloads**: Improved webhook payloads for both retry and final failure events +### Improved +- Better retry error tracking with enhanced logging showing retry strategy in use +- More granular control over retry behavior per task +### Types +- Added `RetryStrategy` type: `'exponential' | 'linear' | 'fixed'` +- Added `RetryConfig` interface with `strategy`, `delayMs`, and `maxDelayMs` options +- Extended `ScheduleExtra` interface to include optional `retryConfig` + ## [1.2.0] - 2026-01-11 ### Performance - Enabled SQLite WAL by default to improve concurrent reads/writes. diff --git a/README.md b/README.md index e985eab..2d2241b 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,43 @@ **BlazerJob** is a lightweight, SQLite-backed task scheduler for Node.js and TypeScript applications. Use it as a library in your code to schedule, execute, and manage asynchronous tasks. +## Why BlazerJob vs BullMQ? + +BlazerJob and BullMQ are both task scheduling solutions, but with different philosophies: + +### BlazerJob Advantages + +| Feature | BlazerJob | BullMQ | +|---------|-----------|---------| +| **Infrastructure** | Zero external dependencies (SQLite) | Requires Redis server | +| **Setup complexity** | Single library, no setup | Redis installation & management required | +| **Deployment** | Single process, embedded DB | Separate Redis deployment | +| **Cost** | Free (no infrastructure) | Redis hosting costs | +| **Performance** | ~4.4k tasks/s (local NVMe) | Higher throughput with Redis | +| **Horizontal scaling** | Limited (single DB file) | Excellent (Redis cluster) | +| **Use case** | Small to medium workloads, embedded apps | Large-scale distributed systems | +| **Blockchain native** | Built-in Cosmos & HTTP connectors | Generic queue (requires custom code) | + +### When to choose BlazerJob: +- ✅ You want **zero infrastructure** overhead +- ✅ Simple deployment (single Node.js process) +- ✅ Small to medium workloads (<5k tasks/second) +- ✅ Embedded applications or scripts +- ✅ Blockchain tasks (Cosmos, HTTP APIs) +- ✅ Development & testing environments +- ✅ Cost-conscious projects + +### When to choose BullMQ: +- ✅ High-throughput requirements (>10k tasks/second) +- ✅ Distributed architecture across multiple servers +- ✅ Already using Redis in your stack +- ✅ Need Redis-specific features (streams, pub/sub) +- ✅ Enterprise-scale job processing + +**In summary:** BlazerJob prioritizes **simplicity and zero dependencies**, while BullMQ prioritizes **scalability and throughput**. Choose based on your infrastructure constraints and workload requirements. + +--- + # Supported Connectors BlazerJob currently supports only the following connectors for actual execution: diff --git a/package.json b/package.json index c10d06f..855d825 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "blazerjob", - "version": "1.2.0", + "version": "1.3.0", "description": "TypeScript library for scheduling, executing, and managing asynchronous tasks (custom, HTTP, Cosmos) with a SQLite backend.", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/src/index.ts b/src/index.ts index 03137b5..3244808 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,10 +11,19 @@ import { DirectSecp256k1HdWallet } from '@cosmjs/proto-signing'; import { makeCosmosTaskFn } from './cosmos/queries'; import { makeHttpTaskFn } from './http/queries'; +export type RetryStrategy = 'exponential' | 'linear' | 'fixed'; + +export interface RetryConfig { + strategy?: RetryStrategy; + delayMs?: number; + maxDelayMs?: number; +} + export interface ScheduleExtra { maxRuns?: number; maxDurationMs?: number; onEnd?: (stats: { runCount: number, errorCount: number }) => void; + retryConfig?: RetryConfig; } export type OnTaskEnd = (taskId: number, stats: { runCount: number, errorCount: number }) => void; @@ -30,8 +39,8 @@ export interface BlazeJobOptions { export class BlazeJob { private db: any; private timer?: NodeJS.Timeout; - // Map: taskId -> { runCount, startedAt, maxRuns, maxDurationMs } - private taskRunStats = new Map void, errorCount?: number }>(); + // Map: taskId -> { runCount, startedAt, maxRuns, maxDurationMs, retryConfig } + private taskRunStats = new Map void, errorCount?: number, retryConfig?: RetryConfig }>(); private taskErrorCount = new Map(); private periodicTaskCount = 0; private onAllTasksEndedCb?: OnAllTasksEnded; @@ -145,7 +154,7 @@ export class BlazeJob { try { config = JSON.parse(config); } catch (e) { - console.error('[BlazeJob][HTTP] Erreur de parsing config:', e, config); + console.error('[BlazeJob][HTTP] Config parsing error:', e, config); break; } } @@ -181,20 +190,137 @@ export class BlazeJob { try { this.stop(); } catch (e) { - console.error('[BlazeJob] Erreur lors de l\'arrêt du scheduler', e); + console.error('[BlazeJob] Error stopping scheduler', e); } try { if (this.db && typeof this.db.close === 'function') this.db.close(); } catch (e) { - console.error('[BlazeJob] Erreur lors de la fermeture de la base', e); + console.error('[BlazeJob] Error closing database', e); } - console.log('[BlazeJob] Toutes les tâches périodiques sont terminées. Arrêt automatique du process.'); + console.log('[BlazeJob] All periodic tasks completed. Auto-exiting process.'); process.exit(0); }, 200); } } } catch (err) { - console.error('[BlazeJob] Erreur lors de l\'exécution de la tâche', task.id, err); + console.error('[BlazeJob] Error executing task', task.id, err); + + // Mettre à jour le compteur d'erreurs + if (stat) { + stat.errorCount = (stat.errorCount || 0) + 1; + } + + // Sauvegarder l'erreur dans la base + const errorMessage = err instanceof Error ? err.message : String(err); + this.db.prepare(`UPDATE tasks SET lastError = ? WHERE id = ?`).run(errorMessage, task.id); + + // Décrémenter retriesLeft + const newRetriesLeft = Math.max(0, task.retriesLeft - 1); + + if (newRetriesLeft > 0) { + // Récupérer la configuration de retry + const retryConfig = stat?.retryConfig || {}; + const strategy = retryConfig.strategy || 'exponential'; + const baseDelayMs = retryConfig.delayMs || 1000; + const maxDelayMs = retryConfig.maxDelayMs || 60000; + const attemptNumber = (stat?.errorCount || 1); + + // Calculer le délai selon la stratégie + let backoffMs: number; + switch (strategy) { + case 'fixed': + backoffMs = baseDelayMs; + break; + case 'linear': + backoffMs = baseDelayMs * attemptNumber; + break; + case 'exponential': + default: + backoffMs = baseDelayMs * Math.pow(2, attemptNumber - 1); + break; + } + + // Limiter au délai maximum + backoffMs = Math.min(backoffMs, maxDelayMs); + const nextRunAt = new Date(Date.now() + backoffMs).toISOString(); + + console.log(`[BlazeJob] Task ${task.id} failed (attempt ${attemptNumber}). Retrying in ${backoffMs}ms (strategy: ${strategy}, retriesLeft: ${newRetriesLeft})`); + + // Replanifier la tâche + this.db.prepare(` + UPDATE tasks + SET status = 'pending', runAt = ?, retriesLeft = ? + WHERE id = ? + `).run(nextRunAt, newRetriesLeft, task.id); + + // Appeler le webhook si configuré (notification de retry) + if (task.webhookUrl) { + BlazeJob.sendWebhook(task.webhookUrl, { + taskId: task.id, + status: 'pending', + executedAt: new Date().toISOString(), + result: 'retry', + output: null, + error: errorMessage, + retriesLeft: newRetriesLeft, + nextRetryAt: nextRunAt, + attemptNumber + }); + } + } else { + // Plus de retries disponibles, marquer comme échouée + console.log(`[BlazeJob] Task ${task.id} permanently failed after ${stat?.errorCount || 1} attempt(s)`); + + this.db.prepare(` + UPDATE tasks + SET status = 'failed', executed_at = ?, retriesLeft = 0 + WHERE id = ? + `).run(new Date().toISOString(), task.id); + + // Appeler le webhook si configuré (notification d'échec définitif) + if (task.webhookUrl) { + BlazeJob.sendWebhook(task.webhookUrl, { + taskId: task.id, + status: 'failed', + executedAt: new Date().toISOString(), + result: 'error', + output: null, + error: errorMessage, + totalAttempts: stat?.errorCount || 1 + }); + } + + // Appeler le callback onEnd si présent + if (stat && stat.onEnd) { + stat.onEnd({ runCount: stat.runCount, errorCount: stat.errorCount || 0 }); + } + + // Nettoyer les stats et décrémenter le compteur de tâches périodiques + this.taskRunStats.delete(task.id); + this.periodicTaskCount--; + + // Vérifier si toutes les tâches sont terminées + if (this.periodicTaskCount === 0 && this.onAllTasksEndedCb) { + this.onAllTasksEndedCb(); + } + + if (this.periodicTaskCount === 0 && this.autoExit) { + setTimeout(() => { + try { + this.stop(); + } catch (e) { + console.error('[BlazeJob] Error stopping scheduler', e); + } + try { + if (this.db && typeof this.db.close === 'function') this.db.close(); + } catch (e) { + console.error('[BlazeJob] Error closing database', e); + } + console.log('[BlazeJob] All periodic tasks completed. Auto-exiting process.'); + process.exit(0); + }, 200); + } + } } finally { this.activeTasksCount--; } @@ -291,7 +417,8 @@ export class BlazeJob { webhookUrl, maxRuns, maxDurationMs, - onEnd + onEnd, + retryConfig } = options; const stmt = this.db.prepare(` INSERT INTO tasks (runAt, interval, priority, retriesLeft, type, config, webhookUrl) @@ -315,7 +442,8 @@ export class BlazeJob { maxRuns, maxDurationMs, onEnd, - errorCount: 0 + errorCount: 0, + retryConfig }); return taskId; } @@ -409,7 +537,7 @@ export async function stopServer() { await app.close(); jobs.stop(); jobs['db'].close(); - console.log('Serveur et scheduler arrêtés proprement.'); + console.log('Server and scheduler stopped gracefully.'); } // Optionnel : gestion du signal SIGTERM/SIGINT diff --git a/src/tests/test_retry_backoff.ts b/src/tests/test_retry_backoff.ts new file mode 100644 index 0000000..67a0f29 --- /dev/null +++ b/src/tests/test_retry_backoff.ts @@ -0,0 +1,69 @@ +import { BlazeJob } from '../index'; +import * as fs from 'fs'; + +const dbPath = 'test_retry_backoff.db'; + +// Nettoyer la base avant le test +if (fs.existsSync(dbPath)) { + fs.unlinkSync(dbPath); +} + +const jobs = new BlazeJob({ + dbPath, + autoExit: true, + concurrency: 1 +}); + +let attemptCount = 0; +const maxAttempts = 3; +const startTime = Date.now(); + +// Tâche qui échoue 2 fois, puis réussit +const taskId = jobs.schedule(async () => { + attemptCount++; + const elapsed = Date.now() - startTime; + + console.log(`[Test] Tentative ${attemptCount} à ${elapsed}ms`); + + if (attemptCount < maxAttempts) { + throw new Error(`Échec simulé - tentative ${attemptCount}/${maxAttempts}`); + } + + console.log(`[Test] Succès après ${attemptCount} tentatives !`); +}, { + runAt: new Date(), + retriesLeft: 3, + priority: 1, + type: 'custom', + onEnd: (stats: { runCount: number, errorCount: number }) => { + console.log('[Test] Statistiques finales:', stats); + console.log(`[Test] Nombre total de tentatives: ${attemptCount}`); + + // Vérifications + if (attemptCount === maxAttempts) { + console.log('✅ Test réussi : La tâche a été réessayée le bon nombre de fois'); + } else { + console.error(`❌ Test échoué : Attendu ${maxAttempts} tentatives, obtenu ${attemptCount}`); + } + + if (stats.errorCount === maxAttempts - 1) { + console.log('✅ Test réussi : Le compteur d\'erreurs est correct'); + } else { + console.error(`❌ Test échoué : Attendu ${maxAttempts - 1} erreurs, obtenu ${stats.errorCount}`); + } + + if (stats.runCount === 1) { + console.log('✅ Test réussi : La tâche a réussi une fois'); + } else { + console.error(`❌ Test échoué : Attendu 1 run réussi, obtenu ${stats.runCount}`); + } + } +}); + +console.log(`[Test] Tâche créée avec ID: ${taskId}`); +console.log('[Test] Configuration:'); +console.log(' - retriesLeft: 3'); +console.log(' - Backoff exponentiel attendu: 1s, 2s'); +console.log(' - Nombre de tentatives attendu: 3 (2 échecs + 1 succès)'); + +jobs.start(); diff --git a/src/tests/test_retry_strategies.ts b/src/tests/test_retry_strategies.ts new file mode 100644 index 0000000..f7a02c2 --- /dev/null +++ b/src/tests/test_retry_strategies.ts @@ -0,0 +1,89 @@ +import { BlazeJob } from '../index'; +import * as fs from 'fs'; + +const dbPath = 'test_retry_strategies.db'; + +if (fs.existsSync(dbPath)) { + fs.unlinkSync(dbPath); +} + +const jobs = new BlazeJob({ + dbPath, + autoExit: true, + concurrency: 3 +}); + +let attemptCountExponential = 0; +let attemptCountLinear = 0; +let attemptCountFixed = 0; + +console.log('=== Test des différentes stratégies de retry ===\n'); + +console.log('1. Test stratégie EXPONENTIAL (1s, 2s, 4s)'); +jobs.schedule(async () => { + attemptCountExponential++; + console.log(`[Exponential] Tentative ${attemptCountExponential} à ${Date.now()}`); + if (attemptCountExponential < 3) { + throw new Error('Échec simulé exponential'); + } + console.log('[Exponential] ✅ Succès après 3 tentatives'); +}, { + runAt: new Date(), + retriesLeft: 3, + type: 'custom', + retryConfig: { + strategy: 'exponential', + delayMs: 1000, + maxDelayMs: 30000 + }, + onEnd: (stats: { runCount: number, errorCount: number }) => { + console.log('[Exponential] Stats finales:', stats); + } +}); + +console.log('2. Test stratégie LINEAR (2s, 4s, 6s)'); +jobs.schedule(async () => { + attemptCountLinear++; + console.log(`[Linear] Tentative ${attemptCountLinear} à ${Date.now()}`); + if (attemptCountLinear < 3) { + throw new Error('Échec simulé linear'); + } + console.log('[Linear] ✅ Succès après 3 tentatives'); +}, { + runAt: new Date(), + retriesLeft: 3, + type: 'custom', + retryConfig: { + strategy: 'linear', + delayMs: 2000, + maxDelayMs: 30000 + }, + onEnd: (stats: { runCount: number, errorCount: number }) => { + console.log('[Linear] Stats finales:', stats); + } +}); + +console.log('3. Test stratégie FIXED (1.5s entre chaque retry)'); +jobs.schedule(async () => { + attemptCountFixed++; + console.log(`[Fixed] Tentative ${attemptCountFixed} à ${Date.now()}`); + if (attemptCountFixed < 3) { + throw new Error('Échec simulé fixed'); + } + console.log('[Fixed] ✅ Succès après 3 tentatives'); +}, { + runAt: new Date(), + retriesLeft: 3, + type: 'custom', + retryConfig: { + strategy: 'fixed', + delayMs: 1500, + maxDelayMs: 30000 + }, + onEnd: (stats: { runCount: number, errorCount: number }) => { + console.log('[Fixed] Stats finales:', stats); + } +}); + +console.log('\n=== Démarrage des tests ===\n'); +jobs.start(); diff --git a/src/tests/test_retry_webhook.ts b/src/tests/test_retry_webhook.ts new file mode 100644 index 0000000..8a55909 --- /dev/null +++ b/src/tests/test_retry_webhook.ts @@ -0,0 +1,76 @@ +import { BlazeJob } from '../index'; +import * as fs from 'fs'; +import Fastify from 'fastify'; + +const dbPath = 'test_retry_webhook.db'; + +if (fs.existsSync(dbPath)) { + fs.unlinkSync(dbPath); +} + +const webhookServer = Fastify({ logger: false }); +const webhookEvents: any[] = []; + +webhookServer.post('/webhook', async (request, reply) => { + const payload = request.body; + console.log('[Webhook reçu]', JSON.stringify(payload, null, 2)); + webhookEvents.push(payload); + reply.send({ received: true }); +}); + +(async () => { + await webhookServer.listen({ port: 9999 }); + console.log('Serveur webhook en écoute sur http://localhost:9999'); + + const jobs = new BlazeJob({ + dbPath, + autoExit: true, + concurrency: 1 + }); + + let attemptCount = 0; + + console.log('\n=== Test webhook sur retry ===\n'); + console.log('La tâche va échouer 2 fois avant de réussir'); + console.log('Vous devriez voir 3 webhooks :'); + console.log(' 1. result: "retry" (après 1ère erreur)'); + console.log(' 2. result: "retry" (après 2ème erreur)'); + console.log(' 3. result: "success" (succès final)\n'); + + jobs.schedule(async () => { + attemptCount++; + console.log(`[Task] Tentative ${attemptCount}`); + + if (attemptCount < 3) { + throw new Error(`Échec simulé - tentative ${attemptCount}`); + } + + console.log('[Task] ✅ Succès !'); + }, { + runAt: new Date(), + retriesLeft: 3, + type: 'custom', + webhookUrl: 'http://localhost:9999/webhook', + retryConfig: { + strategy: 'fixed', + delayMs: 500 + }, + onEnd: (stats: { runCount: number, errorCount: number }) => { + console.log('\n=== Résumé ==='); + console.log('Stats:', stats); + console.log('Nombre total de webhooks reçus:', webhookEvents.length); + + console.log('\nDétails des webhooks:'); + webhookEvents.forEach((event, index) => { + console.log(` ${index + 1}. result="${event.result}", status="${event.status}", error="${event.error || 'none'}"`); + }); + + setTimeout(async () => { + await webhookServer.close(); + console.log('\nServeur webhook arrêté'); + }, 500); + } + }); + + jobs.start(); +})();