feat: implement circuit breaker, retry with backoff, and graceful degradation for Stellar calls

.github/workflows/ci.yml

@@ -45,7 +45,12 @@ jobs:
       REDIS_URL: redis://localhost:6379
       JWT_SECRET: test-secret-key-that-is-at-least-64-characters-long-for-testing-purposes-only
       STELLAR_NETWORK: testnet
-      SOROBAN_RPC_URL: https://soroban-testnet.stellar.org
+      STELLAR_HORIZON_URL: https://horizon-testnet.stellar.org
+      STELLAR_SOROBAN_RPC_URL: https://soroban-testnet.stellar.org
+      STELLAR_PLATFORM_SECRET: SCZANGBA5YHTNYVVVXKJQQXRG5UJUG7D3DV6VVON3JXKXJ3FPHN3A5J3
+      STELLAR_QUIZ_CONTRACT_ID: CB6Q2YKQQHH7GV7CU5RZDYM5S5OE2GABYLG5IY6YO5XLBAALBQKXYB53
+      STELLAR_REWARD_CONTRACT_ID: CBAKHFY4SIBRIVYH2Y2QDUIUZPGYGS4B26YBHC6RLV5QZ7OHH5FOF55T
+      STELLAR_CREDENTIAL_CONTRACT_ID: CD4ZJWLPGYLCYR7G5DZQ4EJWVMMF5VXU5Z2ECRSKGWV6GBV5S3F52K7
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4

drizzle.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "drizzle-kit";
+
+export default defineConfig({
+  schema: "./src/database/schema.ts",
+  out: "./src/database/migrations",
+  dialect: "postgresql",
+  dbCredentials: {
+    url: process.env.DATABASE_URL!,
+  },
+});

package.json

@@ -21,6 +21,7 @@
     "@fastify/jwt": "^9.0.2",
     "@fastify/rate-limit": "^10.2.0",
     "@stellar/stellar-sdk": "^13.3.0",
+    "cockatiel": "^4.0.0",
     "dotenv": "^16.4.7",
     "drizzle-orm": "^0.38.3",
     "fastify": "^5.2.1",

src/config/index.ts

@@ -33,9 +33,28 @@
 
 export type Env = z.infer<typeof envSchema>;
 
+let _config: Env | null = null;
+
 function loadConfig(): Env {
   const result = envSchema.safeParse(process.env);
   if (!result.success) {
+    if (process.env.NODE_ENV === "test") {
+      // In test mode, warn but don't exit — tests mock what they need
+      console.warn(
+        "Missing env vars in test mode (expected if mocking config):",
+        result.error.flatten().fieldErrors
+      );
+      return envSchema.parse({
+        DATABASE_URL: "postgresql://localhost:5432/test",
+        JWT_SECRET: "test-secret-key-that-is-at-least-64-characters-long",
+        STELLAR_HORIZON_URL: "https://horizon-testnet.stellar.org",
+        STELLAR_SOROBAN_RPC_URL: "https://soroban-testnet.stellar.org",
+        STELLAR_PLATFORM_SECRET: "test",
+        STELLAR_QUIZ_CONTRACT_ID: "test",
+        STELLAR_REWARD_CONTRACT_ID: "test",
+        STELLAR_CREDENTIAL_CONTRACT_ID: "test",
+      });
+    }
     console.error(
       "Invalid environment variables:",
       result.error.flatten().fieldErrors
@@ -45,4 +64,16 @@
   return result.data;
 }
 
-export const config = loadConfig();
+function ensureConfig(): Env {
+  if (!_config) {
+    _config = loadConfig();
+  }
+  return _config;
+}
+
+// Lazy config — loadConfig() only runs on first property access, not at import time
+export const config: Env = new Proxy({} as Env, {
+  get(_, prop) {
+    return (ensureConfig() as any)[prop];
+  },
+});

src/database/migrations/0002_add_reward_failed.sql

@@ -0,0 +1,2 @@
+-- Add reward_failed column to quiz_submissions to track permanently failed reward claims
+ALTER TABLE quiz_submissions ADD COLUMN IF NOT EXISTS reward_failed boolean NOT NULL DEFAULT false;

src/database/schema.ts

@@ -108,6 +108,7 @@ export const quizSubmissions = pgTable(
     score: integer("score"),
     feedback: text("feedback"),
     rewardClaimed: boolean("reward_claimed").notNull().default(false),
+    rewardFailed: boolean("reward_failed").notNull().default(false),
     txHash: varchar("tx_hash", { length: 64 }),
     submittedAt: timestamp("submitted_at", { withTimezone: true })
       .notNull()

src/modules/rewards/reward.service.ts

@@ -10,18 +10,81 @@ import { NotFoundError, ForbiddenError, ConflictError } from "../../utils/errors
 import { withLock } from "../../utils/lock.js";
 import { invokeContract } from "../../stellar/transactions.js";
 import { createQuizProof } from "../../stellar/signatures.js";
+import { isCircuitBreakerError } from "../../stellar/resilience.js";
 import { config } from "../../config/index.js";
 import { logger } from "../../utils/logger.js";
+import { enqueueReward } from "../../services/retry-queue.js";
 import StellarSdk from "@stellar/stellar-sdk";
 import type { RewardClaimResult, RewardHistoryItem } from "./reward.types.js";
 
 const REWARD_AMOUNT = 10; // credits per passed quiz
 
+/**
+ * Shared reward claim execution logic.
+ * Used by both the direct claim path and the background retry processor.
+ * Returns true if the claim succeeded, false if it should be retried.
+ */
+export async function processRewardClaim(
+  submissionId: string,
+  userId: string,
+  score: number
+): Promise<boolean> {
+  const [submission] = await db
+    .select()
+    .from(quizSubmissions)
+    .where(eq(quizSubmissions.id, submissionId));
+
+  if (!submission || submission.rewardClaimed) {
+    return true;
+  }
+
+  const [quiz] = await db
+    .select()
+    .from(quizzes)
+    .where(eq(quizzes.id, submission.quizId));
+
+  if (!quiz) return true;
+
+  const proof = createQuizProof(userId, submission.quizId, score);
+
+  const [user] = await db
+    .select()
+    .from(users)
+    .where(eq(users.id, userId));
+
+  if (!user) return true;
+
+  const txHash = await invokeContract(
+    config.STELLAR_REWARD_CONTRACT_ID,
+    "claim_reward",
+    [
+      StellarSdk.Address.fromString(user.stellarAddress).toScVal(),
+      StellarSdk.nativeToScVal(score, { type: "u32" }),
+      StellarSdk.nativeToScVal(Buffer.from(proof.signature, "base64")),
+    ]
+  );
+
+  await db
+    .update(quizSubmissions)
+    .set({ rewardClaimed: true, txHash })
+    .where(eq(quizSubmissions.id, submissionId));
+
+  await db
+    .update(users)
+    .set({
+      credits: sql`${users.credits} + ${REWARD_AMOUNT}`,
+    })
+    .where(eq(users.id, userId));
+
+  return true;
+}
+
 export class RewardService {
   /**
    * Claim a reward for a passed quiz submission.
    * Uses distributed locking + database transaction with row-level lock
    * to prevent double-spend from concurrent requests.
+   * Gracefully degrades when Stellar is unavailable by queuing the claim.
    */
   async claimReward(
     userId: string,
@@ -63,7 +126,7 @@ export class RewardService {
 
         const proof = createQuizProof(userId, submission.quizId, submission.score);
 
-        let txHash: string;
+        let txHash: string | null = null;
         try {
           const [user] = await tx
             .select()
@@ -85,6 +148,22 @@ export class RewardService {
           );
         } catch (err) {
           if (err instanceof NotFoundError) throw err;
+
+          if (isCircuitBreakerError(err)) {
+            logger.warn(
+              { submissionId },
+              "Stellar circuit breaker open — queuing reward for later"
+            );
+            await enqueueReward({ submissionId, userId, score: submission.score });
+            return {
+              submissionId,
+              amount: REWARD_AMOUNT,
+              txHash: null,
+              queued: true,
+              message: "Reward claim queued — Stellar is temporarily unavailable",
+            };
+          }
+
           logger.error({ err, submissionId }, "On-chain reward claim failed");
           throw new Error("Failed to process on-chain reward");
         }
@@ -110,6 +189,7 @@ export class RewardService {
           submissionId,
           amount: REWARD_AMOUNT,
           txHash,
+          queued: false,
           message: `Successfully claimed ${REWARD_AMOUNT} credits`,
         };
       });

src/modules/rewards/reward.types.ts

@@ -13,7 +13,8 @@ export type ClaimRewardBody = z.infer<typeof claimRewardSchema>;
 export interface RewardClaimResult {
   submissionId: string;
   amount: number;
-  txHash: string;
+  txHash: string | null;
+  queued: boolean;
   message: string;
 }
 

src/server.ts

@@ -2,10 +2,20 @@ import Fastify from "fastify";
 import cors from "@fastify/cors";
 import jwt from "@fastify/jwt";
 import rateLimit from "@fastify/rate-limit";
+import { sql } from "drizzle-orm";
 import { config } from "./config/index.js";
 import { logger } from "./utils/logger.js";
 import { registerErrorHandler } from "./middleware/error-handler.js";
 import { rateLimitOptions } from "./middleware/rate-limit.js";
+import { db } from "./config/database.js";
+import { redis } from "./config/redis.js";
+import { stellarClient } from "./stellar/client.js";
+import {
+  startRetryProcessor,
+  stopRetryProcessor,
+  type RetryJob,
+} from "./services/retry-queue.js";
+import { processRewardClaim } from "./modules/rewards/reward.service.js";
 
 // Route modules
 import { authRoutes } from "./modules/auth/auth.routes.js";
@@ -19,6 +29,22 @@ import { credentialRoutes } from "./modules/credentials/credential.routes.js";
 import { closeDatabase } from "./config/database.js";
 import { closeRedis } from "./config/redis.js";
 
+async function processRetryJob(job: RetryJob): Promise<boolean> {
+  try {
+    const success = await processRewardClaim(job.submissionId, job.userId, job.score);
+    if (success) {
+      logger.info(
+        { submissionId: job.submissionId },
+        "Queued reward processed successfully"
+      );
+    }
+    return success;
+  } catch (err) {
+    logger.error({ err, submissionId: job.submissionId }, "Retry job failed");
+    return false;
+  }
+}
+
 async function buildApp() {
   const app = Fastify({
     logger: {
@@ -49,11 +75,53 @@ async function buildApp() {
   registerErrorHandler(app);
 
   // ─── Health Check ───────────────────────────────────────────────────────
-  app.get("/health", async () => ({
-    status: "ok",
-    timestamp: new Date().toISOString(),
-    uptime: process.uptime(),
-  }));
+  app.get("/health", async (_request, reply) => {
+    const [dbCheck, redisCheck, stellarCheck] = await Promise.allSettled([
+      db.execute(sql`SELECT 1`),
+      redis.ping(),
+      stellarClient.getHorizonServer().root(),
+    ]);
+
+    const allHealthy = [dbCheck, redisCheck, stellarCheck].every(
+      (c) => c.status === "fulfilled"
+    );
+
+    const status = allHealthy ? "healthy" : "degraded";
+
+    return reply.status(allHealthy ? 200 : 503).send({
+      status,
+      timestamp: new Date().toISOString(),
+      uptime: process.uptime(),
+      checks: {
+        database: dbCheck.status === "fulfilled" ? "ok" : "error",
+        redis: redisCheck.status === "fulfilled" ? "ok" : "error",
+        stellar: stellarCheck.status === "fulfilled" ? "ok" : "error",
+      },
+    });
+  });
+
+  app.get("/health/live", async () => ({ status: "ok" }));
+
+  app.get("/health/ready", async (_request, reply) => {
+    const [dbCheck, redisCheck, stellarCheck] = await Promise.allSettled([
+      db.execute(sql`SELECT 1`),
+      redis.ping(),
+      stellarClient.getHorizonServer().root(),
+    ]);
+
+    const allHealthy = [dbCheck, redisCheck, stellarCheck].every(
+      (c) => c.status === "fulfilled"
+    );
+
+    return reply.status(allHealthy ? 200 : 503).send({
+      status: allHealthy ? "ready" : "not_ready",
+      checks: {
+        database: dbCheck.status === "fulfilled" ? "ok" : "error",
+        redis: redisCheck.status === "fulfilled" ? "ok" : "error",
+        stellar: stellarCheck.status === "fulfilled" ? "ok" : "error",
+      },
+    });
+  });
 
   // ─── API Routes ─────────────────────────────────────────────────────────
   await app.register(authRoutes, { prefix: "/api/auth" });
@@ -69,9 +137,11 @@ async function buildApp() {
 async function start() {
   const app = await buildApp();
 
-  // Graceful shutdown
+  startRetryProcessor(processRetryJob);
+
   const shutdown = async (signal: string) => {
     logger.info({ signal }, "Received shutdown signal");
+    stopRetryProcessor();
     await app.close();
     await closeDatabase();
     await closeRedis();
@@ -94,7 +164,8 @@ async function start() {
   }
 }
 
-// Allow importing the app for testing without starting the server
 export { buildApp };
 
-start();
+if (process.env.NODE_ENV !== "test") {
+  start();
+}