import type { FastifyInstance } from "fastify"; import { z } from "zod"; import crypto from "crypto"; import postgres from "postgres"; import { createDatabase, createDatabaseUser, deleteDatabase, deleteDatabaseUser } from "../services/do"; import { addCustomerToPool, removeCustomerFromPool } from "../services/pgbouncer"; import { addCustomerChart, removeCustomerChart, upgradeCustomerChart, upgradeAllCustomerCharts, getLatestChartVersion } from "../services/git"; import { setupCustomerDatabase, teardownCustomerDatabase } from "../services/db"; import { createNamespace, deleteNamespace, createSecret, createDockerRegistrySecret, patchSecret, getSecret, k8sFetch, syncArgoApp } from "../lib/k8s"; import { deleteSpacesObjects, getSpacesUsage } from "../services/spaces"; import { createCustomerDnsRecord, deleteCustomerDnsRecord, getCustomerDnsRecord, checkCustomerHealth } from "../services/cloudflare"; import { db } from "../db/manager"; import { config } from "../lib/config"; import { getCachedStatus, setCachedStatus } from "../lib/cache"; const MODULES = ["pos", "inventory", "rentals", "scheduling", "repairs", "accounting"] as const; const PGBOUNCER_HOST = "pgbouncer.pgbouncer.svc"; const PGBOUNCER_PORT = 5432; const ProvisionSchema = z.object({ slug: z.string().min(2).max(32).regex(/^[a-z0-9-]+$/, "lowercase letters, numbers, and hyphens only"), name: z.string().min(1).max(128), appVersion: z.string().default("*"), modules: z.array(z.enum(MODULES)).default([]), startDate: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).default(() => new Date().toISOString().slice(0, 10)), expirationDate: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).nullable().default(null), }); const ListQuerySchema = z.object({ page: z.coerce.number().int().min(1).default(1), limit: z.coerce.number().int().min(1).max(100).default(25), q: z.string().optional(), sort: z.enum(["slug", "start_date", "expiration_date", "created_at", "updated_at"]).default("created_at"), order: z.enum(["asc", "desc"]).default("desc"), }); export async function customerRoutes(app: FastifyInstance) { app.get("/customers", async (req, reply) => { const query = ListQuerySchema.parse(req.query); const offset = (query.page - 1) * query.limit; const search = query.q ? `%${query.q}%` : null; const [rows, [{ count }]] = await Promise.all([ search ? db`SELECT * FROM customers WHERE slug ILIKE ${search} OR name ILIKE ${search} ORDER BY ${db(query.sort)} ${db.unsafe(query.order)} LIMIT ${query.limit} OFFSET ${offset}` : db`SELECT * FROM customers ORDER BY ${db(query.sort)} ${db.unsafe(query.order)} LIMIT ${query.limit} OFFSET ${offset}`, search ? db`SELECT COUNT(*)::int FROM customers WHERE slug ILIKE ${search} OR name ILIKE ${search}` : db`SELECT COUNT(*)::int FROM customers`, ]); return reply.send({ data: rows, pagination: { page: query.page, limit: query.limit, total: count, totalPages: Math.ceil(count / query.limit), }, }); }); app.post("/customers", async (req, reply) => { const body = ProvisionSchema.parse(req.body); const slug = body.slug; const namespace = `customer-${slug}`; app.log.info({ slug }, "provisioning customer"); type StepStatus = "pending" | "done" | "failed"; const steps: Record = { database: "pending", database_user: "pending", database_setup: "pending", pool: "pending", namespace: "pending", secrets: "pending", storage: "pending", dns: "pending", chart: "pending", }; const setStep = async (step: string, status: StepStatus) => { steps[step] = status; await db`UPDATE customers SET steps = ${db.json(steps)}, updated_at = NOW() WHERE slug = ${slug}`; }; await db` INSERT INTO customers (slug, name, modules, start_date, expiration_date, status, steps) VALUES (${slug}, ${body.name}, ${body.modules}, ${body.startDate}, ${body.expirationDate}, 'provisioning', ${db.json(steps)}) ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, modules = EXCLUDED.modules, start_date = EXCLUDED.start_date, expiration_date = EXCLUDED.expiration_date, status = 'provisioning', steps = ${db.json(steps)}, updated_at = NOW() `; try { const [, user] = await Promise.all([ createDatabase(slug).then(r => { setStep("database", "done"); return r; }), createDatabaseUser(slug).then(r => { setStep("database_user", "done"); return r; }), ]); await setupCustomerDatabase(slug, user.name); await setStep("database_setup", "done"); await addCustomerToPool(slug, user.password); await setStep("pool", "done"); // Create k8s namespace and secrets before pushing chart so ArgoCD can deploy immediately await createNamespace(namespace); await createDockerRegistrySecret(namespace, "registry-lunarfront", { server: "registry.digitalocean.com", username: "token", password: config.doToken, }); const jwtSecret = crypto.randomBytes(32).toString("hex"); const databaseUrl = `postgresql://${slug}:${user.password}@${PGBOUNCER_HOST}:${PGBOUNCER_PORT}/${slug}`; await createSecret(namespace, "lunarfront-secrets", { "database-url": databaseUrl, "jwt-secret": jwtSecret, "redis-url": config.managedValkeyUrl, "redis-key-prefix": slug, }); await setStep("namespace", "done"); await setStep("secrets", "done"); await patchSecret(namespace, "lunarfront-secrets", { "spaces-key": config.spacesKey, "spaces-secret": config.spacesSecret, "spaces-bucket": config.spacesBucket, "spaces-endpoint": `https://${config.spacesRegion}.digitaloceanspaces.com`, "spaces-prefix": `${slug}/`, }); await setStep("storage", "done"); await createCustomerDnsRecord(slug); await setStep("dns", "done"); await addCustomerChart(slug, body.appVersion); await setStep("chart", "done"); await db`UPDATE customers SET status = 'provisioned', updated_at = NOW() WHERE slug = ${slug}`; } catch (err) { const failedStep = Object.entries(steps).find(([, v]) => v === "pending")?.[0]; if (failedStep) await setStep(failedStep, "failed"); await db`UPDATE customers SET status = 'failed', updated_at = NOW() WHERE slug = ${slug}`; app.log.error({ slug, err }, "provisioning failed"); return reply.code(500).send({ message: (err as Error).message ?? "Provisioning failed" }); } app.log.info({ slug }, "customer provisioned"); return reply.code(201).send({ slug, status: "provisioned" }); }); app.delete("/customers/:slug", async (req, reply) => { const { slug } = req.params as { slug: string }; const namespace = `customer-${slug}`; app.log.info({ slug }, "deprovisioning customer"); removeCustomerChart(slug); await removeCustomerFromPool(slug); await teardownCustomerDatabase(slug, slug); // Delete all objects under this customer's prefix in Spaces try { await deleteSpacesObjects(config.spacesKey, config.spacesSecret, config.spacesBucket, config.spacesRegion, `${slug}/`); } catch {} try { await deleteCustomerDnsRecord(slug); } catch {} await Promise.all([ deleteDatabase(slug), deleteDatabaseUser(slug), deleteNamespace(namespace), ]); await db`DELETE FROM customers WHERE slug = ${slug}`; app.log.info({ slug }, "customer deprovisioned"); return reply.code(204).send(); }); // Live overview: cached status (pods + ArgoCD) + latest size snapshot + 30d history app.get("/customers/:slug/overview", async (req, reply) => { const { slug } = req.params as { slug: string }; const namespace = `customer-${slug}`; const { refresh } = req.query as { refresh?: string }; const [customer] = await db`SELECT * FROM customers WHERE slug = ${slug}`; if (!customer) return reply.code(404).send({ message: "Not found" }); // ── Status (Redis cache, 2min TTL) ──────────────────────────────────────── let statusEntry = refresh ? null : await getCachedStatus(slug); if (!statusEntry) { const [podsResult, argoResult] = await Promise.allSettled([ k8sFetch(`/api/v1/namespaces/${namespace}/pods`).then(r => r.json()), k8sFetch(`/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/customer-${slug}`).then(r => r.json()), ]); const podsRaw = podsResult.status === "fulfilled" ? podsResult.value : null; const argoRaw = argoResult.status === "fulfilled" ? argoResult.value : null; const pods = (podsRaw?.items ?? []).map((pod: any) => ({ name: pod.metadata.name, ready: (pod.status.containerStatuses ?? []).every((c: any) => c.ready), readyCount: (pod.status.containerStatuses ?? []).filter((c: any) => c.ready).length, totalCount: (pod.status.containerStatuses ?? []).length, status: pod.status.phase ?? "Unknown", restarts: (pod.status.containerStatuses ?? []).reduce((s: number, c: any) => s + (c.restartCount ?? 0), 0), startedAt: pod.status.startTime ?? null, })); const argocd = argoRaw?.status ? { syncStatus: argoRaw.status.sync?.status ?? "Unknown", healthStatus: argoRaw.status.health?.status ?? "Unknown", conditions: (argoRaw.status.conditions ?? []).map((c: any) => ({ type: c.type, message: c.message })), } : null; const liveStatus = { pods, argocd }; await setCachedStatus(slug, liveStatus); statusEntry = { data: liveStatus, cachedAt: new Date().toISOString() }; } // ── Infrastructure checks ───────────────────────────────────────────────── const [dbCheck, pgbouncerCheck, sizeHistory, secrets, dnsCheck, healthCheck] = await Promise.allSettled([ // Try connecting to the customer DB directly (async () => { const sql = postgres(config.doadminDbUrl.replace(/\/([^/?]+)(\?|$)/, `/${slug}$2`), { max: 1, connect_timeout: 5 }); try { await sql`SELECT 1`; return true; } finally { await sql.end(); } })(), // Try connecting via pgbouncer (async () => { const sql = postgres(`postgresql://${slug}@${PGBOUNCER_HOST}:${PGBOUNCER_PORT}/${slug}`, { max: 1, connect_timeout: 5 }); try { await sql`SELECT 1`; return true; } finally { await sql.end(); } })(), db` SELECT recorded_at, db_size_bytes, spaces_size_bytes, spaces_object_count FROM customer_size_snapshots WHERE slug = ${slug} ORDER BY recorded_at DESC LIMIT 30 `, getSecret(namespace, "lunarfront-secrets").catch(() => null), getCustomerDnsRecord(slug), checkCustomerHealth(slug), ]); const dbExists = dbCheck.status === "fulfilled" ? dbCheck.value : false; const pgbouncerReachable = pgbouncerCheck.status === "fulfilled" ? pgbouncerCheck.value : false; const secretData = secrets.status === "fulfilled" ? secrets.value : null; const dns = dnsCheck.status === "fulfilled" ? dnsCheck.value : { exists: false, proxied: false, ip: null }; const health = healthCheck.status === "fulfilled" ? healthCheck.value : { reachable: false, status: null }; const infra = { database: { exists: dbExists }, pgbouncer: { configured: pgbouncerReachable }, spaces: { configured: !!(secretData?.["spaces-prefix"]), bucket: secretData?.["spaces-bucket"] ?? null, prefix: secretData?.["spaces-prefix"] ?? null, }, dns, health, }; return reply.send({ customer, status: { ...statusEntry.data, cachedAt: statusEntry.cachedAt }, infra, sizeHistory: sizeHistory.status === "fulfilled" ? sizeHistory.value : [], }); }); // Remove only the manager DB record without touching infrastructure — // useful for cleaning up failed partial deployments // Deactivate: remove ArgoCD chart (stops pods) but keep DB, namespace, secrets, and manager record app.post("/customers/:slug/deactivate", async (req, reply) => { const { slug } = req.params as { slug: string }; removeCustomerChart(slug); await removeCustomerFromPool(slug); await db`UPDATE customers SET status = 'inactive', updated_at = NOW() WHERE slug = ${slug}`; app.log.info({ slug }, "customer deactivated"); return reply.code(200).send({ slug, status: "inactive" }); }); // Reactivate: push ArgoCD chart back, re-add to pool app.post("/customers/:slug/reactivate", async (req, reply) => { const { slug } = req.params as { slug: string }; const [customer] = await db`SELECT * FROM customers WHERE slug = ${slug}`; if (!customer) return reply.code(404).send({ message: "Not found" }); // Re-add to pgbouncer — need the password from the k8s secret const namespace = `customer-${slug}`; const secrets = await getSecret(namespace, "lunarfront-secrets"); const dbUrl = new URL(secrets["database-url"]); await addCustomerToPool(slug, dbUrl.password); await addCustomerChart(slug, "*"); await db`UPDATE customers SET status = 'provisioned', updated_at = NOW() WHERE slug = ${slug}`; app.log.info({ slug }, "customer reactivated"); return reply.code(200).send({ slug, status: "provisioned" }); }); app.post("/customers/:slug/upgrade", async (req, reply) => { const { slug } = req.params as { slug: string }; const [customer] = await db`SELECT * FROM customers WHERE slug = ${slug}`; if (!customer) return reply.code(404).send({ message: "Not found" }); const version = await getLatestChartVersion(); await upgradeCustomerChart(slug, version); await syncArgoApp(`customer-${slug}`); await db`UPDATE customers SET updated_at = NOW() WHERE slug = ${slug}`; app.log.info({ slug, version }, "customer chart upgraded"); return reply.send({ slug, version }); }); app.post("/customers/upgrade-all", async (req, reply) => { const customers = await db`SELECT slug FROM customers WHERE status = 'provisioned'`; if (!customers.length) return reply.send({ upgraded: [], version: null }); const version = await getLatestChartVersion(); const slugs = customers.map((c: any) => c.slug); await upgradeAllCustomerCharts(slugs, version); await Promise.all(slugs.map((s: string) => syncArgoApp(`customer-${s}`))); app.log.info({ slugs, version }, "all customers chart upgraded"); return reply.send({ upgraded: slugs, version }); }); // Remove only the manager DB record without touching infrastructure app.delete("/customers/:slug/record", async (req, reply) => { const { slug } = req.params as { slug: string }; await db`DELETE FROM customers WHERE slug = ${slug}`; app.log.info({ slug }, "customer record removed"); return reply.code(204).send(); }); }