351 lines
15 KiB
TypeScript
351 lines
15 KiB
TypeScript
import type { FastifyInstance } from "fastify";
|
|
import { z } from "zod";
|
|
import crypto from "crypto";
|
|
import postgres from "postgres";
|
|
import { createDatabase, createDatabaseUser, deleteDatabase, deleteDatabaseUser } from "../services/do";
|
|
import { addCustomerToPool, removeCustomerFromPool } from "../services/pgbouncer";
|
|
import { addCustomerChart, removeCustomerChart, upgradeCustomerChart, upgradeAllCustomerCharts, getLatestChartVersion } from "../services/git";
|
|
import { setupCustomerDatabase, teardownCustomerDatabase } from "../services/db";
|
|
import { createNamespace, deleteNamespace, createSecret, createDockerRegistrySecret, patchSecret, getSecret, k8sFetch, syncArgoApp } from "../lib/k8s";
|
|
import { deleteSpacesObjects, getSpacesUsage } from "../services/spaces";
|
|
import { createCustomerDnsRecord, deleteCustomerDnsRecord, getCustomerDnsRecord, checkCustomerHealth } from "../services/cloudflare";
|
|
import { db } from "../db/manager";
|
|
import { config } from "../lib/config";
|
|
import { getCachedStatus, setCachedStatus } from "../lib/cache";
|
|
|
|
const MODULES = ["pos", "inventory", "rentals", "scheduling", "repairs", "accounting"] as const;
|
|
const PGBOUNCER_HOST = "pgbouncer.pgbouncer.svc";
|
|
const PGBOUNCER_PORT = 5432;
|
|
|
|
const ProvisionSchema = z.object({
|
|
slug: z.string().min(2).max(32).regex(/^[a-z0-9-]+$/, "lowercase letters, numbers, and hyphens only"),
|
|
name: z.string().min(1).max(128),
|
|
appVersion: z.string().default("*"),
|
|
modules: z.array(z.enum(MODULES)).default([]),
|
|
startDate: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).default(() => new Date().toISOString().slice(0, 10)),
|
|
expirationDate: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).nullable().default(null),
|
|
});
|
|
|
|
const ListQuerySchema = z.object({
|
|
page: z.coerce.number().int().min(1).default(1),
|
|
limit: z.coerce.number().int().min(1).max(100).default(25),
|
|
q: z.string().optional(),
|
|
sort: z.enum(["slug", "start_date", "expiration_date", "created_at", "updated_at"]).default("created_at"),
|
|
order: z.enum(["asc", "desc"]).default("desc"),
|
|
});
|
|
|
|
export async function customerRoutes(app: FastifyInstance) {
|
|
app.get("/customers", async (req, reply) => {
|
|
const query = ListQuerySchema.parse(req.query);
|
|
const offset = (query.page - 1) * query.limit;
|
|
const search = query.q ? `%${query.q}%` : null;
|
|
|
|
const [rows, [{ count }]] = await Promise.all([
|
|
search
|
|
? db`SELECT * FROM customers WHERE slug ILIKE ${search} OR name ILIKE ${search} ORDER BY ${db(query.sort)} ${db.unsafe(query.order)} LIMIT ${query.limit} OFFSET ${offset}`
|
|
: db`SELECT * FROM customers ORDER BY ${db(query.sort)} ${db.unsafe(query.order)} LIMIT ${query.limit} OFFSET ${offset}`,
|
|
search
|
|
? db`SELECT COUNT(*)::int FROM customers WHERE slug ILIKE ${search} OR name ILIKE ${search}`
|
|
: db`SELECT COUNT(*)::int FROM customers`,
|
|
]);
|
|
|
|
return reply.send({
|
|
data: rows,
|
|
pagination: {
|
|
page: query.page,
|
|
limit: query.limit,
|
|
total: count,
|
|
totalPages: Math.ceil(count / query.limit),
|
|
},
|
|
});
|
|
});
|
|
|
|
app.post("/customers", async (req, reply) => {
|
|
const body = ProvisionSchema.parse(req.body);
|
|
const slug = body.slug;
|
|
const namespace = `customer-${slug}`;
|
|
|
|
app.log.info({ slug }, "provisioning customer");
|
|
|
|
type StepStatus = "pending" | "done" | "failed";
|
|
const steps: Record<string, StepStatus> = {
|
|
database: "pending",
|
|
database_user: "pending",
|
|
database_setup: "pending",
|
|
pool: "pending",
|
|
namespace: "pending",
|
|
secrets: "pending",
|
|
storage: "pending",
|
|
dns: "pending",
|
|
chart: "pending",
|
|
};
|
|
|
|
const setStep = async (step: string, status: StepStatus) => {
|
|
steps[step] = status;
|
|
await db`UPDATE customers SET steps = ${db.json(steps)}, updated_at = NOW() WHERE slug = ${slug}`;
|
|
};
|
|
|
|
await db`
|
|
INSERT INTO customers (slug, name, modules, start_date, expiration_date, status, steps)
|
|
VALUES (${slug}, ${body.name}, ${body.modules}, ${body.startDate}, ${body.expirationDate}, 'provisioning', ${db.json(steps)})
|
|
ON CONFLICT (slug) DO UPDATE SET
|
|
name = EXCLUDED.name,
|
|
modules = EXCLUDED.modules,
|
|
start_date = EXCLUDED.start_date,
|
|
expiration_date = EXCLUDED.expiration_date,
|
|
status = 'provisioning',
|
|
steps = ${db.json(steps)},
|
|
updated_at = NOW()
|
|
`;
|
|
|
|
try {
|
|
const [, user] = await Promise.all([
|
|
createDatabase(slug).then(r => { setStep("database", "done"); return r; }),
|
|
createDatabaseUser(slug).then(r => { setStep("database_user", "done"); return r; }),
|
|
]);
|
|
|
|
await setupCustomerDatabase(slug, user.name);
|
|
await setStep("database_setup", "done");
|
|
|
|
await addCustomerToPool(slug, user.password);
|
|
await setStep("pool", "done");
|
|
|
|
// Create k8s namespace and secrets before pushing chart so ArgoCD can deploy immediately
|
|
await createNamespace(namespace);
|
|
await createDockerRegistrySecret(namespace, "registry-lunarfront", {
|
|
server: "registry.digitalocean.com",
|
|
username: "token",
|
|
password: config.doToken,
|
|
});
|
|
const jwtSecret = crypto.randomBytes(32).toString("hex");
|
|
const databaseUrl = `postgresql://${slug}:${user.password}@${PGBOUNCER_HOST}:${PGBOUNCER_PORT}/${slug}`;
|
|
await createSecret(namespace, "lunarfront-secrets", {
|
|
"database-url": databaseUrl,
|
|
"jwt-secret": jwtSecret,
|
|
"redis-url": config.managedValkeyUrl,
|
|
"redis-key-prefix": slug,
|
|
});
|
|
await setStep("namespace", "done");
|
|
await setStep("secrets", "done");
|
|
|
|
await patchSecret(namespace, "lunarfront-secrets", {
|
|
"spaces-key": config.spacesKey,
|
|
"spaces-secret": config.spacesSecret,
|
|
"spaces-bucket": config.spacesBucket,
|
|
"spaces-endpoint": `https://${config.spacesRegion}.digitaloceanspaces.com`,
|
|
"spaces-prefix": `${slug}/`,
|
|
});
|
|
await setStep("storage", "done");
|
|
|
|
await createCustomerDnsRecord(slug);
|
|
await setStep("dns", "done");
|
|
|
|
await addCustomerChart(slug, body.appVersion);
|
|
await setStep("chart", "done");
|
|
|
|
await db`UPDATE customers SET status = 'provisioned', updated_at = NOW() WHERE slug = ${slug}`;
|
|
} catch (err) {
|
|
const failedStep = Object.entries(steps).find(([, v]) => v === "pending")?.[0];
|
|
if (failedStep) await setStep(failedStep, "failed");
|
|
await db`UPDATE customers SET status = 'failed', updated_at = NOW() WHERE slug = ${slug}`;
|
|
app.log.error({ slug, err }, "provisioning failed");
|
|
return reply.code(500).send({ message: (err as Error).message ?? "Provisioning failed" });
|
|
}
|
|
|
|
app.log.info({ slug }, "customer provisioned");
|
|
return reply.code(201).send({ slug, status: "provisioned" });
|
|
});
|
|
|
|
app.delete("/customers/:slug", async (req, reply) => {
|
|
const { slug } = req.params as { slug: string };
|
|
const namespace = `customer-${slug}`;
|
|
|
|
app.log.info({ slug }, "deprovisioning customer");
|
|
|
|
removeCustomerChart(slug);
|
|
await removeCustomerFromPool(slug);
|
|
await teardownCustomerDatabase(slug, slug);
|
|
|
|
// Delete all objects under this customer's prefix in Spaces
|
|
try {
|
|
await deleteSpacesObjects(config.spacesKey, config.spacesSecret, config.spacesBucket, config.spacesRegion, `${slug}/`);
|
|
} catch {}
|
|
|
|
try {
|
|
await deleteCustomerDnsRecord(slug);
|
|
} catch {}
|
|
|
|
await Promise.all([
|
|
deleteDatabase(slug),
|
|
deleteDatabaseUser(slug),
|
|
deleteNamespace(namespace),
|
|
]);
|
|
|
|
await db`DELETE FROM customers WHERE slug = ${slug}`;
|
|
|
|
app.log.info({ slug }, "customer deprovisioned");
|
|
return reply.code(204).send();
|
|
});
|
|
|
|
// Live overview: cached status (pods + ArgoCD) + latest size snapshot + 30d history
|
|
app.get("/customers/:slug/overview", async (req, reply) => {
|
|
const { slug } = req.params as { slug: string };
|
|
const namespace = `customer-${slug}`;
|
|
const { refresh } = req.query as { refresh?: string };
|
|
|
|
const [customer] = await db`SELECT * FROM customers WHERE slug = ${slug}`;
|
|
if (!customer) return reply.code(404).send({ message: "Not found" });
|
|
|
|
// ── Status (Redis cache, 2min TTL) ────────────────────────────────────────
|
|
let statusEntry = refresh ? null : await getCachedStatus(slug);
|
|
if (!statusEntry) {
|
|
const [podsResult, argoResult] = await Promise.allSettled([
|
|
k8sFetch(`/api/v1/namespaces/${namespace}/pods`).then(r => r.json()),
|
|
k8sFetch(`/apis/argoproj.io/v1alpha1/namespaces/argocd/applications/customer-${slug}`).then(r => r.json()),
|
|
]);
|
|
|
|
const podsRaw = podsResult.status === "fulfilled" ? podsResult.value : null;
|
|
const argoRaw = argoResult.status === "fulfilled" ? argoResult.value : null;
|
|
|
|
const pods = (podsRaw?.items ?? []).map((pod: any) => ({
|
|
name: pod.metadata.name,
|
|
ready: (pod.status.containerStatuses ?? []).every((c: any) => c.ready),
|
|
readyCount: (pod.status.containerStatuses ?? []).filter((c: any) => c.ready).length,
|
|
totalCount: (pod.status.containerStatuses ?? []).length,
|
|
status: pod.status.phase ?? "Unknown",
|
|
restarts: (pod.status.containerStatuses ?? []).reduce((s: number, c: any) => s + (c.restartCount ?? 0), 0),
|
|
startedAt: pod.status.startTime ?? null,
|
|
}));
|
|
|
|
const argocd = argoRaw?.status ? {
|
|
syncStatus: argoRaw.status.sync?.status ?? "Unknown",
|
|
healthStatus: argoRaw.status.health?.status ?? "Unknown",
|
|
conditions: (argoRaw.status.conditions ?? []).map((c: any) => ({ type: c.type, message: c.message })),
|
|
} : null;
|
|
|
|
const liveStatus = { pods, argocd };
|
|
await setCachedStatus(slug, liveStatus);
|
|
statusEntry = { data: liveStatus, cachedAt: new Date().toISOString() };
|
|
}
|
|
|
|
// ── Infrastructure checks ─────────────────────────────────────────────────
|
|
const [dbCheck, pgbouncerCheck, sizeHistory, secrets, dnsCheck, healthCheck] = await Promise.allSettled([
|
|
// Try connecting to the customer DB directly
|
|
(async () => {
|
|
const sql = postgres(config.doadminDbUrl.replace(/\/([^/?]+)(\?|$)/, `/${slug}$2`), { max: 1, connect_timeout: 5 });
|
|
try {
|
|
await sql`SELECT 1`;
|
|
return true;
|
|
} finally {
|
|
await sql.end();
|
|
}
|
|
})(),
|
|
// Try connecting via pgbouncer
|
|
(async () => {
|
|
const sql = postgres(`postgresql://${slug}@${PGBOUNCER_HOST}:${PGBOUNCER_PORT}/${slug}`, { max: 1, connect_timeout: 5 });
|
|
try {
|
|
await sql`SELECT 1`;
|
|
return true;
|
|
} finally {
|
|
await sql.end();
|
|
}
|
|
})(),
|
|
db`
|
|
SELECT recorded_at, db_size_bytes, spaces_size_bytes, spaces_object_count
|
|
FROM customer_size_snapshots
|
|
WHERE slug = ${slug}
|
|
ORDER BY recorded_at DESC
|
|
LIMIT 30
|
|
`,
|
|
getSecret(namespace, "lunarfront-secrets").catch(() => null),
|
|
getCustomerDnsRecord(slug),
|
|
checkCustomerHealth(slug),
|
|
]);
|
|
|
|
const dbExists = dbCheck.status === "fulfilled" ? dbCheck.value : false;
|
|
const pgbouncerReachable = pgbouncerCheck.status === "fulfilled" ? pgbouncerCheck.value : false;
|
|
const secretData = secrets.status === "fulfilled" ? secrets.value : null;
|
|
const dns = dnsCheck.status === "fulfilled" ? dnsCheck.value : { exists: false, proxied: false, ip: null };
|
|
const health = healthCheck.status === "fulfilled" ? healthCheck.value : { reachable: false, status: null };
|
|
const infra = {
|
|
database: { exists: dbExists },
|
|
pgbouncer: { configured: pgbouncerReachable },
|
|
spaces: {
|
|
configured: !!(secretData?.["spaces-prefix"]),
|
|
bucket: secretData?.["spaces-bucket"] ?? null,
|
|
prefix: secretData?.["spaces-prefix"] ?? null,
|
|
},
|
|
dns,
|
|
health,
|
|
};
|
|
|
|
return reply.send({
|
|
customer,
|
|
status: { ...statusEntry.data, cachedAt: statusEntry.cachedAt },
|
|
infra,
|
|
sizeHistory: sizeHistory.status === "fulfilled" ? sizeHistory.value : [],
|
|
});
|
|
});
|
|
|
|
// Remove only the manager DB record without touching infrastructure —
|
|
// useful for cleaning up failed partial deployments
|
|
// Deactivate: remove ArgoCD chart (stops pods) but keep DB, namespace, secrets, and manager record
|
|
app.post("/customers/:slug/deactivate", async (req, reply) => {
|
|
const { slug } = req.params as { slug: string };
|
|
removeCustomerChart(slug);
|
|
await removeCustomerFromPool(slug);
|
|
await db`UPDATE customers SET status = 'inactive', updated_at = NOW() WHERE slug = ${slug}`;
|
|
app.log.info({ slug }, "customer deactivated");
|
|
return reply.code(200).send({ slug, status: "inactive" });
|
|
});
|
|
|
|
// Reactivate: push ArgoCD chart back, re-add to pool
|
|
app.post("/customers/:slug/reactivate", async (req, reply) => {
|
|
const { slug } = req.params as { slug: string };
|
|
const [customer] = await db`SELECT * FROM customers WHERE slug = ${slug}`;
|
|
if (!customer) return reply.code(404).send({ message: "Not found" });
|
|
|
|
// Re-add to pgbouncer — need the password from the k8s secret
|
|
const namespace = `customer-${slug}`;
|
|
const secrets = await getSecret(namespace, "lunarfront-secrets");
|
|
const dbUrl = new URL(secrets["database-url"]);
|
|
await addCustomerToPool(slug, dbUrl.password);
|
|
|
|
await addCustomerChart(slug, "*");
|
|
await db`UPDATE customers SET status = 'provisioned', updated_at = NOW() WHERE slug = ${slug}`;
|
|
app.log.info({ slug }, "customer reactivated");
|
|
return reply.code(200).send({ slug, status: "provisioned" });
|
|
});
|
|
|
|
app.post("/customers/:slug/upgrade", async (req, reply) => {
|
|
const { slug } = req.params as { slug: string };
|
|
const [customer] = await db`SELECT * FROM customers WHERE slug = ${slug}`;
|
|
if (!customer) return reply.code(404).send({ message: "Not found" });
|
|
const version = await getLatestChartVersion();
|
|
await upgradeCustomerChart(slug, version);
|
|
await syncArgoApp(`customer-${slug}`);
|
|
await db`UPDATE customers SET updated_at = NOW() WHERE slug = ${slug}`;
|
|
app.log.info({ slug, version }, "customer chart upgraded");
|
|
return reply.send({ slug, version });
|
|
});
|
|
|
|
app.post("/customers/upgrade-all", async (req, reply) => {
|
|
const customers = await db`SELECT slug FROM customers WHERE status = 'provisioned'`;
|
|
if (!customers.length) return reply.send({ upgraded: [], version: null });
|
|
const version = await getLatestChartVersion();
|
|
const slugs = customers.map((c: any) => c.slug);
|
|
await upgradeAllCustomerCharts(slugs, version);
|
|
await Promise.all(slugs.map((s: string) => syncArgoApp(`customer-${s}`)));
|
|
app.log.info({ slugs, version }, "all customers chart upgraded");
|
|
return reply.send({ upgraded: slugs, version });
|
|
});
|
|
|
|
// Remove only the manager DB record without touching infrastructure
|
|
app.delete("/customers/:slug/record", async (req, reply) => {
|
|
const { slug } = req.params as { slug: string };
|
|
await db`DELETE FROM customers WHERE slug = ${slug}`;
|
|
app.log.info({ slug }, "customer record removed");
|
|
return reply.code(204).send();
|
|
});
|
|
}
|