feat: provision k8s namespace and secrets during customer setup

- k8s.ts: add createNamespace, deleteNamespace, createSecret, createDockerRegistrySecret
- customers.ts: create namespace + DOCR pull secret + app secrets (DATABASE_URL, JWT_SECRET, REDIS_URL) before pushing ArgoCD chart
- customers.ts: delete namespace on deprovision, search name field too
- git.ts: use DOCR OCI chart URL and helm parameters for customer ArgoCD apps
- Add 'namespace' and 'secrets' steps to provisioning step tracker

src/lib/k8s.ts

@@ -1,5 +1,4 @@
 import { readFileSync } from "fs";
-import { execSync } from "child_process";
 
 // Read in-cluster service account token and CA
 const SA_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token";
@@ -11,7 +10,7 @@ function token() {
   return readFileSync(SA_TOKEN_PATH, "utf-8").trim();
 }
 
-async function k8sFetch(path: string, options: RequestInit = {}) {
+async function k8sFetch(path: string, options: RequestInit = {}, allowStatuses: number[] = []) {
   const res = await fetch(`${K8S_API}${path}`, {
     ...options,
     headers: {
@@ -22,15 +21,16 @@ async function k8sFetch(path: string, options: RequestInit = {}) {
     // @ts-ignore - bun supports this
     tls: { rejectUnauthorized: false },
   });
-  if (!res.ok) {
+  if (!res.ok && !allowStatuses.includes(res.status)) {
     const body = await res.text();
     throw new Error(`k8s API ${options.method ?? "GET"} ${path} → ${res.status}: ${body}`);
   }
-  return res.json();
+  return res;
 }
 
 export async function getSecret(namespace: string, name: string): Promise<Record<string, string>> {
-  const secret = await k8sFetch(`/api/v1/namespaces/${namespace}/secrets/${name}`) as { data: Record<string, string> };
+  const res = await k8sFetch(`/api/v1/namespaces/${namespace}/secrets/${name}`);
+  const secret = await res.json() as { data: Record<string, string> };
   return Object.fromEntries(
     Object.entries(secret.data).map(([k, v]) => [k, Buffer.from(v, "base64").toString()])
   );
@@ -40,39 +40,85 @@ export async function patchSecret(namespace: string, name: string, data: Record<
   const encoded = Object.fromEntries(
     Object.entries(data).map(([k, v]) => [k, Buffer.from(v).toString("base64")])
   );
-  return k8sFetch(`/api/v1/namespaces/${namespace}/secrets/${name}`, {
+  return (await k8sFetch(`/api/v1/namespaces/${namespace}/secrets/${name}`, {
     method: "PATCH",
     headers: { "Content-Type": "application/strategic-merge-patch+json" },
     body: JSON.stringify({ data: encoded }),
+  })).json();
+}
+
+export async function createSecret(namespace: string, name: string, data: Record<string, string>) {
+  const encoded = Object.fromEntries(
+    Object.entries(data).map(([k, v]) => [k, Buffer.from(v).toString("base64")])
+  );
+  await k8sFetch(`/api/v1/namespaces/${namespace}/secrets`, {
+    method: "POST",
+    body: JSON.stringify({
+      apiVersion: "v1",
+      kind: "Secret",
+      metadata: { name, namespace },
+      data: encoded,
+    }),
+  }, [409]); // 409 = already exists, ignore
+}
+
+export async function createDockerRegistrySecret(
+  namespace: string,
+  name: string,
+  opts: { server: string; username: string; password: string },
+) {
+  const auth = Buffer.from(`${opts.username}:${opts.password}`).toString("base64");
+  const dockerConfig = JSON.stringify({
+    auths: { [opts.server]: { username: opts.username, password: opts.password, auth } },
   });
+  const encoded = Buffer.from(dockerConfig).toString("base64");
+  await k8sFetch(`/api/v1/namespaces/${namespace}/secrets`, {
+    method: "POST",
+    body: JSON.stringify({
+      apiVersion: "v1",
+      kind: "Secret",
+      metadata: { name, namespace },
+      type: "kubernetes.io/dockerconfigjson",
+      data: { ".dockerconfigjson": encoded },
+    }),
+  }, [409]);
+}
+
+export async function createNamespace(name: string) {
+  await k8sFetch(`/api/v1/namespaces`, {
+    method: "POST",
+    body: JSON.stringify({
+      apiVersion: "v1",
+      kind: "Namespace",
+      metadata: { name },
+    }),
+  }, [409]); // 409 = already exists, ignore
+}
+
+export async function deleteNamespace(name: string) {
+  await k8sFetch(`/api/v1/namespaces/${name}`, { method: "DELETE" }, [404]);
 }
 
 export async function patchConfigMap(namespace: string, name: string, data: Record<string, string>) {
-  return k8sFetch(`/api/v1/namespaces/${namespace}/configmaps/${name}`, {
+  return (await k8sFetch(`/api/v1/namespaces/${namespace}/configmaps/${name}`, {
     method: "PATCH",
     headers: { "Content-Type": "application/strategic-merge-patch+json" },
     body: JSON.stringify({ data }),
-  });
+  })).json();
 }
 
 export async function getConfigMap(namespace: string, name: string): Promise<Record<string, string>> {
-  const cm = await k8sFetch(`/api/v1/namespaces/${namespace}/configmaps/${name}`) as { data?: Record<string, string> };
+  const res = await k8sFetch(`/api/v1/namespaces/${namespace}/configmaps/${name}`);
+  const cm = await res.json() as { data?: Record<string, string> };
   return cm.data ?? {};
 }
 
-export async function createArgoCDApp(manifest: object) {
-  return k8sFetch(`/apis/argoproj.io/v1alpha1/namespaces/argocd/applications`, {
-    method: "POST",
-    body: JSON.stringify(manifest),
-  });
-}
-
 export async function rolloutRestart(namespace: string, deployment: string) {
-  return k8sFetch(`/apis/apps/v1/namespaces/${namespace}/deployments/${deployment}`, {
+  return (await k8sFetch(`/apis/apps/v1/namespaces/${namespace}/deployments/${deployment}`, {
     method: "PATCH",
     headers: { "Content-Type": "application/strategic-merge-patch+json" },
     body: JSON.stringify({
       spec: { template: { metadata: { annotations: { "kubectl.kubernetes.io/restartedAt": new Date().toISOString() } } } },
     }),
-  });
+  })).json();
 }

src/routes/customers.ts

@@ -1,12 +1,17 @@
 import type { FastifyInstance } from "fastify";
 import { z } from "zod";
+import crypto from "crypto";
 import { createDatabase, createDatabaseUser, deleteDatabase, deleteDatabaseUser } from "../services/do";
 import { addCustomerToPool, removeCustomerFromPool } from "../services/pgbouncer";
 import { addCustomerChart, removeCustomerChart } from "../services/git";
 import { setupCustomerDatabase, teardownCustomerDatabase } from "../services/db";
+import { createNamespace, deleteNamespace, createSecret, createDockerRegistrySecret } from "../lib/k8s";
 import { db } from "../db/manager";
+import { config } from "../lib/config";
 
 const MODULES = ["pos", "inventory", "rentals", "scheduling", "repairs", "accounting"] as const;
+const PGBOUNCER_HOST = "pgbouncer.pgbouncer.svc";
+const PGBOUNCER_PORT = 5432;
 
 const ProvisionSchema = z.object({
   slug: z.string().min(2).max(32).regex(/^[a-z0-9-]+$/, "lowercase letters, numbers, and hyphens only"),
@@ -33,10 +38,10 @@ export async function customerRoutes(app: FastifyInstance) {
 
     const [rows, [{ count }]] = await Promise.all([
       search
-        ? db`SELECT * FROM customers WHERE slug ILIKE ${search} ORDER BY ${db(query.sort)} ${db.unsafe(query.order)} LIMIT ${query.limit} OFFSET ${offset}`
+        ? db`SELECT * FROM customers WHERE slug ILIKE ${search} OR name ILIKE ${search} ORDER BY ${db(query.sort)} ${db.unsafe(query.order)} LIMIT ${query.limit} OFFSET ${offset}`
         : db`SELECT * FROM customers ORDER BY ${db(query.sort)} ${db.unsafe(query.order)} LIMIT ${query.limit} OFFSET ${offset}`,
       search
-        ? db`SELECT COUNT(*)::int FROM customers WHERE slug ILIKE ${search}`
+        ? db`SELECT COUNT(*)::int FROM customers WHERE slug ILIKE ${search} OR name ILIKE ${search}`
         : db`SELECT COUNT(*)::int FROM customers`,
     ]);
 
@@ -54,6 +59,7 @@ export async function customerRoutes(app: FastifyInstance) {
   app.post("/customers", async (req, reply) => {
     const body = ProvisionSchema.parse(req.body);
     const slug = body.slug;
+    const namespace = `customer-${slug}`;
 
     app.log.info({ slug }, "provisioning customer");
 
@@ -63,6 +69,8 @@ export async function customerRoutes(app: FastifyInstance) {
       database_user: "pending",
       database_setup: "pending",
       pool: "pending",
+      namespace: "pending",
+      secrets: "pending",
       chart: "pending",
     };
 
@@ -71,7 +79,6 @@ export async function customerRoutes(app: FastifyInstance) {
       await db`UPDATE customers SET steps = ${db.json(steps)}, updated_at = NOW() WHERE slug = ${slug}`;
     };
 
-    // Insert record immediately so partial failures are visible in the UI
     await db`
       INSERT INTO customers (slug, name, modules, start_date, expiration_date, status, steps)
       VALUES (${slug}, ${body.name}, ${body.modules}, ${body.startDate}, ${body.expirationDate}, 'provisioning', ${db.json(steps)})
@@ -97,6 +104,24 @@ export async function customerRoutes(app: FastifyInstance) {
       await addCustomerToPool(slug, user.password);
       await setStep("pool", "done");
 
+      // Create k8s namespace and secrets before pushing chart so ArgoCD can deploy immediately
+      await createNamespace(namespace);
+      await createDockerRegistrySecret(namespace, "registry-lunarfront", {
+        server: "registry.digitalocean.com",
+        username: "token",
+        password: config.doToken,
+      });
+      const jwtSecret = crypto.randomBytes(32).toString("hex");
+      const databaseUrl = `postgresql://${slug}:${user.password}@${PGBOUNCER_HOST}:${PGBOUNCER_PORT}/${slug}`;
+      const redisUrl = `redis://${namespace}-valkey:6379`;
+      await createSecret(namespace, "lunarfront-secrets", {
+        "database-url": databaseUrl,
+        "jwt-secret": jwtSecret,
+        "redis-url": redisUrl,
+      });
+      await setStep("namespace", "done");
+      await setStep("secrets", "done");
+
       addCustomerChart(slug, body.appVersion);
       await setStep("chart", "done");
 
@@ -115,6 +140,7 @@ export async function customerRoutes(app: FastifyInstance) {
 
   app.delete("/customers/:slug", async (req, reply) => {
     const { slug } = req.params as { slug: string };
+    const namespace = `customer-${slug}`;
 
     app.log.info({ slug }, "deprovisioning customer");
 
@@ -124,6 +150,7 @@ export async function customerRoutes(app: FastifyInstance) {
     await Promise.all([
       deleteDatabase(slug),
       deleteDatabaseUser(slug),
+      deleteNamespace(namespace),
     ]);
 
     await db`DELETE FROM customers WHERE slug = ${slug}`;

src/services/git.ts

@@ -54,16 +54,14 @@ metadata:
   namespace: argocd
 spec:
   project: default
-  sources:
+  source:
-    - repoURL: git.lunarfront.tech/ryan/lunarfront-app
+    repoURL: registry.digitalocean.com/lunarfront
-      chart: lunarfront
+    chart: lunarfront
-      targetRevision: "${appVersion}"
+    targetRevision: "${appVersion}"
-      helm:
+    helm:
-        valueFiles:
+      parameters:
-          - $values/customers/${slug}.yaml
+        - name: ingress.host
-    - repoURL: ssh://git@git-ssh.lunarfront.tech/ryan/lunarfront-charts.git
+          value: ${slug}.lunarfront.app
-      targetRevision: main
-      ref: values
   destination:
     server: https://kubernetes.default.svc
     namespace: customer-${slug}