From 8159b870747288f31ad5d098471492e0fe1b274c Mon Sep 17 00:00:00 2001 From: Joel Brock Date: Fri, 5 Jun 2026 07:11:18 -0700 Subject: [PATCH] File upload pipeline: spike + endpoint skeleton (Phase 1, in progress) Lays groundwork for closing the file-upload gap discovered while wiring the org-contact custom fields. Currently no file fields in the form actually persist to CiviCRM -- the renderer FileList drops at the onSubmit JSON.stringify, and there is no /api/upload route or Attachment.create call anywhere. This commit adds: 1. scripts/spike-attachment-upload.mjs One-off spike to answer the open question that gates the rest of the work: does APIv4 Attachment.create accept an unbound upload, or must we attach to an entity at create time? If unbound works we can use the planned two-step pattern (upload returns a file id; submit references it). If not, activity-bound file fields need a different flow because the activity does not exist yet at upload time. The spike also exercises the Contact.update + .file_name read-back path against the Certificate_of_Incorporation field on a real org contact, then cleans up after itself. Usage: node --env-file=.env.local scripts/spike-attachment-upload.mjs \ --org-id= [--keep] 2. app/api/upload/route.ts Structural pieces that do not depend on the spike outcome: - multipart parsing via Request.formData() - 5 MB hard cap (under Amplify Lambda 6 MB sync payload limit) - MIME allowlist (PDF, DOC/DOCX, XLS/XLSX, JPEG/PNG/GIF/WEBP) - magic-byte sniff to cross-check the client-reported MIME - filename sanitization (path traversal scrub, length cap) - checksum verification, rate limiting, field-ref allowlist - STUB-mode short-circuit for local dev without live Civi - explicit 501 where the Civi Attachment.create wiring goes, with a comment pointing at the spike that resolves it Result: endpoint compiles, registers as a Next route, returns 501 with a clear message; build passes; nothing wired into the frontend yet so the existing form is unaffected. Phase 2 (renderer upload-on-pick), Phase 3 (submit reshape), Phase 4 (promote Certificate_of_Incorporation to editable) follow once the spike output picks the Attachment.create variant. --- app/api/upload/route.ts | 233 ++++++++++++++++++++++++ scripts/spike-attachment-upload.mjs | 267 ++++++++++++++++++++++++++++ 2 files changed, 500 insertions(+) create mode 100644 app/api/upload/route.ts create mode 100644 scripts/spike-attachment-upload.mjs diff --git a/app/api/upload/route.ts b/app/api/upload/route.ts new file mode 100644 index 0000000..f8bc515 --- /dev/null +++ b/app/api/upload/route.ts @@ -0,0 +1,233 @@ +/** + * POST /api/upload + * + * Multipart endpoint that accepts a single file plus the form's auth pair + * (cid + cs) and the target Civi field reference. Verifies, validates, + * stores in CiviCRM via Attachment.create, returns { id, file_name }. + * + * The form's file renderer calls this on file-pick (not at submit time) + * so submit can stay a simple JSON POST. The returned { id, file_name } + * is what gets put into RHF state and ultimately submitted as the field + * value — the same shape /api/data uses for prefill, so the renderer's + * FilePriorIndicator works unchanged for fresh uploads too. + * + * Request: multipart/form-data with parts: + * - file the binary + * - cid contact id (form auth) + * - cs checksum (form auth) + * - fieldRef the Civi field reference, e.g. "Stage_1.Vision_Upload" + * or "Food_Co_op_Organizing.Certificate_of_Incorporation" + * + * Response: { id: number, file_name: string } OR { error: string } + * + * STUB MODE: if CiviCRM env vars are unset, returns a fake id + the + * uploaded filename so frontend dev works without a live CRM. + */ + +import { NextResponse } from "next/server"; +import { verifyChecksum } from "@/lib/civicrm"; +import { allFields } from "@/config/form"; +import { rateLimit, clientIp } from "@/lib/rate-limit"; + +// 5 MB hard cap. Sits under AWS Amplify Lambda's 6 MB sync invocation +// payload limit with headroom for multipart envelope overhead. +const MAX_BYTES = 5 * 1024 * 1024; + +// Per the v1 spec: PDF, DOC/DOCX, XLS/XLSX, common image types. +const ALLOWED_MIME = new Set([ + "application/pdf", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "image/jpeg", + "image/png", + "image/gif", + "image/webp", +]); + +// Magic-byte sniff for the most common forgeries. Don't trust client- +// reported MIME alone — a renamed .exe shouldn't slip past us on the +// strength of a "Content-Type: application/pdf" header. +function sniffMime(bytes: Uint8Array): string | null { + if (bytes.length < 8) return null; + const b = bytes; + // %PDF + if (b[0] === 0x25 && b[1] === 0x50 && b[2] === 0x44 && b[3] === 0x46) { + return "application/pdf"; + } + // PK (ZIP container — docx/xlsx) + if (b[0] === 0x50 && b[1] === 0x4b && (b[2] === 0x03 || b[2] === 0x05 || b[2] === 0x07)) { + return "application/zip"; // accept-with-allowlist handles docx/xlsx + } + // OLE compound (legacy doc/xls) + if ( + b[0] === 0xd0 && b[1] === 0xcf && b[2] === 0x11 && b[3] === 0xe0 && + b[4] === 0xa1 && b[5] === 0xb1 && b[6] === 0x1a && b[7] === 0xe1 + ) { + return "application/x-ole-storage"; + } + // JPEG + if (b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg"; + // PNG + if (b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "image/png"; + // GIF + if (b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x38) return "image/gif"; + // WEBP — "RIFF????WEBP" + if (b[0] === 0x52 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x46 && + b[8] === 0x57 && b[9] === 0x45 && b[10] === 0x42 && b[11] === 0x50) { + return "image/webp"; + } + return null; +} + +// Office formats (docx/xlsx) sniff as application/zip via PK header but +// are allowed at the client-reported MIME level. The mime check below +// keeps both axes honest: client mime must be in ALLOWED_MIME, AND the +// magic bytes must be plausible for that mime. +function mimePlausible(clientMime: string, sniffed: string | null): boolean { + if (!sniffed) return false; + if (sniffed === clientMime) return true; + // docx/xlsx are zips under the hood — accept the alias. + const zipAliased = new Set([ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ]); + if (sniffed === "application/zip" && zipAliased.has(clientMime)) return true; + // Legacy doc/xls share the OLE container. + const oleAliased = new Set(["application/msword", "application/vnd.ms-excel"]); + if (sniffed === "application/x-ole-storage" && oleAliased.has(clientMime)) return true; + return false; +} + +// Path-traversal scrub + length cap. Civi will store its own normalized +// name internally; this is purely defensive. +function sanitizeFilename(name: string): string { + const base = name.split(/[\\/]/).pop() ?? name; + // Drop control chars and anything that's not letters/digits/dot/dash/underscore/space. + const cleaned = base.replace(/[^\w.\- ]/g, "_").trim(); + return cleaned.slice(0, 200) || "upload"; +} + +function isStubMode(): boolean { + return !( + process.env.CIVI_BASE_URL && + process.env.CIVI_API_KEY && + process.env.CIVI_SITE_KEY + ); +} + +const FILE_FIELD_REFS = new Set( + allFields + .filter((f) => f.type === "file" && (f.civiField || f.civiContactField)) + .map((f) => f.civiField ?? f.civiContactField!), +); + +export async function POST(req: Request) { + // Generous-but-not-unlimited: 5 uploads per minute per IP. Captures + // accidental retry loops without throttling legitimate use (a form + // with 4 file fields fills in well under a minute). + const ip = clientIp(req); + const rl = rateLimit(`upload:${ip}`, { capacity: 5, windowMs: 60_000 }); + if (!rl.allowed) { + return NextResponse.json( + { error: "Too many uploads. Please wait a moment and try again." }, + { status: 429, headers: { "Retry-After": String(Math.ceil(rl.resetMs / 1000)) } }, + ); + } + + // Parse multipart. Next 16 supports Request.formData() natively. + let form: FormData; + try { + form = await req.formData(); + } catch { + return NextResponse.json({ error: "Expected multipart/form-data." }, { status: 400 }); + } + + const cid = (form.get("cid") as string | null) ?? ""; + const cs = (form.get("cs") as string | null) ?? ""; + const fieldRef = (form.get("fieldRef") as string | null) ?? ""; + const fileEntry = form.get("file"); + + if (!cid || !cs) { + return NextResponse.json({ error: "Missing cid or cs." }, { status: 400 }); + } + if (!fieldRef || !FILE_FIELD_REFS.has(fieldRef)) { + // Refusing unknown fieldRefs blocks the obvious abuse vector: a + // client posting an upload pointed at an arbitrary Civi field. + return NextResponse.json({ error: "Unknown or non-file field reference." }, { status: 400 }); + } + if (!(fileEntry instanceof File)) { + return NextResponse.json({ error: "Missing file part." }, { status: 400 }); + } + if (fileEntry.size === 0) { + return NextResponse.json({ error: "Empty file." }, { status: 400 }); + } + if (fileEntry.size > MAX_BYTES) { + return NextResponse.json( + { error: `File too large. Maximum is ${MAX_BYTES / (1024 * 1024)} MB.` }, + { status: 413 }, + ); + } + const clientMime = fileEntry.type || "application/octet-stream"; + if (!ALLOWED_MIME.has(clientMime)) { + return NextResponse.json( + { error: `File type ${clientMime} is not allowed.` }, + { status: 415 }, + ); + } + + const bytes = new Uint8Array(await fileEntry.arrayBuffer()); + const sniffed = sniffMime(bytes); + if (!mimePlausible(clientMime, sniffed)) { + return NextResponse.json( + { error: "File contents don't match the declared type." }, + { status: 415 }, + ); + } + + const safeName = sanitizeFilename(fileEntry.name); + + if (isStubMode()) { + console.warn( + "[upload:STUB] would Attachment.create", + JSON.stringify({ name: safeName, mime: clientMime, bytes: fileEntry.size, fieldRef }), + ); + return NextResponse.json({ id: -1, file_name: safeName, stub: true }); + } + + // Auth: verify the form's checksum before doing anything Civi-side. + const ok = await verifyChecksum(cid, cs); + if (!ok) { + return NextResponse.json( + { error: "Link is invalid or has expired." }, + { status: 401 }, + ); + } + + // TODO(file-pipeline-phase-1): wire CiviCRM Attachment.create. + // + // Awaiting spike output from scripts/spike-attachment-upload.mjs to + // determine which of two patterns to use: + // + // A. Unbound: Attachment.create with no entity_table/entity_id, then + // use the returned id as the field value at submit time. Preferred. + // + // B. Bound-at-upload: Attachment.create requires entity_table + + // entity_id. For contact-bound fields (Food_Co_op_Organizing.*) + // we can bind to the org contact. For activity-bound fields the + // activity doesn't exist yet — we'd need to attach to the org + // contact temporarily, then re-link to the activity post-create + // (or restructure submit to two-phase: create activity, then attach). + // + // Until the spike resolves, this endpoint returns 501 so it can't + // silently confuse the frontend. + return NextResponse.json( + { + error: + "Upload pipeline pending: CiviCRM Attachment.create wiring blocked on spike. " + + "See scripts/spike-attachment-upload.mjs.", + }, + { status: 501 }, + ); +} diff --git a/scripts/spike-attachment-upload.mjs b/scripts/spike-attachment-upload.mjs new file mode 100644 index 0000000..907b7e0 --- /dev/null +++ b/scripts/spike-attachment-upload.mjs @@ -0,0 +1,267 @@ +#!/usr/bin/env node +// scripts/spike-attachment-upload.mjs +// +// One-off SPIKE to validate the CiviCRM file-attachment pipeline before +// we commit to a v1 design for the form's /api/upload endpoint. +// +// We need to know, against THIS Civi instance: +// +// Q1. Does APIv4 Attachment.create succeed with NO entity binding? +// (Required for our planned two-step upload pattern: upload first, +// then reference the returned id from Activity.create on submit.) +// +// Q2. If Q1 is no, does Attachment.create require entity_table + +// entity_id at upload time? In that case the activity-bound file +// fields need a different flow (create empty activity first, attach, +// then update — or attach to the org contact temporarily). +// +// Q3. Can we write the returned attachment id as the value of a custom +// File field on Contact.update? (Test against Food_Co_op_Organizing. +// Certificate_of_Incorporation specifically.) +// +// Q4. Can /api/data's existing Contact.get with `.file_name` join read +// it back correctly? +// +// Q5. Does Attachment.delete clean it up afterward? (Needed for the +// teardown step here AND for the future orphan-cleanup Civi job.) +// +// USAGE +// +// node --env-file=.env.local scripts/spike-attachment-upload.mjs \ +// --org-id= \ +// [--keep] # don't delete the test attachment at the end +// +// REQUIRED ENV: CIVI_BASE_URL, CIVI_API_KEY, CIVI_SITE_KEY +// (plus CIVI_HTTP_AUTH_USER/PASS if Civi sits behind webserver basic auth) + +import { Buffer } from "node:buffer"; + +const args = process.argv.slice(2); +const orgIdArg = args.find((a) => a.startsWith("--org-id=")); +const KEEP = args.includes("--keep"); +const ORG_ID = orgIdArg ? Number(orgIdArg.slice("--org-id=".length)) : null; + +if (!ORG_ID) { + console.error( + "Usage: node --env-file=.env.local scripts/spike-attachment-upload.mjs --org-id= [--keep]", + ); + process.exit(1); +} + +const CUSTOM_FIELD = "Food_Co_op_Organizing.Certificate_of_Incorporation"; +const TEST_FILENAME = `spike-${Date.now()}.txt`; +const TEST_MIME = "text/plain"; +const TEST_BODY = "civi-webform attachment spike — safe to delete"; + +// ── CiviCRM APIv4 client (matches lib/civicrm.ts) ───────────────────── +async function civi(entity, action, params) { + const { + CIVI_BASE_URL, + CIVI_API_KEY, + CIVI_SITE_KEY, + CIVI_HTTP_AUTH_USER, + CIVI_HTTP_AUTH_PASS, + } = process.env; + if (!CIVI_BASE_URL || !CIVI_API_KEY || !CIVI_SITE_KEY) { + throw new Error("Missing CIVI_BASE_URL / CIVI_API_KEY / CIVI_SITE_KEY."); + } + const url = `${CIVI_BASE_URL}/civicrm/ajax/api4/${entity}/${action}`; + const headers = { + "Content-Type": "application/x-www-form-urlencoded", + "X-Civi-Auth": `Bearer ${CIVI_API_KEY}`, + "X-Civi-Key": CIVI_SITE_KEY, + }; + if (CIVI_HTTP_AUTH_USER && CIVI_HTTP_AUTH_PASS) { + headers["Authorization"] = + "Basic " + + Buffer.from(`${CIVI_HTTP_AUTH_USER}:${CIVI_HTTP_AUTH_PASS}`).toString( + "base64", + ); + } + const res = await fetch(url, { + method: "POST", + headers, + body: new URLSearchParams({ params: JSON.stringify(params) }), + }); + const text = await res.text(); + let json; + try { + json = JSON.parse(text); + } catch { + throw new Error(`${entity}.${action} non-JSON response (HTTP ${res.status}): ${text}`); + } + if (!res.ok || json.error_message) { + throw new Error( + `${entity}.${action} failed (HTTP ${res.status}): ${json.error_message ?? text}`, + ); + } + return json; +} + +function divider(label) { + console.log(`\n── ${label} ${"─".repeat(Math.max(0, 60 - label.length))}`); +} + +// ── Q1 / Q2: try Attachment.create both ways and see which Civi accepts. +// +// APIv4 Attachment.create expected params: +// name, mime_type, content (base64), entity_table?, entity_id? +// +async function tryUnbound() { + divider("Q1: Attachment.create WITHOUT entity binding"); + try { + const res = await civi("Attachment", "create", { + values: { + name: TEST_FILENAME, + mime_type: TEST_MIME, + content: Buffer.from(TEST_BODY, "utf8").toString("base64"), + }, + }); + console.log("RESULT: success ✓"); + console.log(JSON.stringify(res, null, 2)); + return res.values?.[0]?.id ?? null; + } catch (err) { + console.log("RESULT: failed"); + console.log(err.message); + return null; + } +} + +async function tryBoundToContact() { + divider(`Q2: Attachment.create BOUND to civicrm_contact id=${ORG_ID}`); + try { + const res = await civi("Attachment", "create", { + values: { + name: TEST_FILENAME, + mime_type: TEST_MIME, + content: Buffer.from(TEST_BODY, "utf8").toString("base64"), + entity_table: "civicrm_contact", + entity_id: ORG_ID, + }, + }); + console.log("RESULT: success ✓"); + console.log(JSON.stringify(res, null, 2)); + return res.values?.[0]?.id ?? null; + } catch (err) { + console.log("RESULT: failed"); + console.log(err.message); + return null; + } +} + +// ── Q3: write the file id as the value of the custom File field on the +// org contact. If the contact already has a certificate, save and restore. +async function testCustomFieldWrite(attachmentId) { + divider(`Q3: Contact.update writing ${CUSTOM_FIELD} = ${attachmentId}`); + + const before = await civi("Contact", "get", { + select: ["id", CUSTOM_FIELD, `${CUSTOM_FIELD}.file_name`], + where: [["id", "=", ORG_ID]], + }); + const prior = before.values?.[0] ?? {}; + console.log("Prior value on contact:", JSON.stringify(prior, null, 2)); + + try { + const res = await civi("Contact", "update", { + where: [["id", "=", ORG_ID]], + values: { [CUSTOM_FIELD]: attachmentId }, + }); + console.log("Update result:", JSON.stringify(res, null, 2)); + console.log("RESULT: success ✓"); + return { ok: true, prior: prior[CUSTOM_FIELD] ?? null }; + } catch (err) { + console.log("Update failed:", err.message); + return { ok: false, prior: prior[CUSTOM_FIELD] ?? null }; + } +} + +// ── Q4: read back the file via /api/data's join pattern. +async function testReadBack() { + divider("Q4: Contact.get with .file_name join"); + const res = await civi("Contact", "get", { + select: ["id", "display_name", CUSTOM_FIELD, `${CUSTOM_FIELD}.file_name`], + where: [["id", "=", ORG_ID]], + }); + console.log(JSON.stringify(res, null, 2)); + const row = res.values?.[0]; + if (row && row[CUSTOM_FIELD] && row[`${CUSTOM_FIELD}.file_name`]) { + console.log("RESULT: file id + filename round-trip works ✓"); + } else { + console.log("RESULT: round-trip incomplete — see payload above"); + } +} + +// ── Q5: clean up. +async function cleanup(attachmentId, restorePriorTo) { + if (KEEP) { + console.log("\n--keep set; not deleting attachment", attachmentId); + return; + } + divider("Q5: cleanup"); + + // Restore the contact's prior certificate value (so the spike doesn't + // leave the org pointing at a deleted attachment). + if (restorePriorTo !== undefined) { + try { + await civi("Contact", "update", { + where: [["id", "=", ORG_ID]], + values: { [CUSTOM_FIELD]: restorePriorTo }, + }); + console.log(`Restored prior ${CUSTOM_FIELD} value:`, restorePriorTo); + } catch (err) { + console.log("Restore failed:", err.message); + } + } + + try { + const res = await civi("Attachment", "delete", { + where: [["id", "=", attachmentId]], + }); + console.log(`Attachment.delete id=${attachmentId}:`, JSON.stringify(res)); + console.log("RESULT: cleanup OK ✓"); + } catch (err) { + console.log("Attachment.delete failed:", err.message); + console.log( + `*** MANUAL CLEANUP NEEDED: Attachment id=${attachmentId} is orphaned in CiviCRM ***`, + ); + } +} + +// ── orchestrator ───────────────────────────────────────────────────── +async function main() { + console.log("CIVI spike — attachment pipeline"); + console.log("Org contact:", ORG_ID); + console.log("Custom field:", CUSTOM_FIELD); + console.log("Test file:", TEST_FILENAME); + + let attachmentId = await tryUnbound(); + let restorePriorTo; + + if (!attachmentId) { + attachmentId = await tryBoundToContact(); + if (!attachmentId) { + console.log( + "\nNeither variant of Attachment.create succeeded. Stop here and", + "investigate Civi permissions / extension version.", + ); + process.exit(2); + } + console.log( + "\nNOTE: Civi rejected unbound attachment. v1 design must attach", + "the file to an entity at upload time (cannot decouple upload from", + "submit). Document this and adjust the plan.", + ); + } + + const write = await testCustomFieldWrite(attachmentId); + restorePriorTo = write.prior; + await testReadBack(); + await cleanup(attachmentId, restorePriorTo); + + console.log("\nSpike complete."); +} + +main().catch((err) => { + console.error("\nFATAL:", err); + process.exit(1); +});