File upload pipeline: spike + endpoint skeleton (Phase 1, in progress)

Lays groundwork for closing the file-upload gap discovered while wiring
the org-contact custom fields. Currently no file fields in the form
actually persist to CiviCRM -- the renderer FileList drops at the
onSubmit JSON.stringify, and there is no /api/upload route or
Attachment.create call anywhere.

This commit adds:

1. scripts/spike-attachment-upload.mjs

   One-off spike to answer the open question that gates the rest of the
   work: does APIv4 Attachment.create accept an unbound upload, or must
   we attach to an entity at create time? If unbound works we can use
   the planned two-step pattern (upload returns a file id; submit
   references it). If not, activity-bound file fields need a different
   flow because the activity does not exist yet at upload time.

   The spike also exercises the Contact.update + .file_name read-back
   path against the Certificate_of_Incorporation field on a real org
   contact, then cleans up after itself.

   Usage:
     node --env-file=.env.local scripts/spike-attachment-upload.mjs \
       --org-id=<id> [--keep]

2. app/api/upload/route.ts

   Structural pieces that do not depend on the spike outcome:
     - multipart parsing via Request.formData()
     - 5 MB hard cap (under Amplify Lambda 6 MB sync payload limit)
     - MIME allowlist (PDF, DOC/DOCX, XLS/XLSX, JPEG/PNG/GIF/WEBP)
     - magic-byte sniff to cross-check the client-reported MIME
     - filename sanitization (path traversal scrub, length cap)
     - checksum verification, rate limiting, field-ref allowlist
     - STUB-mode short-circuit for local dev without live Civi
     - explicit 501 where the Civi Attachment.create wiring goes,
       with a comment pointing at the spike that resolves it

   Result: endpoint compiles, registers as a Next route, returns 501
   with a clear message; build passes; nothing wired into the frontend
   yet so the existing form is unaffected.

Phase 2 (renderer upload-on-pick), Phase 3 (submit reshape), Phase 4
(promote Certificate_of_Incorporation to editable) follow once the
spike output picks the Attachment.create variant.
This commit is contained in:
Joel Brock
2026-06-05 07:11:18 -07:00
parent 8ecf64c79b
commit 8159b87074
2 changed files with 500 additions and 0 deletions
+233
View File
@@ -0,0 +1,233 @@
/**
* POST /api/upload
*
* Multipart endpoint that accepts a single file plus the form's auth pair
* (cid + cs) and the target Civi field reference. Verifies, validates,
* stores in CiviCRM via Attachment.create, returns { id, file_name }.
*
* The form's file renderer calls this on file-pick (not at submit time)
* so submit can stay a simple JSON POST. The returned { id, file_name }
* is what gets put into RHF state and ultimately submitted as the field
* value — the same shape /api/data uses for prefill, so the renderer's
* FilePriorIndicator works unchanged for fresh uploads too.
*
* Request: multipart/form-data with parts:
* - file the binary
* - cid contact id (form auth)
* - cs checksum (form auth)
* - fieldRef the Civi field reference, e.g. "Stage_1.Vision_Upload"
* or "Food_Co_op_Organizing.Certificate_of_Incorporation"
*
* Response: { id: number, file_name: string } OR { error: string }
*
* STUB MODE: if CiviCRM env vars are unset, returns a fake id + the
* uploaded filename so frontend dev works without a live CRM.
*/
import { NextResponse } from "next/server";
import { verifyChecksum } from "@/lib/civicrm";
import { allFields } from "@/config/form";
import { rateLimit, clientIp } from "@/lib/rate-limit";
// 5 MB hard cap. Sits under AWS Amplify Lambda's 6 MB sync invocation
// payload limit with headroom for multipart envelope overhead.
const MAX_BYTES = 5 * 1024 * 1024;
// Per the v1 spec: PDF, DOC/DOCX, XLS/XLSX, common image types.
const ALLOWED_MIME = new Set<string>([
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
]);
// Magic-byte sniff for the most common forgeries. Don't trust client-
// reported MIME alone — a renamed .exe shouldn't slip past us on the
// strength of a "Content-Type: application/pdf" header.
function sniffMime(bytes: Uint8Array): string | null {
if (bytes.length < 8) return null;
const b = bytes;
// %PDF
if (b[0] === 0x25 && b[1] === 0x50 && b[2] === 0x44 && b[3] === 0x46) {
return "application/pdf";
}
// PK (ZIP container — docx/xlsx)
if (b[0] === 0x50 && b[1] === 0x4b && (b[2] === 0x03 || b[2] === 0x05 || b[2] === 0x07)) {
return "application/zip"; // accept-with-allowlist handles docx/xlsx
}
// OLE compound (legacy doc/xls)
if (
b[0] === 0xd0 && b[1] === 0xcf && b[2] === 0x11 && b[3] === 0xe0 &&
b[4] === 0xa1 && b[5] === 0xb1 && b[6] === 0x1a && b[7] === 0xe1
) {
return "application/x-ole-storage";
}
// JPEG
if (b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return "image/jpeg";
// PNG
if (b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) return "image/png";
// GIF
if (b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x38) return "image/gif";
// WEBP — "RIFF????WEBP"
if (b[0] === 0x52 && b[1] === 0x49 && b[2] === 0x46 && b[3] === 0x46 &&
b[8] === 0x57 && b[9] === 0x45 && b[10] === 0x42 && b[11] === 0x50) {
return "image/webp";
}
return null;
}
// Office formats (docx/xlsx) sniff as application/zip via PK header but
// are allowed at the client-reported MIME level. The mime check below
// keeps both axes honest: client mime must be in ALLOWED_MIME, AND the
// magic bytes must be plausible for that mime.
function mimePlausible(clientMime: string, sniffed: string | null): boolean {
if (!sniffed) return false;
if (sniffed === clientMime) return true;
// docx/xlsx are zips under the hood — accept the alias.
const zipAliased = new Set([
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
]);
if (sniffed === "application/zip" && zipAliased.has(clientMime)) return true;
// Legacy doc/xls share the OLE container.
const oleAliased = new Set(["application/msword", "application/vnd.ms-excel"]);
if (sniffed === "application/x-ole-storage" && oleAliased.has(clientMime)) return true;
return false;
}
// Path-traversal scrub + length cap. Civi will store its own normalized
// name internally; this is purely defensive.
function sanitizeFilename(name: string): string {
const base = name.split(/[\\/]/).pop() ?? name;
// Drop control chars and anything that's not letters/digits/dot/dash/underscore/space.
const cleaned = base.replace(/[^\w.\- ]/g, "_").trim();
return cleaned.slice(0, 200) || "upload";
}
function isStubMode(): boolean {
return !(
process.env.CIVI_BASE_URL &&
process.env.CIVI_API_KEY &&
process.env.CIVI_SITE_KEY
);
}
const FILE_FIELD_REFS = new Set(
allFields
.filter((f) => f.type === "file" && (f.civiField || f.civiContactField))
.map((f) => f.civiField ?? f.civiContactField!),
);
export async function POST(req: Request) {
// Generous-but-not-unlimited: 5 uploads per minute per IP. Captures
// accidental retry loops without throttling legitimate use (a form
// with 4 file fields fills in well under a minute).
const ip = clientIp(req);
const rl = rateLimit(`upload:${ip}`, { capacity: 5, windowMs: 60_000 });
if (!rl.allowed) {
return NextResponse.json(
{ error: "Too many uploads. Please wait a moment and try again." },
{ status: 429, headers: { "Retry-After": String(Math.ceil(rl.resetMs / 1000)) } },
);
}
// Parse multipart. Next 16 supports Request.formData() natively.
let form: FormData;
try {
form = await req.formData();
} catch {
return NextResponse.json({ error: "Expected multipart/form-data." }, { status: 400 });
}
const cid = (form.get("cid") as string | null) ?? "";
const cs = (form.get("cs") as string | null) ?? "";
const fieldRef = (form.get("fieldRef") as string | null) ?? "";
const fileEntry = form.get("file");
if (!cid || !cs) {
return NextResponse.json({ error: "Missing cid or cs." }, { status: 400 });
}
if (!fieldRef || !FILE_FIELD_REFS.has(fieldRef)) {
// Refusing unknown fieldRefs blocks the obvious abuse vector: a
// client posting an upload pointed at an arbitrary Civi field.
return NextResponse.json({ error: "Unknown or non-file field reference." }, { status: 400 });
}
if (!(fileEntry instanceof File)) {
return NextResponse.json({ error: "Missing file part." }, { status: 400 });
}
if (fileEntry.size === 0) {
return NextResponse.json({ error: "Empty file." }, { status: 400 });
}
if (fileEntry.size > MAX_BYTES) {
return NextResponse.json(
{ error: `File too large. Maximum is ${MAX_BYTES / (1024 * 1024)} MB.` },
{ status: 413 },
);
}
const clientMime = fileEntry.type || "application/octet-stream";
if (!ALLOWED_MIME.has(clientMime)) {
return NextResponse.json(
{ error: `File type ${clientMime} is not allowed.` },
{ status: 415 },
);
}
const bytes = new Uint8Array(await fileEntry.arrayBuffer());
const sniffed = sniffMime(bytes);
if (!mimePlausible(clientMime, sniffed)) {
return NextResponse.json(
{ error: "File contents don't match the declared type." },
{ status: 415 },
);
}
const safeName = sanitizeFilename(fileEntry.name);
if (isStubMode()) {
console.warn(
"[upload:STUB] would Attachment.create",
JSON.stringify({ name: safeName, mime: clientMime, bytes: fileEntry.size, fieldRef }),
);
return NextResponse.json({ id: -1, file_name: safeName, stub: true });
}
// Auth: verify the form's checksum before doing anything Civi-side.
const ok = await verifyChecksum(cid, cs);
if (!ok) {
return NextResponse.json(
{ error: "Link is invalid or has expired." },
{ status: 401 },
);
}
// TODO(file-pipeline-phase-1): wire CiviCRM Attachment.create.
//
// Awaiting spike output from scripts/spike-attachment-upload.mjs to
// determine which of two patterns to use:
//
// A. Unbound: Attachment.create with no entity_table/entity_id, then
// use the returned id as the field value at submit time. Preferred.
//
// B. Bound-at-upload: Attachment.create requires entity_table +
// entity_id. For contact-bound fields (Food_Co_op_Organizing.*)
// we can bind to the org contact. For activity-bound fields the
// activity doesn't exist yet — we'd need to attach to the org
// contact temporarily, then re-link to the activity post-create
// (or restructure submit to two-phase: create activity, then attach).
//
// Until the spike resolves, this endpoint returns 501 so it can't
// silently confuse the frontend.
return NextResponse.json(
{
error:
"Upload pipeline pending: CiviCRM Attachment.create wiring blocked on spike. " +
"See scripts/spike-attachment-upload.mjs.",
},
{ status: 501 },
);
}
+267
View File
@@ -0,0 +1,267 @@
#!/usr/bin/env node
// scripts/spike-attachment-upload.mjs
//
// One-off SPIKE to validate the CiviCRM file-attachment pipeline before
// we commit to a v1 design for the form's /api/upload endpoint.
//
// We need to know, against THIS Civi instance:
//
// Q1. Does APIv4 Attachment.create succeed with NO entity binding?
// (Required for our planned two-step upload pattern: upload first,
// then reference the returned id from Activity.create on submit.)
//
// Q2. If Q1 is no, does Attachment.create require entity_table +
// entity_id at upload time? In that case the activity-bound file
// fields need a different flow (create empty activity first, attach,
// then update — or attach to the org contact temporarily).
//
// Q3. Can we write the returned attachment id as the value of a custom
// File field on Contact.update? (Test against Food_Co_op_Organizing.
// Certificate_of_Incorporation specifically.)
//
// Q4. Can /api/data's existing Contact.get with `.file_name` join read
// it back correctly?
//
// Q5. Does Attachment.delete clean it up afterward? (Needed for the
// teardown step here AND for the future orphan-cleanup Civi job.)
//
// USAGE
//
// node --env-file=.env.local scripts/spike-attachment-upload.mjs \
// --org-id=<orgContactId> \
// [--keep] # don't delete the test attachment at the end
//
// REQUIRED ENV: CIVI_BASE_URL, CIVI_API_KEY, CIVI_SITE_KEY
// (plus CIVI_HTTP_AUTH_USER/PASS if Civi sits behind webserver basic auth)
import { Buffer } from "node:buffer";
const args = process.argv.slice(2);
const orgIdArg = args.find((a) => a.startsWith("--org-id="));
const KEEP = args.includes("--keep");
const ORG_ID = orgIdArg ? Number(orgIdArg.slice("--org-id=".length)) : null;
if (!ORG_ID) {
console.error(
"Usage: node --env-file=.env.local scripts/spike-attachment-upload.mjs --org-id=<n> [--keep]",
);
process.exit(1);
}
const CUSTOM_FIELD = "Food_Co_op_Organizing.Certificate_of_Incorporation";
const TEST_FILENAME = `spike-${Date.now()}.txt`;
const TEST_MIME = "text/plain";
const TEST_BODY = "civi-webform attachment spike — safe to delete";
// ── CiviCRM APIv4 client (matches lib/civicrm.ts) ─────────────────────
async function civi(entity, action, params) {
const {
CIVI_BASE_URL,
CIVI_API_KEY,
CIVI_SITE_KEY,
CIVI_HTTP_AUTH_USER,
CIVI_HTTP_AUTH_PASS,
} = process.env;
if (!CIVI_BASE_URL || !CIVI_API_KEY || !CIVI_SITE_KEY) {
throw new Error("Missing CIVI_BASE_URL / CIVI_API_KEY / CIVI_SITE_KEY.");
}
const url = `${CIVI_BASE_URL}/civicrm/ajax/api4/${entity}/${action}`;
const headers = {
"Content-Type": "application/x-www-form-urlencoded",
"X-Civi-Auth": `Bearer ${CIVI_API_KEY}`,
"X-Civi-Key": CIVI_SITE_KEY,
};
if (CIVI_HTTP_AUTH_USER && CIVI_HTTP_AUTH_PASS) {
headers["Authorization"] =
"Basic " +
Buffer.from(`${CIVI_HTTP_AUTH_USER}:${CIVI_HTTP_AUTH_PASS}`).toString(
"base64",
);
}
const res = await fetch(url, {
method: "POST",
headers,
body: new URLSearchParams({ params: JSON.stringify(params) }),
});
const text = await res.text();
let json;
try {
json = JSON.parse(text);
} catch {
throw new Error(`${entity}.${action} non-JSON response (HTTP ${res.status}): ${text}`);
}
if (!res.ok || json.error_message) {
throw new Error(
`${entity}.${action} failed (HTTP ${res.status}): ${json.error_message ?? text}`,
);
}
return json;
}
function divider(label) {
console.log(`\n── ${label} ${"─".repeat(Math.max(0, 60 - label.length))}`);
}
// ── Q1 / Q2: try Attachment.create both ways and see which Civi accepts.
//
// APIv4 Attachment.create expected params:
// name, mime_type, content (base64), entity_table?, entity_id?
//
async function tryUnbound() {
divider("Q1: Attachment.create WITHOUT entity binding");
try {
const res = await civi("Attachment", "create", {
values: {
name: TEST_FILENAME,
mime_type: TEST_MIME,
content: Buffer.from(TEST_BODY, "utf8").toString("base64"),
},
});
console.log("RESULT: success ✓");
console.log(JSON.stringify(res, null, 2));
return res.values?.[0]?.id ?? null;
} catch (err) {
console.log("RESULT: failed");
console.log(err.message);
return null;
}
}
async function tryBoundToContact() {
divider(`Q2: Attachment.create BOUND to civicrm_contact id=${ORG_ID}`);
try {
const res = await civi("Attachment", "create", {
values: {
name: TEST_FILENAME,
mime_type: TEST_MIME,
content: Buffer.from(TEST_BODY, "utf8").toString("base64"),
entity_table: "civicrm_contact",
entity_id: ORG_ID,
},
});
console.log("RESULT: success ✓");
console.log(JSON.stringify(res, null, 2));
return res.values?.[0]?.id ?? null;
} catch (err) {
console.log("RESULT: failed");
console.log(err.message);
return null;
}
}
// ── Q3: write the file id as the value of the custom File field on the
// org contact. If the contact already has a certificate, save and restore.
async function testCustomFieldWrite(attachmentId) {
divider(`Q3: Contact.update writing ${CUSTOM_FIELD} = ${attachmentId}`);
const before = await civi("Contact", "get", {
select: ["id", CUSTOM_FIELD, `${CUSTOM_FIELD}.file_name`],
where: [["id", "=", ORG_ID]],
});
const prior = before.values?.[0] ?? {};
console.log("Prior value on contact:", JSON.stringify(prior, null, 2));
try {
const res = await civi("Contact", "update", {
where: [["id", "=", ORG_ID]],
values: { [CUSTOM_FIELD]: attachmentId },
});
console.log("Update result:", JSON.stringify(res, null, 2));
console.log("RESULT: success ✓");
return { ok: true, prior: prior[CUSTOM_FIELD] ?? null };
} catch (err) {
console.log("Update failed:", err.message);
return { ok: false, prior: prior[CUSTOM_FIELD] ?? null };
}
}
// ── Q4: read back the file via /api/data's join pattern.
async function testReadBack() {
divider("Q4: Contact.get with .file_name join");
const res = await civi("Contact", "get", {
select: ["id", "display_name", CUSTOM_FIELD, `${CUSTOM_FIELD}.file_name`],
where: [["id", "=", ORG_ID]],
});
console.log(JSON.stringify(res, null, 2));
const row = res.values?.[0];
if (row && row[CUSTOM_FIELD] && row[`${CUSTOM_FIELD}.file_name`]) {
console.log("RESULT: file id + filename round-trip works ✓");
} else {
console.log("RESULT: round-trip incomplete — see payload above");
}
}
// ── Q5: clean up.
async function cleanup(attachmentId, restorePriorTo) {
if (KEEP) {
console.log("\n--keep set; not deleting attachment", attachmentId);
return;
}
divider("Q5: cleanup");
// Restore the contact's prior certificate value (so the spike doesn't
// leave the org pointing at a deleted attachment).
if (restorePriorTo !== undefined) {
try {
await civi("Contact", "update", {
where: [["id", "=", ORG_ID]],
values: { [CUSTOM_FIELD]: restorePriorTo },
});
console.log(`Restored prior ${CUSTOM_FIELD} value:`, restorePriorTo);
} catch (err) {
console.log("Restore failed:", err.message);
}
}
try {
const res = await civi("Attachment", "delete", {
where: [["id", "=", attachmentId]],
});
console.log(`Attachment.delete id=${attachmentId}:`, JSON.stringify(res));
console.log("RESULT: cleanup OK ✓");
} catch (err) {
console.log("Attachment.delete failed:", err.message);
console.log(
`*** MANUAL CLEANUP NEEDED: Attachment id=${attachmentId} is orphaned in CiviCRM ***`,
);
}
}
// ── orchestrator ─────────────────────────────────────────────────────
async function main() {
console.log("CIVI spike — attachment pipeline");
console.log("Org contact:", ORG_ID);
console.log("Custom field:", CUSTOM_FIELD);
console.log("Test file:", TEST_FILENAME);
let attachmentId = await tryUnbound();
let restorePriorTo;
if (!attachmentId) {
attachmentId = await tryBoundToContact();
if (!attachmentId) {
console.log(
"\nNeither variant of Attachment.create succeeded. Stop here and",
"investigate Civi permissions / extension version.",
);
process.exit(2);
}
console.log(
"\nNOTE: Civi rejected unbound attachment. v1 design must attach",
"the file to an entity at upload time (cannot decouple upload from",
"submit). Document this and adjust the plan.",
);
}
const write = await testCustomFieldWrite(attachmentId);
restorePriorTo = write.prior;
await testReadBack();
await cleanup(attachmentId, restorePriorTo);
console.log("\nSpike complete.");
}
main().catch((err) => {
console.error("\nFATAL:", err);
process.exit(1);
});