From 2400931a04e5b0b450a0089576e96d34edbf204f Mon Sep 17 00:00:00 2001 From: Joel Brock Date: Fri, 5 Jun 2026 07:48:57 -0700 Subject: [PATCH] File upload pipeline: wire end-to-end via APIv4 File.create MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the file-upload gap. Files now actually land in CiviCRM (verified empirically against the live Civi instance via spike scripts). Spike findings (see scripts/spike-file-upload.mjs): - APIv4 Attachment is NOT exposed on this Civi - APIv4 File + EntityFile ARE exposed; File.create accepts inline base64 `content` and returns a usable file id - Custom file fields store the file id directly in the custom column, so EntityFile linkage is unnecessary for this use case - Round-trip via Contact.update + Contact.get .file_name join verified on a real org contact Pipeline: Renderer (FileField) picks up onChange → POST /api/upload (multipart) with file + cid + cs + fieldRef → verifyChecksum, MIME allowlist + magic-byte sniff, 5 MB cap → civi.File.create({ file_name, mime_type, content: base64 }) → returns { id, file_name } → renderer stores in RHF state via setValue Form submit → POST /api/submit (JSON) with the {id, file_name} value → submit detects the file shape and writes the id as the value of the activity/contact custom field File changes: app/api/upload/route.ts Replaced the 501 stub with the real File.create call. Comment documents that EntityFile linkage is intentionally skipped and that orphan cleanup is owned by a CiviCRM scheduled job. app/api/submit/route.ts For type:"file" values shaped as {id, file_name}, write the id as the custom field value (activity or contact, depending on the civiField / civiContactField the field declares). components/fields/FieldRenderer.tsx Replaced the bare register() with FileField, an upload-on-pick subcomponent. The native input is NOT register()'d: its FileList value was the original bug. FileField owns its uploading + error state and writes {id, file_name} via setValue on success. Submit is blocked upstream while uploads are in flight. components/StageSection.tsx, components/EngagementForm.tsx Thread setValue, cid, cs, and an onUploadStateChange callback through to FieldRenderer. EngagementForm tracks uploads-in-flight count; onSubmit refuses to submit while the count is > 0. config/form.ts Promotes Certificate_of_Incorporation from readonly to a real file field now that the pipeline works. app/api/data/route.ts Drops the readonly carveout that was only needed while the certificate was readonly. scripts/list-civi-entities.mjs (new) APIv4 entity probe + APIv3 attachment-API probe. Used to determine that File (not Attachment) was the right entity on this Civi. scripts/spike-file-upload.mjs (new) The actual end-to-end test that proved out the pipeline before wiring. Safe to re-run on any Civi instance during future audits. Not in this change: - Orphan attachment cleanup (CiviCRM scheduled job, Civi admin scope) - Per-field MIME allowlists (single global list for v1) - S3 / presigned-URL path for >5 MB files (deferred; capped at 5 MB today to stay under Amplify Lambda's 6 MB sync payload limit) --- app/api/data/route.ts | 8 +- app/api/submit/route.ts | 16 ++- app/api/upload/route.ts | 66 ++++++---- components/EngagementForm.tsx | 25 ++++ components/StageSection.tsx | 32 +++++ components/fields/FieldRenderer.tsx | 157 +++++++++++++++++++++--- config/form.ts | 7 +- scripts/list-civi-entities.mjs | 117 ++++++++++++++++++ scripts/spike-file-upload.mjs | 184 ++++++++++++++++++++++++++++ 9 files changed, 563 insertions(+), 49 deletions(-) create mode 100644 scripts/list-civi-entities.mjs create mode 100644 scripts/spike-file-upload.mjs diff --git a/app/api/data/route.ts b/app/api/data/route.ts index 027c673..622e9e4 100644 --- a/app/api/data/route.ts +++ b/app/api/data/route.ts @@ -170,11 +170,11 @@ export async function GET(req: NextRequest) { .map((f) => f.civiContactField) .filter((s): s is string => Boolean(s)); // For file-typed org-contact fields, also pull the joined .file_name so - // we can surface a human-readable filename in the readonly indicator. + // the prior-attachment indicator shows the filename, not just the file id. const orgContactFileRefs = allFields - .filter((f) => f.type === "file" || (f.type === "readonly" && f.civiContactField?.toLowerCase().includes("certificate"))) - .map((f) => f.civiContactField) - .filter((s): s is string => Boolean(s)); + .filter((f) => f.type === "file" && f.civiContactField) + .map((f) => f.civiContactField!) + .filter(Boolean); const orgContactSelect = [ "id", "display_name", diff --git a/app/api/submit/route.ts b/app/api/submit/route.ts index e36edf4..7f612b4 100644 --- a/app/api/submit/route.ts +++ b/app/api/submit/route.ts @@ -120,10 +120,22 @@ async function runSubmit(cid: string, cs: string, values: Record("File", "create", { + values: { + file_name: safeName, + mime_type: clientMime, + content: Buffer.from(bytes).toString("base64"), + }, + }); + const id = res.values?.[0]?.id; + if (!id) throw new Error("File.create returned no id"); + fileId = Number(id); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error("[upload] File.create failed:", msg); + return NextResponse.json( + { error: "Could not save the upload. Please try again." }, + { status: 502 }, + ); + } + + return NextResponse.json({ id: fileId, file_name: safeName }); } diff --git a/components/EngagementForm.tsx b/components/EngagementForm.tsx index 9aeb48c..a5a0896 100644 --- a/components/EngagementForm.tsx +++ b/components/EngagementForm.tsx @@ -66,9 +66,17 @@ export function EngagementForm({ config, cid, cs }: EngagementFormProps) { control, watch, setFocus, + setValue, formState: { errors, isDirty }, } = useForm({ mode: "onBlur" }); + // Count of file uploads currently in flight. Each calls the + // handler with +1 when it starts and -1 when it finishes; submit is + // blocked while the count is > 0 so users can't ship a half-uploaded form. + const [uploadsInFlight, setUploadsInFlight] = useState(0); + const handleUploadStateChange = (delta: 1 | -1) => + setUploadsInFlight((n) => Math.max(0, n + delta)); + // Subscribe ONLY to current_stage. That's the single field that affects // section visibility, so re-rendering the whole form on every keystroke // (which `watch()` with no args would do) is wasteful — particularly with @@ -279,6 +287,19 @@ export function EngagementForm({ config, cid, cs }: EngagementFormProps) { } const onSubmit = async (values: Record) => { + // Block while any file upload is in flight — submitting now would + // ship the form without the pending {id, file_name} value for that + // field, which would silently clear the prior attachment. + if (uploadsInFlight > 0) { + setSubmitState({ + kind: "error", + message: + uploadsInFlight === 1 + ? "A file is still uploading. Please wait a moment and try again." + : `${uploadsInFlight} files are still uploading. Please wait a moment and try again.`, + }); + return; + } setSubmitState({ kind: "submitting" }); try { // Strip values for hidden fields — never write data the user couldn't see. @@ -393,6 +414,7 @@ export function EngagementForm({ config, cid, cs }: EngagementFormProps) { ); diff --git a/components/StageSection.tsx b/components/StageSection.tsx index 72dae78..139c070 100644 --- a/components/StageSection.tsx +++ b/components/StageSection.tsx @@ -4,6 +4,7 @@ import { useEffect, useId, useMemo, useState } from "react"; import type { FieldConfig, StageSectionConfig, SelectOption } from "@/types/form"; import type { UseFormRegister, + UseFormSetValue, FieldValues, FieldErrors, Control, @@ -16,6 +17,7 @@ import { evaluate } from "@/lib/conditional"; interface StageSectionProps { section: StageSectionConfig; register: UseFormRegister; + setValue: UseFormSetValue; control: Control; errors: FieldErrors; /** Live form values, used to evaluate per-field visibility rules. */ @@ -32,6 +34,12 @@ interface StageSectionProps { defaultOpen: boolean; /** Option groups fetched from CiviCRM, keyed by option_group_id. */ options: Record; + /** Form auth pair, passed through to file fields for /api/upload. */ + cid: string; + cs: string; + /** Called by file fields when an upload starts/finishes; lets the form + * track in-flight uploads and block submit until they settle. */ + onUploadStateChange?: (delta: 1 | -1) => void; } /** @@ -48,6 +56,7 @@ interface StageSectionProps { export function StageSection({ section, register, + setValue, control, errors, formValues, @@ -55,6 +64,9 @@ export function StageSection({ locked, defaultOpen, options, + cid, + cs, + onUploadStateChange, }: StageSectionProps) { const [open, setOpen] = useState(defaultOpen); const headingId = useId(); @@ -228,8 +240,12 @@ export function StageSection({ formValues={formValues} options={options} register={register} + setValue={setValue} control={control} errors={errors} + cid={cid} + cs={cs} + onUploadStateChange={onUploadStateChange} /> ))} @@ -251,10 +267,14 @@ export function StageSection({ ); @@ -360,8 +380,12 @@ function FieldGroupCard({ formValues, options, register, + setValue, control, errors, + cid, + cs, + onUploadStateChange, }: { label?: string; intro?: string; @@ -369,8 +393,12 @@ function FieldGroupCard({ formValues: Record; options: Record; register: UseFormRegister; + setValue: UseFormSetValue; control: Control; errors: FieldErrors; + cid: string; + cs: string; + onUploadStateChange?: (delta: 1 | -1) => void; }) { return (
@@ -404,10 +432,14 @@ function FieldGroupCard({
); diff --git a/components/fields/FieldRenderer.tsx b/components/fields/FieldRenderer.tsx index 4410ce8..4adfeb8 100644 --- a/components/fields/FieldRenderer.tsx +++ b/components/fields/FieldRenderer.tsx @@ -1,9 +1,11 @@ "use client"; +import { useState } from "react"; import { useWatch } from "react-hook-form"; import type { FieldConfig, SelectOption } from "@/types/form"; import type { UseFormRegister, + UseFormSetValue, FieldValues, FieldErrors, Control, @@ -12,6 +14,9 @@ import type { interface FieldRendererProps { field: FieldConfig; register: UseFormRegister; + /** RHF setValue — file fields use it to write the uploaded {id, file_name} + * back into form state after /api/upload returns. */ + setValue: UseFormSetValue; errors: FieldErrors; /** Form control — required for currency live-preview formatting. */ control: Control; @@ -23,6 +28,12 @@ interface FieldRendererProps { * `options` array if absent. */ resolvedOptions?: SelectOption[]; + /** Form auth pair, threaded through to file fields for /api/upload. */ + cid: string; + cs: string; + /** Called by file fields when an upload starts (+1) / finishes (-1). The + * form uses the running count to block submit while uploads are in flight. */ + onUploadStateChange?: (delta: 1 | -1) => void; } const DATE_MIN_DEFAULT = "1900-01-01"; @@ -44,10 +55,14 @@ const DATE_MAX_DEFAULT = "2100-12-31"; export function FieldRenderer({ field, register, + setValue, errors, control, readonlyValue, resolvedOptions, + cid, + cs, + onUploadStateChange, }: FieldRendererProps) { const id = `field-${field.name}`; const helpId = field.help ? `${id}-help` : undefined; @@ -218,21 +233,21 @@ export function FieldRenderer({ // ── File ──────────────────────────────────────────────────────────────── if (field.type === "file") { return ( -
-
+ ); } @@ -433,6 +448,118 @@ function FilePriorIndicator({ ); } +/** + * Upload-on-pick file field. The moment the user selects a file the + * browser sends it to /api/upload; on success we replace RHF state with + * the returned {id, file_name}. That same shape is what submit serializes + * and what the prefill path produces for prior attachments — so the rest + * of the pipeline doesn't care whether the value originated as prefill + * or as a fresh upload. + * + * The native is intentionally NOT register()'d here: + * its `value` is a FileList that doesn't survive JSON.stringify, which is + * the whole bug we're closing. We manage state manually via setValue. + */ +function FileField({ + field, + id, + helpId, + errorId, + errorMsg, + describedBy, + control, + setValue, + cid, + cs, + onUploadStateChange, +}: { + field: FieldConfig; + id: string; + helpId?: string; + errorId?: string; + errorMsg?: string; + describedBy?: string; + control: Control; + setValue: UseFormSetValue; + cid: string; + cs: string; + onUploadStateChange?: (delta: 1 | -1) => void; + register: UseFormRegister; + requiredOpt: string | false; +}) { + const [uploading, setUploading] = useState(false); + const [uploadError, setUploadError] = useState(null); + // Target Civi field reference — whichever side this field maps to. + const fieldRef = field.civiField ?? field.civiContactField ?? ""; + + async function handlePick(e: React.ChangeEvent) { + const file = e.target.files?.[0]; + if (!file) return; + + setUploadError(null); + setUploading(true); + onUploadStateChange?.(1); + + const fd = new FormData(); + fd.set("file", file); + fd.set("cid", cid); + fd.set("cs", cs); + fd.set("fieldRef", fieldRef); + + try { + const res = await fetch("/api/upload", { method: "POST", body: fd }); + const json = (await res.json().catch(() => ({}))) as { + id?: number; + file_name?: string; + error?: string; + }; + if (!res.ok) { + setUploadError(json.error ?? `Upload failed (HTTP ${res.status}).`); + // Reset the file input so the user can retry the same file. + e.target.value = ""; + } else if (json.id != null && json.file_name) { + setValue(field.name, { id: json.id, file_name: json.file_name }, { + shouldDirty: true, + shouldValidate: true, + }); + } else { + setUploadError("Upload succeeded but no file id was returned."); + } + } catch (err) { + setUploadError(err instanceof Error ? err.message : "Upload network error."); + e.target.value = ""; + } finally { + setUploading(false); + onUploadStateChange?.(-1); + } + } + + return ( +
+
+ ); +} + function Label({ id, field }: { id: string; field: FieldConfig }) { return (