feat: reader-compat batch — JSON Feed, OPML export, conditional GET, dedup

Batch of four reader-facing improvements (TODO "Compat lecteurs + dedup"):

- JSON Feed at /json/:feedId (feed lib .json1()); all formats cross-link
- OPML export at /admin/opml (admin-protected; the registry lists every
  feed URL, so it must not be public)
- Conditional GET on /rss + /atom: strong ETag + Last-Modified, 304 on
  If-None-Match/If-Modified-Since, validators shared via http-cache.ts
- Duplicate-send dedup in ingestion: match by Message-ID, fall back to a
  SHA-256 of normalized subject+content; a duplicate is a no-op and bumps
  the new emails_deduplicated counter (status page + /api/v1/stats)

429 tests green, tsc clean, build dry-run OK. Docs (README/CLAUDE/TODO +
landing cards) updated.

Co-Authored-By: Claude Opus 4.7 <[email protected]>

src/application/email-processor.test.ts

@@ -595,6 +595,134 @@ describe("processEmail — attachments", () => {
   });
 });
 
+describe("processEmail — deduplication", () => {
+  let env: ReturnType<typeof createMockEnv>;
+
+  beforeEach(async () => {
+    env = createMockEnv();
+    await env.EMAIL_STORAGE.put(
+      `feed:${VALID_FEED_ID}:config`,
+      JSON.stringify({}),
+    );
+  });
+
+  it("stores only one email when the same Message-ID is delivered twice", async () => {
+    const headers = { "Message-ID": "<[email protected]>" };
+    await processEmail(makeInput({ headers }), env as any);
+    await processEmail(makeInput({ headers }), env as any);
+
+    const metadata = await env.EMAIL_STORAGE.get(
+      `feed:${VALID_FEED_ID}:metadata`,
+      "json",
+    );
+    expect(metadata.emails).toHaveLength(1);
+  });
+
+  it("increments emails_deduplicated counter on the second delivery", async () => {
+    const headers = { "Message-ID": "<[email protected]>" };
+    await processEmail(makeInput({ headers }), env as any);
+    await processEmail(makeInput({ headers }), env as any);
+
+    const counters = await getCounters(env.EMAIL_STORAGE as any);
+    expect(counters.emails_deduplicated).toBe(1);
+  });
+
+  it("deduplicates by hash when no Message-ID header is present", async () => {
+    const input = makeInput({
+      subject: "Weekly Digest",
+      content: "<p>Same content</p>",
+    });
+    await processEmail(input, env as any);
+    await processEmail(input, env as any);
+
+    const metadata = await env.EMAIL_STORAGE.get(
+      `feed:${VALID_FEED_ID}:metadata`,
+      "json",
+    );
+    expect(metadata.emails).toHaveLength(1);
+
+    const counters = await getCounters(env.EMAIL_STORAGE as any);
+    expect(counters.emails_deduplicated).toBe(1);
+  });
+
+  it("does not deduplicate emails with different subjects (no Message-ID)", async () => {
+    await processEmail(
+      makeInput({ subject: "First", content: "<p>body</p>" }),
+      env as any,
+    );
+    await processEmail(
+      makeInput({ subject: "Second", content: "<p>body</p>" }),
+      env as any,
+    );
+
+    const metadata = await env.EMAIL_STORAGE.get(
+      `feed:${VALID_FEED_ID}:metadata`,
+      "json",
+    );
+    expect(metadata.emails).toHaveLength(2);
+
+    const counters = await getCounters(env.EMAIL_STORAGE as any);
+    expect(counters.emails_deduplicated).toBe(0);
+  });
+
+  it("does not false-positive against pre-feature entries lacking messageId/dedupHash", async () => {
+    // Seed a legacy metadata entry with no messageId or dedupHash
+    await env.EMAIL_STORAGE.put(
+      `feed:${VALID_FEED_ID}:metadata`,
+      JSON.stringify({
+        emails: [
+          {
+            key: `feed:${VALID_FEED_ID}:999`,
+            subject: "Old Subject",
+            receivedAt: 999,
+            size: 50,
+            // intentionally no messageId, no dedupHash
+          },
+        ],
+      }),
+    );
+
+    // A new, distinct email should be stored without triggering false dedup
+    const res = await processEmail(
+      makeInput({ subject: "New Distinct Email", content: "<p>fresh</p>" }),
+      env as any,
+    );
+    expect(res.ok).toBe(true);
+
+    const metadata = await env.EMAIL_STORAGE.get(
+      `feed:${VALID_FEED_ID}:metadata`,
+      "json",
+    );
+    expect(metadata.emails).toHaveLength(2);
+
+    const counters = await getCounters(env.EMAIL_STORAGE as any);
+    expect(counters.emails_deduplicated).toBe(0);
+  });
+
+  it("returns { ok: true } for a genuine duplicate (not a rejection)", async () => {
+    const headers = { "Message-ID": "<[email protected]>" };
+    await processEmail(makeInput({ headers }), env as any);
+    const res = await processEmail(makeInput({ headers }), env as any);
+    expect(res).toMatchObject({ ok: true });
+  });
+
+  it("stores messageId and dedupHash in the email metadata entry", async () => {
+    const headers = { "Message-ID": "<[email protected]>" };
+    await processEmail(
+      makeInput({ subject: "Sub", content: "<p>c</p>", headers }),
+      env as any,
+    );
+
+    const metadata = await env.EMAIL_STORAGE.get(
+      `feed:${VALID_FEED_ID}:metadata`,
+      "json",
+    );
+    expect(metadata.emails[0].messageId).toBe("<[email protected]>");
+    expect(typeof metadata.emails[0].dedupHash).toBe("string");
+    expect(metadata.emails[0].dedupHash).toHaveLength(64); // SHA-256 hex
+  });
+});
+
 describe("processEmail — monitoring counters", () => {
   it("increments emails_received and sets last_email_at on success", async () => {
     const env = createMockEnv();
@@ -709,6 +837,7 @@ describe("processEmail — unsubscribe capture", () => {
   it("keeps one entry per sender and overwrites with the latest URL", async () => {
     await processEmail(
       makeInput({
+        subject: "Issue 1 from A",
         senders: ["[email protected]"],
         headers: {
           "list-unsubscribe": "<https://one.com/u/1>",
@@ -719,6 +848,7 @@ describe("processEmail — unsubscribe capture", () => {
     );
     await processEmail(
       makeInput({
+        subject: "Issue 1 from B",
         senders: ["[email protected]"],
         headers: {
           "list-unsubscribe": "<https://two.com/u/1>",
@@ -729,6 +859,7 @@ describe("processEmail — unsubscribe capture", () => {
     );
     await processEmail(
       makeInput({
+        subject: "Issue 2 from A",
         senders: ["[email protected]"],
         headers: {
           "list-unsubscribe": "<https://one.com/u/2>",

src/application/email-processor.ts

@@ -109,12 +109,62 @@ async function loadAcceptingFeed(
   return { ok: true, feed };
 }
 
+/**
+ * Compute a SHA-256 hex digest of a normalised string combining subject and
+ * content. Used as a dedup fallback when no Message-ID header is present.
+ * "Normalised" means lower-cased and all whitespace runs collapsed to a single
+ * space — so minor whitespace differences in re-sent mails still match.
+ */
+async function computeDedupHash(
+  subject: string,
+  content: string,
+): Promise<string> {
+  const normalize = (s: string) => s.toLowerCase().replace(/\s+/g, " ").trim();
+  const raw = `${normalize(subject)}\n${normalize(content)}`;
+  const buf = await crypto.subtle.digest(
+    "SHA-256",
+    new TextEncoder().encode(raw),
+  );
+  return Array.from(new Uint8Array(buf))
+    .map((b) => b.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+/**
+ * Extract the Message-ID from request headers (case-insensitive key lookup).
+ * Returns undefined when absent or empty.
+ */
+function extractMessageId(
+  headers: Record<string, string> | undefined,
+): string | undefined {
+  if (!headers) return undefined;
+  const value = Object.entries(headers).find(
+    ([k]) => k.toLowerCase() === "message-id",
+  )?.[1];
+  const trimmed = value?.trim();
+  return trimmed || undefined;
+}
+
 async function storeEmail(
   feed: Feed,
   input: ProcessEmailInput,
   env: Env,
   ctx?: ExecutionContext,
-): Promise<void> {
+): Promise<boolean> {
+  // ── Dedup check ──────────────────────────────────────────────────────────
+  // Compute both dedup signals up-front (hash is async) so we only do it once.
+  const messageId = extractMessageId(input.headers);
+  const dedupHash = await computeDedupHash(input.subject, input.content);
+
+  if (feed.hasDuplicate(messageId, dedupHash)) {
+    logger.info("Duplicate email skipped", {
+      feedId: feed.id.value,
+      ...(messageId ? { messageId } : { dedupHash }),
+    });
+    await bumpCounters(env.EMAIL_STORAGE, { emails_deduplicated: 1 });
+    return false; // signal: skipped (not stored)
+  }
+
   const attachmentBucket = getAttachmentBucket(env);
   const inlineCids = extractInlineCids(input.content);
   const storedAttachments: AttachmentData[] =
@@ -149,6 +199,8 @@ async function storeEmail(
     size: serialisedSize,
     ...(downloadableIds.length > 0 ? { attachmentIds: downloadableIds } : {}),
     ...(inlineIds.length > 0 ? { inlineAttachmentIds: inlineIds } : {}),
+    ...(messageId ? { messageId } : {}),
+    dedupHash,
   };
 
   // Track the latest sender's domain (feed icon) and capture the RFC 8058
@@ -198,6 +250,7 @@ async function storeEmail(
     ? (p) => ctx.waitUntil(p)
     : () => {};
   await dispatchFeedEvents(feed, env, schedule);
+  return true; // signal: stored
 }
 
 export async function processEmail(

src/application/stats.ts

@@ -12,6 +12,7 @@ const EMPTY_COUNTERS: Counters = {
   emails_received: 0,
   emails_rejected: 0,
   emails_forwarded: 0,
+  emails_deduplicated: 0,
   unsubscribes_sent: 0,
 };
 
@@ -43,6 +44,7 @@ export async function bumpCounters(
     current.emails_received += changes.emails_received ?? 0;
     current.emails_rejected += changes.emails_rejected ?? 0;
     current.emails_forwarded += changes.emails_forwarded ?? 0;
+    current.emails_deduplicated += changes.emails_deduplicated ?? 0;
     current.unsubscribes_sent += changes.unsubscribes_sent ?? 0;
     if (changes.last_email_at) current.last_email_at = changes.last_email_at;
     if (changes.last_feed_created_at)