refactor(domain): introduce FeedRepository as the single KV access layer
Centralise the KV key schema and all get/put access behind a FeedRepository class under src/domain/. Every feed/email/list/icon/websub/counter key was previously inlined across ~12 modules with two divergent storeEmail and addFeedToList implementations; the dead src/utils/storage.ts write path is removed and the email key convention unified on feed:<id>:<ts>. Behaviour-preserving: existing tests pass unchanged in logic, plus a new feed-repository.test.ts covering CRUD, key builders, list ops and counters. Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
Julien HerrandClaude Opus 4.7 committed
1 parent
a0eaebe749
commit
2b3f00f7e3
22 files changed
+616
-539
No files matched your search
@@ -6,8 +6,9 @@ import {
|
||||
extractEmailDomain,
|
||||
getCachedIcon,
|
||||
} from "./favicon-fetcher";
|
||||
import { iconKey } from "./storage";
|
||||
import { MAX_ICON_BYTES } from "../config/constants";
|
||||
|
||||
const iconKey = (domain: string) => `icon:${domain}`;
|
||||
import type { Env } from "../types";
|
||||
|
||||
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]);
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
ICON_TTL_SECONDS,
|
||||
MAX_ICON_BYTES,
|
||||
} from "../config/constants";
|
||||
import { iconKey } from "./storage";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
import { logger } from "../lib/logger";
|
||||
|
||||
interface IconRecord {
|
||||
@@ -92,8 +92,8 @@ export async function cacheFaviconForDomain(
|
||||
env: Env,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const key = iconKey(domain);
|
||||
const existing = await env.EMAIL_STORAGE.get(key, "text");
|
||||
const repo = FeedRepository.from(env);
|
||||
const existing = await repo.getIconText(domain);
|
||||
if (existing !== null) return; // present (incl. negative) → nothing to do
|
||||
|
||||
const icon = await resolveIcon(domain);
|
||||
@@ -104,9 +104,7 @@ export async function cacheFaviconForDomain(
|
||||
}
|
||||
: { data: null, contentType: "" };
|
||||
|
||||
await env.EMAIL_STORAGE.put(key, JSON.stringify(record), {
|
||||
expirationTtl: ICON_TTL_SECONDS,
|
||||
});
|
||||
await repo.putIcon(domain, JSON.stringify(record), ICON_TTL_SECONDS);
|
||||
} catch (error) {
|
||||
logger.warn("Favicon cache failed", { domain, error: String(error) });
|
||||
}
|
||||
@@ -119,10 +117,7 @@ export async function getCachedIcon(
|
||||
domain: string,
|
||||
env: Env,
|
||||
): Promise<{ bytes: ArrayBuffer; contentType: string } | null> {
|
||||
const record = (await env.EMAIL_STORAGE.get(
|
||||
iconKey(domain),
|
||||
"json",
|
||||
)) as IconRecord | null;
|
||||
const record = await FeedRepository.from(env).getIconJson<IconRecord>(domain);
|
||||
if (!record || record.data === null) return null;
|
||||
return {
|
||||
bytes: base64ToArrayBuffer(record.data),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Env, FeedConfig, FeedMetadata, EmailData } from "../types";
|
||||
import { Env, FeedConfig, EmailData } from "../types";
|
||||
import { MAX_FEED_ITEMS } from "../config/constants";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
|
||||
export interface FeedData {
|
||||
feedConfig: FeedConfig;
|
||||
@@ -10,19 +11,12 @@ export async function fetchFeedData(
|
||||
feedId: string,
|
||||
env: Env,
|
||||
): Promise<FeedData | null> {
|
||||
const storage = env.EMAIL_STORAGE;
|
||||
|
||||
const feedMetadata = (await storage.get(
|
||||
`feed:${feedId}:metadata`,
|
||||
"json",
|
||||
)) as FeedMetadata | null;
|
||||
const repo = FeedRepository.from(env);
|
||||
|
||||
const feedMetadata = await repo.getMetadata(feedId);
|
||||
if (!feedMetadata) return null;
|
||||
|
||||
const feedConfig = ((await storage.get(
|
||||
`feed:${feedId}:config`,
|
||||
"json",
|
||||
)) as FeedConfig | null) ?? {
|
||||
const feedConfig = (await repo.getConfig(feedId)) ?? {
|
||||
title: `Newsletter Feed ${feedId}`,
|
||||
description: "Converted email newsletter",
|
||||
language: "en",
|
||||
@@ -32,7 +26,7 @@ export async function fetchFeedData(
|
||||
const emailRefs = feedMetadata.emails.slice(0, MAX_FEED_ITEMS);
|
||||
const emails: EmailData[] = [];
|
||||
for (const ref of emailRefs) {
|
||||
const data = (await storage.get(ref.key, "json")) as EmailData | null;
|
||||
const data = await repo.getEmail(ref.key);
|
||||
if (data) emails.push(data);
|
||||
}
|
||||
|
||||
|
||||
+12
-26
@@ -1,8 +1,6 @@
|
||||
import { Counters, Env, StatsResponse } from "../types";
|
||||
import { STATS_KEY } from "../config/constants";
|
||||
import { logger } from "../lib/logger";
|
||||
import { listAllFeeds } from "../routes/admin/helpers";
|
||||
import { getFeedMetadata } from "./storage";
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
import { getAttachmentBucket } from "./attachments";
|
||||
|
||||
const EMPTY_COUNTERS: Counters = {
|
||||
@@ -15,9 +13,7 @@ const EMPTY_COUNTERS: Counters = {
|
||||
|
||||
export async function getCounters(kv: KVNamespace): Promise<Counters> {
|
||||
try {
|
||||
const stored = (await kv.get(STATS_KEY, {
|
||||
type: "json",
|
||||
})) as Counters | null;
|
||||
const stored = await new FeedRepository(kv).getCountersRaw();
|
||||
return { ...EMPTY_COUNTERS, ...(stored || {}) };
|
||||
} catch (error) {
|
||||
logger.error("Error reading counters", { error: String(error) });
|
||||
@@ -48,7 +44,7 @@ export async function bumpCounters(
|
||||
current.last_feed_created_at = changes.last_feed_created_at;
|
||||
if (!current.first_seen) current.first_seen = new Date().toISOString();
|
||||
|
||||
await kv.put(STATS_KEY, JSON.stringify(current));
|
||||
await new FeedRepository(kv).putCounters(current);
|
||||
} catch (error) {
|
||||
logger.error("Error updating counters", { error: String(error) });
|
||||
}
|
||||
@@ -58,26 +54,15 @@ export async function countKeysByPrefix(
|
||||
kv: KVNamespace,
|
||||
prefix: string,
|
||||
): Promise<number> {
|
||||
let total = 0;
|
||||
let cursor: string | undefined;
|
||||
try {
|
||||
do {
|
||||
const listed = await kv.list({ prefix, cursor, limit: 1000 });
|
||||
total += listed.keys.length;
|
||||
cursor = listed.list_complete ? undefined : listed.cursor;
|
||||
} while (cursor);
|
||||
} catch (error) {
|
||||
logger.error("Error counting keys", { prefix, error: String(error) });
|
||||
}
|
||||
return total;
|
||||
return new FeedRepository(kv).countKeysByPrefix(prefix);
|
||||
}
|
||||
|
||||
export async function getStats(env: Env): Promise<StatsResponse> {
|
||||
const kv = env.EMAIL_STORAGE;
|
||||
const repo = FeedRepository.from(env);
|
||||
const [counters, feeds, websubCount] = await Promise.all([
|
||||
getCounters(kv),
|
||||
listAllFeeds(kv),
|
||||
countKeysByPrefix(kv, "websub:"),
|
||||
getCounters(env.EMAIL_STORAGE),
|
||||
repo.listFeeds(),
|
||||
repo.countSubscriptionKeys(),
|
||||
]);
|
||||
|
||||
return {
|
||||
@@ -119,9 +104,10 @@ export async function scanR2Usage(
|
||||
export async function scanKvUsage(kv: KVNamespace): Promise<{ bytes: number }> {
|
||||
let bytes = 0;
|
||||
try {
|
||||
const feeds = await listAllFeeds(kv);
|
||||
const repo = new FeedRepository(kv);
|
||||
const feeds = await repo.listFeeds();
|
||||
for (const feed of feeds) {
|
||||
const metadata = await getFeedMetadata(kv, feed.id);
|
||||
const metadata = await repo.getMetadata(feed.id);
|
||||
if (!metadata) continue;
|
||||
for (const email of metadata.emails) {
|
||||
bytes += email.size ?? 0;
|
||||
@@ -152,7 +138,7 @@ export async function setStorageSnapshot(
|
||||
current.kv_bytes_estimated = snapshot.kv_bytes_estimated;
|
||||
current.storage_scanned_at = new Date().toISOString();
|
||||
if (!current.first_seen) current.first_seen = new Date().toISOString();
|
||||
await kv.put(STATS_KEY, JSON.stringify(current));
|
||||
await new FeedRepository(kv).putCounters(current);
|
||||
} catch (error) {
|
||||
logger.error("Error writing storage snapshot", { error: String(error) });
|
||||
}
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
import {
|
||||
EmailData,
|
||||
FeedConfig,
|
||||
FeedMetadata,
|
||||
FeedList,
|
||||
EmailMetadata,
|
||||
} from "../types";
|
||||
import { MAX_METADATA_EMAILS } from "../config/constants";
|
||||
|
||||
/**
|
||||
* KV key for a domain's cached favicon (shared across feeds from the same sender).
|
||||
*/
|
||||
export function iconKey(domain: string): string {
|
||||
return `icon:${domain}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store email data in KV
|
||||
*/
|
||||
export async function storeEmail(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
emailData: EmailData,
|
||||
): Promise<string> {
|
||||
// Generate a unique key for this email
|
||||
const timestamp = Date.now();
|
||||
const key = `feed:${feedId}:email:${timestamp}`;
|
||||
|
||||
// Store the email content
|
||||
await kv.put(key, JSON.stringify(emailData));
|
||||
|
||||
// Update the feed's metadata (list of emails)
|
||||
await updateFeedMetadata(kv, feedId, {
|
||||
key,
|
||||
subject: emailData.subject,
|
||||
receivedAt: timestamp,
|
||||
});
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update feed metadata with a new email
|
||||
*/
|
||||
async function updateFeedMetadata(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
emailMetadata: EmailMetadata,
|
||||
): Promise<void> {
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
const existingMetadata = (await kv.get(feedMetadataKey, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
|
||||
const metadata: FeedMetadata = existingMetadata || { emails: [] };
|
||||
|
||||
// Add new email to the beginning of the list
|
||||
metadata.emails.unshift(emailMetadata);
|
||||
|
||||
// Keep only the last MAX_METADATA_EMAILS in the metadata; delete orphaned KV entries
|
||||
const toDelete =
|
||||
metadata.emails.length > MAX_METADATA_EMAILS
|
||||
? metadata.emails.slice(MAX_METADATA_EMAILS)
|
||||
: [];
|
||||
if (toDelete.length > 0) {
|
||||
metadata.emails = metadata.emails.slice(0, MAX_METADATA_EMAILS);
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
kv.put(feedMetadataKey, JSON.stringify(metadata)),
|
||||
...toDelete.map((e) => kv.delete(e.key)),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get feed metadata
|
||||
*/
|
||||
export async function getFeedMetadata(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
): Promise<FeedMetadata | null> {
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
return (await kv.get(feedMetadataKey, {
|
||||
type: "json",
|
||||
})) as FeedMetadata | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get feed configuration
|
||||
*/
|
||||
export async function getFeedConfig(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
): Promise<FeedConfig | null> {
|
||||
const feedConfigKey = `feed:${feedId}:config`;
|
||||
return (await kv.get(feedConfigKey, { type: "json" })) as FeedConfig | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get email data
|
||||
*/
|
||||
export async function getEmailData(
|
||||
kv: KVNamespace,
|
||||
key: string,
|
||||
): Promise<EmailData | null> {
|
||||
return (await kv.get(key, { type: "json" })) as EmailData | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new feed
|
||||
*/
|
||||
export async function createFeed(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
feedConfig: FeedConfig,
|
||||
): Promise<void> {
|
||||
// Store feed configuration
|
||||
const feedConfigKey = `feed:${feedId}:config`;
|
||||
await kv.put(feedConfigKey, JSON.stringify(feedConfig));
|
||||
|
||||
// Create empty metadata for the feed
|
||||
const feedMetadataKey = `feed:${feedId}:metadata`;
|
||||
await kv.put(
|
||||
feedMetadataKey,
|
||||
JSON.stringify({
|
||||
emails: [],
|
||||
}),
|
||||
);
|
||||
|
||||
// Add feed to the list of all feeds
|
||||
await addFeedToList(kv, feedId, feedConfig.title, feedConfig.description);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a feed to the global list
|
||||
*/
|
||||
export async function addFeedToList(
|
||||
kv: KVNamespace,
|
||||
feedId: string,
|
||||
title: string,
|
||||
description?: string,
|
||||
): Promise<void> {
|
||||
const feedListKey = "feeds:list";
|
||||
const existingList = (await kv.get(feedListKey, {
|
||||
type: "json",
|
||||
})) as FeedList | null;
|
||||
|
||||
const feedList: FeedList = existingList || { feeds: [] };
|
||||
|
||||
feedList.feeds.push({
|
||||
id: feedId,
|
||||
title,
|
||||
description,
|
||||
});
|
||||
|
||||
await kv.put(feedListKey, JSON.stringify(feedList));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all feeds
|
||||
*/
|
||||
export async function getAllFeeds(kv: KVNamespace): Promise<FeedList> {
|
||||
const feedListKey = "feeds:list";
|
||||
const feedList = (await kv.get(feedListKey, {
|
||||
type: "json",
|
||||
})) as FeedList | null;
|
||||
|
||||
return feedList || { feeds: [] };
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
notifySubscribers,
|
||||
verifyAndStoreSubscription,
|
||||
verifyAndDeleteSubscription,
|
||||
subscriptionKey,
|
||||
} from "./websub";
|
||||
import type { Env, WebSubSubscription } from "../types";
|
||||
|
||||
@@ -51,8 +50,12 @@ describe("getSubscriptions / saveSubscriptions", () => {
|
||||
expect(await getSubscriptions("feed1", env)).toEqual(subs);
|
||||
});
|
||||
|
||||
it("uses the correct KV key", () => {
|
||||
expect(subscriptionKey("abc")).toBe("websub:subs:abc");
|
||||
it("uses the correct KV key", async () => {
|
||||
const env = mockEnv();
|
||||
await saveSubscriptions("abc", [], env);
|
||||
expect(
|
||||
await env.EMAIL_STORAGE.get("websub:subs:abc", { type: "json" }),
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+10
-30
@@ -1,25 +1,13 @@
|
||||
import {
|
||||
Env,
|
||||
FeedConfig,
|
||||
FeedMetadata,
|
||||
EmailData,
|
||||
WebSubSubscription,
|
||||
} from "../types";
|
||||
import { Env, FeedConfig, EmailData, WebSubSubscription } from "../types";
|
||||
import { generateRssFeed, generateAtomFeed } from "./feed-generator";
|
||||
import { baseUrl, feedRssUrl, feedAtomUrl, feedUrl } from "./urls";
|
||||
|
||||
const KV_PREFIX = "websub:subs:";
|
||||
|
||||
export function subscriptionKey(feedId: string): string {
|
||||
return `${KV_PREFIX}${feedId}`;
|
||||
}
|
||||
import { FeedRepository } from "../domain/feed-repository";
|
||||
|
||||
export async function getSubscriptions(
|
||||
feedId: string,
|
||||
env: Env,
|
||||
): Promise<WebSubSubscription[]> {
|
||||
const raw = await env.EMAIL_STORAGE.get(subscriptionKey(feedId), "json");
|
||||
return (raw as WebSubSubscription[] | null) ?? [];
|
||||
return FeedRepository.from(env).getSubscriptions(feedId);
|
||||
}
|
||||
|
||||
export async function saveSubscriptions(
|
||||
@@ -27,10 +15,7 @@ export async function saveSubscriptions(
|
||||
subscriptions: WebSubSubscription[],
|
||||
env: Env,
|
||||
): Promise<void> {
|
||||
await env.EMAIL_STORAGE.put(
|
||||
subscriptionKey(feedId),
|
||||
JSON.stringify(subscriptions),
|
||||
);
|
||||
await FeedRepository.from(env).saveSubscriptions(feedId, subscriptions);
|
||||
}
|
||||
|
||||
export async function buildHmacSignature(
|
||||
@@ -60,16 +45,16 @@ async function buildFeedXml(
|
||||
env: Env,
|
||||
format: "rss" | "atom" = "rss",
|
||||
): Promise<string | null> {
|
||||
const [rawMetadata, rawConfig] = await Promise.all([
|
||||
env.EMAIL_STORAGE.get(`feed:${feedId}:metadata`, "json"),
|
||||
env.EMAIL_STORAGE.get(`feed:${feedId}:config`, "json"),
|
||||
const repo = FeedRepository.from(env);
|
||||
const [feedMetadata, rawConfig] = await Promise.all([
|
||||
repo.getMetadata(feedId),
|
||||
repo.getConfig(feedId),
|
||||
]);
|
||||
|
||||
const feedMetadata = rawMetadata as FeedMetadata | null;
|
||||
if (!feedMetadata) return null;
|
||||
|
||||
const base = baseUrl(env);
|
||||
const feedConfig = (rawConfig as FeedConfig | null) ?? {
|
||||
const feedConfig: FeedConfig = rawConfig ?? {
|
||||
title: `Newsletter Feed ${feedId}`,
|
||||
description: "Converted email newsletter",
|
||||
language: "en",
|
||||
@@ -78,12 +63,7 @@ async function buildFeedXml(
|
||||
|
||||
const emails = feedMetadata.emails.slice(0, 20);
|
||||
const emailsData = (
|
||||
await Promise.all(
|
||||
emails.map(
|
||||
(m) =>
|
||||
env.EMAIL_STORAGE.get(m.key, "json") as Promise<EmailData | null>,
|
||||
),
|
||||
)
|
||||
await Promise.all(emails.map((m) => repo.getEmail(m.key)))
|
||||
).filter((d): d is EmailData => d !== null);
|
||||
|
||||
if (format === "atom") {
|
||||
|
||||
Reference in new issue
Block a user