import { unfurl } from "unfurl.js"; import { readFileSync, writeFileSync, mkdirSync, existsSync } from "fs"; import { resolve } from "path"; import { createHash } from "crypto"; const CACHE_DIR = resolve(import.meta.dirname, "..", ".cache", "unfurl"); const CACHE_DURATION_MS = 7 * 24 * 60 * 60 * 1000; // 1 week const FAILURE_CACHE_MS = 24 * 60 * 60 * 1000; // 1 day for failed fetches const USER_AGENT = "Mozilla/5.0 (compatible; Indiekit/1.0; +https://getindiekit.com)"; // Concurrency limiter — prevents overwhelming outbound network let activeRequests = 0; const MAX_CONCURRENT = 5; const queue = []; function runNext() { if (queue.length === 0 || activeRequests >= MAX_CONCURRENT) return; activeRequests++; const { resolve: res, fn } = queue.shift(); fn().then(res).finally(() => { activeRequests--; runNext(); }); } function throttled(fn) { return new Promise((res) => { queue.push({ resolve: res, fn }); runNext(); }); } function getCachePath(url) { const hash = createHash("md5").update(url).digest("hex"); return resolve(CACHE_DIR, `${hash}.json`); } export function readCache(url) { const path = getCachePath(url); if (!existsSync(path)) return undefined; // undefined = not cached try { const data = JSON.parse(readFileSync(path, "utf-8")); const age = Date.now() - data.cachedAt; const ttl = data.failed ? FAILURE_CACHE_MS : CACHE_DURATION_MS; if (age < ttl) { return data; // return full cache entry (includes .failed flag) } } catch { // Corrupt cache file, ignore } return undefined; } function writeCache(url, metadata, failed = false) { mkdirSync(CACHE_DIR, { recursive: true }); const path = getCachePath(url); writeFileSync(path, JSON.stringify({ cachedAt: Date.now(), metadata, failed })); } export function extractDomain(url) { try { return new URL(url).hostname.replace(/^www\./, ""); } catch { return url; } } export function escapeHtml(str) { if (!str) return ""; return str .replace(/&/g, "&") .replace(//g, ">") .replace(/"/g, """); } export function renderFallbackLink(url) { const domain = escapeHtml(extractDomain(url)); return `${domain}`; } export function renderCard(url, metadata) { const og = metadata.open_graph || {}; const tc = metadata.twitter_card || {}; const title = og.title || tc.title || metadata.title || extractDomain(url); const description = og.description || tc.description || metadata.description || ""; const image = og.images?.[0]?.url || tc.images?.[0]?.url || null; const favicon = metadata.favicon || null; const domain = extractDomain(url); const maxDesc = 160; const desc = description.length > maxDesc ? description.slice(0, maxDesc).trim() + "\u2026" : description; const imgHtml = image ? `