svemagie/blog-eleventy-indiekit
1
0
Fork 0
mirror of https://github.com/svemagie/blog-eleventy-indiekit.git synced 2026-04-02 08:44:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf: batch unfurl pre-fetch to reduce peak memory

Browse Source 
Replace unbounded Promise.all on 545 interaction URLs with batches
of 50. Add GC calls after the markdown walk and between batches to
free parsed content and resolved promise data before the render phase.

Logs RSS every 5th batch for memory monitoring.

Same pattern as the OG image batch spawning fix.

Confab-Link: http://localhost:8080/sessions/0b241cd6-aff2-4fec-853c-2b5a61e61946
This commit is contained in:
Ricardo
2026-03-10 14:44:51 +01:00
parent 48160a5b13
commit 129e0720af
1 changed files with 18 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
eleventy.config.js
View File

@@ -1205,10 +1205,26 @@ export default function (eleventyConfig) {
}
};
walk(contentDir);
// Free parsed markdown content before starting network-heavy prefetch
if (typeof global.gc === "function") global.gc();
if (urls.size === 0) return;
console.log(`[unfurl] Pre-fetching ${urls.size} interaction URLs...`);
await Promise.all([...urls].map((url) => prefetchUrl(url)));
const urlArray = [...urls];
const UNFURL_BATCH = 50;
const totalBatches = Math.ceil(urlArray.length / UNFURL_BATCH);
console.log(`[unfurl] Pre-fetching ${urlArray.length} interaction URLs (${totalBatches} batches of ${UNFURL_BATCH})...`);
let fetched = 0;
for (let i = 0; i < urlArray.length; i += UNFURL_BATCH) {
const batch = urlArray.slice(i, i + UNFURL_BATCH);
const batchNum = Math.floor(i / UNFURL_BATCH) + 1;
await Promise.all(batch.map((url) => prefetchUrl(url)));
fetched += batch.length;
if (typeof global.gc === "function") global.gc();
if (batchNum === 1 || batchNum % 5 === 0 || batchNum === totalBatches) {
const rss = (process.memoryUsage.rss() / 1024 / 1024).toFixed(0);
console.log(`[unfurl] Batch ${batchNum}/${totalBatches} (${fetched}/${urlArray.length}) | RSS: ${rss} MB`);
}
}
console.log(`[unfurl] Pre-fetch complete.`);
});