fix(og): batch spawning to prevent OOM during watcher rebuilds

Full OG regeneration (2,350 images) OOM-kills when the Eleventy watcher
is running (~1.8 GB RSS), leaving only ~1.2 GB headroom in the 3 GB
container. WASM native memory from Satori/Resvg grows beyond what GC
can reclaim within a single process.

Solution: spawn og-cli in batches of 100 images. Each invocation exits
after its batch, fully releasing all WASM native memory. Exit code 2
signals "more work remains" and the spawner re-loops. Peak memory per
batch stays under ~500 MB regardless of total image count.

Also seed newManifest from existing manifest so unscanned entries survive
batch writes.

Confab-Link: http://localhost:8080/sessions/edb1b7b0-da66-4486-bd9c-d1cfa7553b88
This commit is contained in:
Ricardo
2026-03-09 17:37:17 +01:00
parent 6dd8f03214
commit db10d9cfbf
3 changed files with 66 additions and 20 deletions

View File

@@ -1114,25 +1114,46 @@ export default function (eleventyConfig) {
return digests;
});
// Generate OpenGraph images for posts without photos
// Runs on every build (including watcher rebuilds) — manifest caching makes it fast
// for incremental: only new posts without an OG image get generated (~200ms each)
// Generate OpenGraph images for posts without photos.
// Uses batch spawning: each invocation generates up to BATCH_SIZE images then exits,
// fully releasing WASM native memory (Satori Yoga + Resvg Rust) between batches.
// Exit code 2 = batch complete, more work remains → re-spawn.
// Manifest caching makes incremental builds fast (only new posts get generated).
eleventyConfig.on("eleventy.before", () => {
const contentDir = resolve(__dirname, "content");
const cacheDir = resolve(__dirname, ".cache");
const siteName = process.env.SITE_NAME || "My IndieWeb Blog";
const BATCH_SIZE = 100;
let totalGenerated = 0;
let batch = 0;
try {
execFileSync(process.execPath, [
"--max-old-space-size=512",
"--expose-gc",
resolve(__dirname, "lib", "og-cli.js"),
contentDir,
cacheDir,
siteName,
], {
stdio: "inherit",
env: { ...process.env, NODE_OPTIONS: "" },
});
// eslint-disable-next-line no-constant-condition
while (true) {
batch++;
try {
execFileSync(process.execPath, [
"--max-old-space-size=512",
"--expose-gc",
resolve(__dirname, "lib", "og-cli.js"),
contentDir,
cacheDir,
siteName,
String(BATCH_SIZE),
], {
stdio: "inherit",
env: { ...process.env, NODE_OPTIONS: "" },
});
// Exit code 0 = all done
break;
} catch (err) {
if (err.status === 2) {
// Exit code 2 = batch complete, more images remain
totalGenerated += BATCH_SIZE;
continue;
}
throw err;
}
}
// Sync new OG images to output directory.
// During incremental builds, .cache/og is in watchIgnores so Eleventy's

View File

@@ -4,16 +4,27 @@
* CLI entry point for OG image generation.
* Runs as a separate process to isolate memory from Eleventy.
*
* Usage: node lib/og-cli.js <contentDir> <cacheDir> <siteName>
* Usage: node lib/og-cli.js <contentDir> <cacheDir> <siteName> [batchSize]
*
* batchSize: Max images to generate per invocation (0 = unlimited).
* When set, exits after generating that many images so the caller
* can re-spawn (releasing all WASM native memory between batches).
* Exit code 2 = batch complete, more work remains.
*/
import { generateOgImages } from "./og.js";
const [contentDir, cacheDir, siteName] = process.argv.slice(2);
const [contentDir, cacheDir, siteName, batchSizeStr] = process.argv.slice(2);
if (!contentDir || !cacheDir || !siteName) {
console.error("[og] Usage: node og-cli.js <contentDir> <cacheDir> <siteName>");
console.error("[og] Usage: node og-cli.js <contentDir> <cacheDir> <siteName> [batchSize]");
process.exit(1);
}
await generateOgImages(contentDir, cacheDir, siteName);
const batchSize = parseInt(batchSizeStr, 10) || 0;
const result = await generateOgImages(contentDir, cacheDir, siteName, batchSize);
// Exit code 2 signals "batch complete, more images remain"
if (result?.hasMore) {
process.exit(2);
}

View File

@@ -278,8 +278,10 @@ function scanContentFiles(contentDir) {
* @param {string} contentDir - Path to content/ directory
* @param {string} cacheDir - Path to .cache/ directory
* @param {string} siteName - Site name for the card
* @param {number} batchSize - Max images to generate (0 = unlimited)
* @returns {{ hasMore: boolean }} Whether more images need generation
*/
export async function generateOgImages(contentDir, cacheDir, siteName) {
export async function generateOgImages(contentDir, cacheDir, siteName, batchSize = 0) {
const ogDir = join(cacheDir, "og");
mkdirSync(ogDir, { recursive: true });
@@ -296,7 +298,8 @@ export async function generateOgImages(contentDir, cacheDir, siteName) {
let generated = 0;
let skipped = 0;
const newManifest = {};
// Seed with existing manifest so unscanned entries survive batch writes
const newManifest = { ...manifest };
const SAVE_INTERVAL = 10;
// GC every 5 images to keep WASM native memory bounded.
// Satori (Yoga WASM) + Resvg (Rust WASM) allocate ~50-100 MB native memory
@@ -352,14 +355,25 @@ export async function generateOgImages(contentDir, cacheDir, siteName) {
const rss = process.memoryUsage().rss;
if (rss > peakRss) peakRss = rss;
}
// Batch limit: stop after N images so the caller can re-spawn
// (fully releasing WASM native memory between batches)
if (batchSize > 0 && generated >= batchSize) {
break;
}
}
const hasMore = batchSize > 0 && generated >= batchSize;
if (hasGC) global.gc();
writeFileSync(manifestPath, JSON.stringify(newManifest, null, 2));
const mem = process.memoryUsage();
if (mem.rss > peakRss) peakRss = mem.rss;
console.log(
`[og] Generated ${generated} images, skipped ${skipped} (cached or have photos)` +
(hasMore ? ` [batch, more remain]` : ``) +
` | RSS: ${(mem.rss / 1024 / 1024).toFixed(0)} MB, peak: ${(peakRss / 1024 / 1024).toFixed(0)} MB, heap: ${(mem.heapUsed / 1024 / 1024).toFixed(0)} MB`,
);
return { hasMore };
}