mirror of
https://github.com/svemagie/blog-eleventy-indiekit.git
synced 2026-04-02 08:44:56 +02:00
fix(og): batch spawning to prevent OOM during watcher rebuilds
Full OG regeneration (2,350 images) OOM-kills when the Eleventy watcher is running (~1.8 GB RSS), leaving only ~1.2 GB headroom in the 3 GB container. WASM native memory from Satori/Resvg grows beyond what GC can reclaim within a single process. Solution: spawn og-cli in batches of 100 images. Each invocation exits after its batch, fully releasing all WASM native memory. Exit code 2 signals "more work remains" and the spawner re-loops. Peak memory per batch stays under ~500 MB regardless of total image count. Also seed newManifest from existing manifest so unscanned entries survive batch writes. Confab-Link: http://localhost:8080/sessions/edb1b7b0-da66-4486-bd9c-d1cfa7553b88
This commit is contained in:
@@ -1114,25 +1114,46 @@ export default function (eleventyConfig) {
|
||||
return digests;
|
||||
});
|
||||
|
||||
// Generate OpenGraph images for posts without photos
|
||||
// Runs on every build (including watcher rebuilds) — manifest caching makes it fast
|
||||
// for incremental: only new posts without an OG image get generated (~200ms each)
|
||||
// Generate OpenGraph images for posts without photos.
|
||||
// Uses batch spawning: each invocation generates up to BATCH_SIZE images then exits,
|
||||
// fully releasing WASM native memory (Satori Yoga + Resvg Rust) between batches.
|
||||
// Exit code 2 = batch complete, more work remains → re-spawn.
|
||||
// Manifest caching makes incremental builds fast (only new posts get generated).
|
||||
eleventyConfig.on("eleventy.before", () => {
|
||||
const contentDir = resolve(__dirname, "content");
|
||||
const cacheDir = resolve(__dirname, ".cache");
|
||||
const siteName = process.env.SITE_NAME || "My IndieWeb Blog";
|
||||
const BATCH_SIZE = 100;
|
||||
let totalGenerated = 0;
|
||||
let batch = 0;
|
||||
try {
|
||||
execFileSync(process.execPath, [
|
||||
"--max-old-space-size=512",
|
||||
"--expose-gc",
|
||||
resolve(__dirname, "lib", "og-cli.js"),
|
||||
contentDir,
|
||||
cacheDir,
|
||||
siteName,
|
||||
], {
|
||||
stdio: "inherit",
|
||||
env: { ...process.env, NODE_OPTIONS: "" },
|
||||
});
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
batch++;
|
||||
try {
|
||||
execFileSync(process.execPath, [
|
||||
"--max-old-space-size=512",
|
||||
"--expose-gc",
|
||||
resolve(__dirname, "lib", "og-cli.js"),
|
||||
contentDir,
|
||||
cacheDir,
|
||||
siteName,
|
||||
String(BATCH_SIZE),
|
||||
], {
|
||||
stdio: "inherit",
|
||||
env: { ...process.env, NODE_OPTIONS: "" },
|
||||
});
|
||||
// Exit code 0 = all done
|
||||
break;
|
||||
} catch (err) {
|
||||
if (err.status === 2) {
|
||||
// Exit code 2 = batch complete, more images remain
|
||||
totalGenerated += BATCH_SIZE;
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Sync new OG images to output directory.
|
||||
// During incremental builds, .cache/og is in watchIgnores so Eleventy's
|
||||
|
||||
@@ -4,16 +4,27 @@
|
||||
* CLI entry point for OG image generation.
|
||||
* Runs as a separate process to isolate memory from Eleventy.
|
||||
*
|
||||
* Usage: node lib/og-cli.js <contentDir> <cacheDir> <siteName>
|
||||
* Usage: node lib/og-cli.js <contentDir> <cacheDir> <siteName> [batchSize]
|
||||
*
|
||||
* batchSize: Max images to generate per invocation (0 = unlimited).
|
||||
* When set, exits after generating that many images so the caller
|
||||
* can re-spawn (releasing all WASM native memory between batches).
|
||||
* Exit code 2 = batch complete, more work remains.
|
||||
*/
|
||||
|
||||
import { generateOgImages } from "./og.js";
|
||||
|
||||
const [contentDir, cacheDir, siteName] = process.argv.slice(2);
|
||||
const [contentDir, cacheDir, siteName, batchSizeStr] = process.argv.slice(2);
|
||||
|
||||
if (!contentDir || !cacheDir || !siteName) {
|
||||
console.error("[og] Usage: node og-cli.js <contentDir> <cacheDir> <siteName>");
|
||||
console.error("[og] Usage: node og-cli.js <contentDir> <cacheDir> <siteName> [batchSize]");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
await generateOgImages(contentDir, cacheDir, siteName);
|
||||
const batchSize = parseInt(batchSizeStr, 10) || 0;
|
||||
const result = await generateOgImages(contentDir, cacheDir, siteName, batchSize);
|
||||
|
||||
// Exit code 2 signals "batch complete, more images remain"
|
||||
if (result?.hasMore) {
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
18
lib/og.js
18
lib/og.js
@@ -278,8 +278,10 @@ function scanContentFiles(contentDir) {
|
||||
* @param {string} contentDir - Path to content/ directory
|
||||
* @param {string} cacheDir - Path to .cache/ directory
|
||||
* @param {string} siteName - Site name for the card
|
||||
* @param {number} batchSize - Max images to generate (0 = unlimited)
|
||||
* @returns {{ hasMore: boolean }} Whether more images need generation
|
||||
*/
|
||||
export async function generateOgImages(contentDir, cacheDir, siteName) {
|
||||
export async function generateOgImages(contentDir, cacheDir, siteName, batchSize = 0) {
|
||||
const ogDir = join(cacheDir, "og");
|
||||
mkdirSync(ogDir, { recursive: true });
|
||||
|
||||
@@ -296,7 +298,8 @@ export async function generateOgImages(contentDir, cacheDir, siteName) {
|
||||
|
||||
let generated = 0;
|
||||
let skipped = 0;
|
||||
const newManifest = {};
|
||||
// Seed with existing manifest so unscanned entries survive batch writes
|
||||
const newManifest = { ...manifest };
|
||||
const SAVE_INTERVAL = 10;
|
||||
// GC every 5 images to keep WASM native memory bounded.
|
||||
// Satori (Yoga WASM) + Resvg (Rust WASM) allocate ~50-100 MB native memory
|
||||
@@ -352,14 +355,25 @@ export async function generateOgImages(contentDir, cacheDir, siteName) {
|
||||
const rss = process.memoryUsage().rss;
|
||||
if (rss > peakRss) peakRss = rss;
|
||||
}
|
||||
|
||||
// Batch limit: stop after N images so the caller can re-spawn
|
||||
// (fully releasing WASM native memory between batches)
|
||||
if (batchSize > 0 && generated >= batchSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const hasMore = batchSize > 0 && generated >= batchSize;
|
||||
|
||||
if (hasGC) global.gc();
|
||||
writeFileSync(manifestPath, JSON.stringify(newManifest, null, 2));
|
||||
const mem = process.memoryUsage();
|
||||
if (mem.rss > peakRss) peakRss = mem.rss;
|
||||
console.log(
|
||||
`[og] Generated ${generated} images, skipped ${skipped} (cached or have photos)` +
|
||||
(hasMore ? ` [batch, more remain]` : ``) +
|
||||
` | RSS: ${(mem.rss / 1024 / 1024).toFixed(0)} MB, peak: ${(peakRss / 1024 / 1024).toFixed(0)} MB, heap: ${(mem.heapUsed / 1024 / 1024).toFixed(0)} MB`,
|
||||
);
|
||||
|
||||
return { hasMore };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user