mirror of
https://github.com/svemagie/blog-eleventy-indiekit.git
synced 2026-04-02 16:44:56 +02:00
fix(og): batch spawning to prevent OOM during watcher rebuilds
Full OG regeneration (2,350 images) OOM-kills when the Eleventy watcher is running (~1.8 GB RSS), leaving only ~1.2 GB headroom in the 3 GB container. WASM native memory from Satori/Resvg grows beyond what GC can reclaim within a single process. Solution: spawn og-cli in batches of 100 images. Each invocation exits after its batch, fully releasing all WASM native memory. Exit code 2 signals "more work remains" and the spawner re-loops. Peak memory per batch stays under ~500 MB regardless of total image count. Also seed newManifest from existing manifest so unscanned entries survive batch writes. Confab-Link: http://localhost:8080/sessions/edb1b7b0-da66-4486-bd9c-d1cfa7553b88
This commit is contained in:
@@ -1114,25 +1114,46 @@ export default function (eleventyConfig) {
|
|||||||
return digests;
|
return digests;
|
||||||
});
|
});
|
||||||
|
|
||||||
// Generate OpenGraph images for posts without photos
|
// Generate OpenGraph images for posts without photos.
|
||||||
// Runs on every build (including watcher rebuilds) — manifest caching makes it fast
|
// Uses batch spawning: each invocation generates up to BATCH_SIZE images then exits,
|
||||||
// for incremental: only new posts without an OG image get generated (~200ms each)
|
// fully releasing WASM native memory (Satori Yoga + Resvg Rust) between batches.
|
||||||
|
// Exit code 2 = batch complete, more work remains → re-spawn.
|
||||||
|
// Manifest caching makes incremental builds fast (only new posts get generated).
|
||||||
eleventyConfig.on("eleventy.before", () => {
|
eleventyConfig.on("eleventy.before", () => {
|
||||||
const contentDir = resolve(__dirname, "content");
|
const contentDir = resolve(__dirname, "content");
|
||||||
const cacheDir = resolve(__dirname, ".cache");
|
const cacheDir = resolve(__dirname, ".cache");
|
||||||
const siteName = process.env.SITE_NAME || "My IndieWeb Blog";
|
const siteName = process.env.SITE_NAME || "My IndieWeb Blog";
|
||||||
|
const BATCH_SIZE = 100;
|
||||||
|
let totalGenerated = 0;
|
||||||
|
let batch = 0;
|
||||||
try {
|
try {
|
||||||
execFileSync(process.execPath, [
|
// eslint-disable-next-line no-constant-condition
|
||||||
"--max-old-space-size=512",
|
while (true) {
|
||||||
"--expose-gc",
|
batch++;
|
||||||
resolve(__dirname, "lib", "og-cli.js"),
|
try {
|
||||||
contentDir,
|
execFileSync(process.execPath, [
|
||||||
cacheDir,
|
"--max-old-space-size=512",
|
||||||
siteName,
|
"--expose-gc",
|
||||||
], {
|
resolve(__dirname, "lib", "og-cli.js"),
|
||||||
stdio: "inherit",
|
contentDir,
|
||||||
env: { ...process.env, NODE_OPTIONS: "" },
|
cacheDir,
|
||||||
});
|
siteName,
|
||||||
|
String(BATCH_SIZE),
|
||||||
|
], {
|
||||||
|
stdio: "inherit",
|
||||||
|
env: { ...process.env, NODE_OPTIONS: "" },
|
||||||
|
});
|
||||||
|
// Exit code 0 = all done
|
||||||
|
break;
|
||||||
|
} catch (err) {
|
||||||
|
if (err.status === 2) {
|
||||||
|
// Exit code 2 = batch complete, more images remain
|
||||||
|
totalGenerated += BATCH_SIZE;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Sync new OG images to output directory.
|
// Sync new OG images to output directory.
|
||||||
// During incremental builds, .cache/og is in watchIgnores so Eleventy's
|
// During incremental builds, .cache/og is in watchIgnores so Eleventy's
|
||||||
|
|||||||
@@ -4,16 +4,27 @@
|
|||||||
* CLI entry point for OG image generation.
|
* CLI entry point for OG image generation.
|
||||||
* Runs as a separate process to isolate memory from Eleventy.
|
* Runs as a separate process to isolate memory from Eleventy.
|
||||||
*
|
*
|
||||||
* Usage: node lib/og-cli.js <contentDir> <cacheDir> <siteName>
|
* Usage: node lib/og-cli.js <contentDir> <cacheDir> <siteName> [batchSize]
|
||||||
|
*
|
||||||
|
* batchSize: Max images to generate per invocation (0 = unlimited).
|
||||||
|
* When set, exits after generating that many images so the caller
|
||||||
|
* can re-spawn (releasing all WASM native memory between batches).
|
||||||
|
* Exit code 2 = batch complete, more work remains.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { generateOgImages } from "./og.js";
|
import { generateOgImages } from "./og.js";
|
||||||
|
|
||||||
const [contentDir, cacheDir, siteName] = process.argv.slice(2);
|
const [contentDir, cacheDir, siteName, batchSizeStr] = process.argv.slice(2);
|
||||||
|
|
||||||
if (!contentDir || !cacheDir || !siteName) {
|
if (!contentDir || !cacheDir || !siteName) {
|
||||||
console.error("[og] Usage: node og-cli.js <contentDir> <cacheDir> <siteName>");
|
console.error("[og] Usage: node og-cli.js <contentDir> <cacheDir> <siteName> [batchSize]");
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
await generateOgImages(contentDir, cacheDir, siteName);
|
const batchSize = parseInt(batchSizeStr, 10) || 0;
|
||||||
|
const result = await generateOgImages(contentDir, cacheDir, siteName, batchSize);
|
||||||
|
|
||||||
|
// Exit code 2 signals "batch complete, more images remain"
|
||||||
|
if (result?.hasMore) {
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
|||||||
18
lib/og.js
18
lib/og.js
@@ -278,8 +278,10 @@ function scanContentFiles(contentDir) {
|
|||||||
* @param {string} contentDir - Path to content/ directory
|
* @param {string} contentDir - Path to content/ directory
|
||||||
* @param {string} cacheDir - Path to .cache/ directory
|
* @param {string} cacheDir - Path to .cache/ directory
|
||||||
* @param {string} siteName - Site name for the card
|
* @param {string} siteName - Site name for the card
|
||||||
|
* @param {number} batchSize - Max images to generate (0 = unlimited)
|
||||||
|
* @returns {{ hasMore: boolean }} Whether more images need generation
|
||||||
*/
|
*/
|
||||||
export async function generateOgImages(contentDir, cacheDir, siteName) {
|
export async function generateOgImages(contentDir, cacheDir, siteName, batchSize = 0) {
|
||||||
const ogDir = join(cacheDir, "og");
|
const ogDir = join(cacheDir, "og");
|
||||||
mkdirSync(ogDir, { recursive: true });
|
mkdirSync(ogDir, { recursive: true });
|
||||||
|
|
||||||
@@ -296,7 +298,8 @@ export async function generateOgImages(contentDir, cacheDir, siteName) {
|
|||||||
|
|
||||||
let generated = 0;
|
let generated = 0;
|
||||||
let skipped = 0;
|
let skipped = 0;
|
||||||
const newManifest = {};
|
// Seed with existing manifest so unscanned entries survive batch writes
|
||||||
|
const newManifest = { ...manifest };
|
||||||
const SAVE_INTERVAL = 10;
|
const SAVE_INTERVAL = 10;
|
||||||
// GC every 5 images to keep WASM native memory bounded.
|
// GC every 5 images to keep WASM native memory bounded.
|
||||||
// Satori (Yoga WASM) + Resvg (Rust WASM) allocate ~50-100 MB native memory
|
// Satori (Yoga WASM) + Resvg (Rust WASM) allocate ~50-100 MB native memory
|
||||||
@@ -352,14 +355,25 @@ export async function generateOgImages(contentDir, cacheDir, siteName) {
|
|||||||
const rss = process.memoryUsage().rss;
|
const rss = process.memoryUsage().rss;
|
||||||
if (rss > peakRss) peakRss = rss;
|
if (rss > peakRss) peakRss = rss;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Batch limit: stop after N images so the caller can re-spawn
|
||||||
|
// (fully releasing WASM native memory between batches)
|
||||||
|
if (batchSize > 0 && generated >= batchSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const hasMore = batchSize > 0 && generated >= batchSize;
|
||||||
|
|
||||||
if (hasGC) global.gc();
|
if (hasGC) global.gc();
|
||||||
writeFileSync(manifestPath, JSON.stringify(newManifest, null, 2));
|
writeFileSync(manifestPath, JSON.stringify(newManifest, null, 2));
|
||||||
const mem = process.memoryUsage();
|
const mem = process.memoryUsage();
|
||||||
if (mem.rss > peakRss) peakRss = mem.rss;
|
if (mem.rss > peakRss) peakRss = mem.rss;
|
||||||
console.log(
|
console.log(
|
||||||
`[og] Generated ${generated} images, skipped ${skipped} (cached or have photos)` +
|
`[og] Generated ${generated} images, skipped ${skipped} (cached or have photos)` +
|
||||||
|
(hasMore ? ` [batch, more remain]` : ``) +
|
||||||
` | RSS: ${(mem.rss / 1024 / 1024).toFixed(0)} MB, peak: ${(peakRss / 1024 / 1024).toFixed(0)} MB, heap: ${(mem.heapUsed / 1024 / 1024).toFixed(0)} MB`,
|
` | RSS: ${(mem.rss / 1024 / 1024).toFixed(0)} MB, peak: ${(peakRss / 1024 / 1024).toFixed(0)} MB, heap: ${(mem.heapUsed / 1024 / 1024).toFixed(0)} MB`,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
return { hasMore };
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user