fix: always fetch live page for webmention link extraction
The core bug: stored post content (post.properties.content.html) is only the post body text. Template-rendered microformat links (u-in-reply-to, u-like-of, u-bookmark-of, u-repost-of) live in the 11ty HTML output, not in MongoDB. So replies, likes, bookmarks and reposts never had their target URLs extracted — webmentions were silently skipped. - patch-webmention-sender-livefetch: always fetch the live page; fall back to stored content only if the page is unavailable; skip (don't mark sent) when no content is available so the next poll retries it. Handles both original upstream code and the older retry-patch variant. - patch-webmention-sender-reset-stale: bump to v2 so posts incorrectly marked as sent today (empty results due to the content bug) get reset and retried on next deploy. - Remove patch-webmention-sender-retry: superseded by livefetch. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
svemagie
144
scripts/patch-webmention-sender-livefetch.mjs
Normal file
144
scripts/patch-webmention-sender-livefetch.mjs
Normal file
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* Patch @rmdes/indiekit-endpoint-webmention-sender controller to:
|
||||
*
|
||||
* 1. Always fetch the live page instead of using stored post content.
|
||||
* The stored content (post.properties.content.html) is just the post body —
|
||||
* it never contains template-rendered links like u-in-reply-to, u-like-of,
|
||||
* u-bookmark-of, u-repost-of. Only the live HTML has those.
|
||||
*
|
||||
* 2. Don't permanently mark a post as webmention-sent when the live page
|
||||
* is unreachable (e.g. deploy still in progress). Skip it silently so
|
||||
* the next poll retries it.
|
||||
*
|
||||
* Handles both the original upstream code and the older patch-webmention-sender-retry
|
||||
* variant (which only fixed the no-content case but not the live-fetch case).
|
||||
*/
|
||||
|
||||
import { access, readFile, writeFile } from "node:fs/promises";
|
||||
|
||||
const filePath =
|
||||
"node_modules/@rmdes/indiekit-endpoint-webmention-sender/lib/controllers/webmention-sender.js";
|
||||
|
||||
const patchMarker = "// [patched:livefetch]";
|
||||
|
||||
// Original upstream code
|
||||
const originalBlock = ` // If no content, try fetching the published page
|
||||
let contentToProcess = postContent;
|
||||
if (!contentToProcess) {
|
||||
try {
|
||||
const pageResponse = await fetch(postUrl);
|
||||
if (pageResponse.ok) {
|
||||
contentToProcess = await pageResponse.text();
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(\`[webmention] Could not fetch \${postUrl}: \${error.message}\`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!contentToProcess) {
|
||||
console.log(\`[webmention] No content to process for \${postUrl}\`);
|
||||
await markWebmentionsSent(postsCollection, postUrl, { sent: [], failed: [], skipped: [] });
|
||||
continue;
|
||||
}`;
|
||||
|
||||
// State left by older patch-webmention-sender-retry.mjs (which only fixed the
|
||||
// fetch-failure path but not the live-fetch-always path)
|
||||
const retryPatchedBlock = ` // If no content, try fetching the published page
|
||||
let contentToProcess = postContent;
|
||||
let fetchFailed = false;
|
||||
if (!contentToProcess) {
|
||||
try {
|
||||
const pageResponse = await fetch(postUrl);
|
||||
if (pageResponse.ok) {
|
||||
contentToProcess = await pageResponse.text();
|
||||
} else {
|
||||
fetchFailed = true;
|
||||
}
|
||||
} catch (error) {
|
||||
fetchFailed = true;
|
||||
console.log(\`[webmention] Could not fetch \${postUrl}: \${error.message}\`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!contentToProcess) {
|
||||
if (fetchFailed) {
|
||||
// Page not yet available — skip and retry on next poll rather than
|
||||
// permanently marking this post as sent with zero webmentions.
|
||||
console.log(\`[webmention] Page not yet available for \${postUrl}, will retry next poll\`);
|
||||
continue;
|
||||
}
|
||||
console.log(\`[webmention] No content to process for \${postUrl}\`);
|
||||
await markWebmentionsSent(postsCollection, postUrl, { sent: [], failed: [], skipped: [] });
|
||||
continue;
|
||||
}`;
|
||||
|
||||
const newBlock = ` // [patched:livefetch] Always fetch the live page so template-rendered links
|
||||
// (u-in-reply-to, u-like-of, u-bookmark-of, u-repost-of, etc.) are included.
|
||||
// Stored content only has the post body, not these microformat links.
|
||||
let contentToProcess = "";
|
||||
try {
|
||||
const pageResponse = await fetch(postUrl);
|
||||
if (pageResponse.ok) {
|
||||
contentToProcess = await pageResponse.text();
|
||||
} else {
|
||||
console.log(\`[webmention] Live page returned \${pageResponse.status} for \${postUrl}\`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(\`[webmention] Could not fetch \${postUrl}: \${error.message}\`);
|
||||
}
|
||||
|
||||
// Fall back to stored content if live page is unavailable
|
||||
if (!contentToProcess) {
|
||||
contentToProcess = postContent;
|
||||
}
|
||||
|
||||
if (!contentToProcess) {
|
||||
// Page not reachable yet (deploy in progress?) — skip without marking sent
|
||||
// so the next poll retries it.
|
||||
console.log(\`[webmention] No content available for \${postUrl}, will retry next poll\`);
|
||||
continue;
|
||||
}`;
|
||||
|
||||
async function exists(p) {
|
||||
try {
|
||||
await access(p);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!(await exists(filePath))) {
|
||||
console.log("[patch-webmention-sender-livefetch] File not found, skipping");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const source = await readFile(filePath, "utf8");
|
||||
|
||||
if (source.includes(patchMarker)) {
|
||||
console.log("[patch-webmention-sender-livefetch] Already patched");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const targetBlock = source.includes(originalBlock)
|
||||
? originalBlock
|
||||
: source.includes(retryPatchedBlock)
|
||||
? retryPatchedBlock
|
||||
: null;
|
||||
|
||||
if (!targetBlock) {
|
||||
console.warn(
|
||||
"[patch-webmention-sender-livefetch] Target block not found — upstream format may have changed, skipping"
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const patched = source.replace(targetBlock, newBlock);
|
||||
|
||||
if (!patched.includes(patchMarker)) {
|
||||
console.warn("[patch-webmention-sender-livefetch] Patch validation failed, skipping");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
await writeFile(filePath, patched, "utf8");
|
||||
console.log("[patch-webmention-sender-livefetch] Patched successfully");
|
||||
@@ -9,7 +9,7 @@
|
||||
import { MongoClient } from "mongodb";
|
||||
import config from "../indiekit.config.mjs";
|
||||
|
||||
const MIGRATION_ID = "webmention-sender-reset-stale-v1";
|
||||
const MIGRATION_ID = "webmention-sender-reset-stale-v2";
|
||||
|
||||
const mongodbUrl = config.application?.mongodbUrl;
|
||||
if (!mongodbUrl) {
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
/**
|
||||
* Patch @rmdes/indiekit-endpoint-webmention-sender controller to not silently
|
||||
* mark posts as webmention-sent when the live page fetch fails.
|
||||
*
|
||||
* Root cause: when a post has no stored content (likes, bookmarks, reposts),
|
||||
* the controller tries to fetch the published URL. If the fetch fails (page not
|
||||
* yet deployed), it marks the post as webmention-sent with empty results — and
|
||||
* it is never retried. This patch skips those posts instead so they are picked
|
||||
* up on the next poll once the page is live.
|
||||
*/
|
||||
|
||||
import { access, readFile, writeFile } from "node:fs/promises";
|
||||
|
||||
const candidates = [
|
||||
"node_modules/@rmdes/indiekit-endpoint-webmention-sender/lib/controllers/webmention-sender.js",
|
||||
];
|
||||
|
||||
const marker = "Page not yet available";
|
||||
|
||||
const oldSnippet = ` // If no content, try fetching the published page
|
||||
let contentToProcess = postContent;
|
||||
if (!contentToProcess) {
|
||||
try {
|
||||
const pageResponse = await fetch(postUrl);
|
||||
if (pageResponse.ok) {
|
||||
contentToProcess = await pageResponse.text();
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(\`[webmention] Could not fetch \${postUrl}: \${error.message}\`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!contentToProcess) {
|
||||
console.log(\`[webmention] No content to process for \${postUrl}\`);
|
||||
await markWebmentionsSent(postsCollection, postUrl, { sent: [], failed: [], skipped: [] });
|
||||
continue;
|
||||
}`;
|
||||
|
||||
const newSnippet = ` // If no content, try fetching the published page
|
||||
let contentToProcess = postContent;
|
||||
let fetchFailed = false;
|
||||
if (!contentToProcess) {
|
||||
try {
|
||||
const pageResponse = await fetch(postUrl);
|
||||
if (pageResponse.ok) {
|
||||
contentToProcess = await pageResponse.text();
|
||||
} else {
|
||||
fetchFailed = true;
|
||||
}
|
||||
} catch (error) {
|
||||
fetchFailed = true;
|
||||
console.log(\`[webmention] Could not fetch \${postUrl}: \${error.message}\`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!contentToProcess) {
|
||||
if (fetchFailed) {
|
||||
// Page not yet available — skip and retry on next poll rather than
|
||||
// permanently marking this post as sent with zero webmentions.
|
||||
console.log(\`[webmention] Page not yet available for \${postUrl}, will retry next poll\`);
|
||||
continue;
|
||||
}
|
||||
console.log(\`[webmention] No content to process for \${postUrl}\`);
|
||||
await markWebmentionsSent(postsCollection, postUrl, { sent: [], failed: [], skipped: [] });
|
||||
continue;
|
||||
}`;
|
||||
|
||||
async function exists(filePath) {
|
||||
try {
|
||||
await access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
let filesChecked = 0;
|
||||
let filesPatched = 0;
|
||||
|
||||
for (const filePath of candidates) {
|
||||
if (!(await exists(filePath))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
filesChecked += 1;
|
||||
|
||||
const source = await readFile(filePath, "utf8");
|
||||
|
||||
if (source.includes(marker)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!source.includes(oldSnippet)) {
|
||||
console.log(`[patch] webmention-sender-retry: target snippet not found in ${filePath} (package updated?)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const updated = source.replace(oldSnippet, newSnippet);
|
||||
await writeFile(filePath, updated, "utf8");
|
||||
filesPatched += 1;
|
||||
}
|
||||
|
||||
if (filesChecked === 0) {
|
||||
console.log("[patch] webmention-sender-retry: package file not found");
|
||||
} else if (filesPatched === 0) {
|
||||
console.log("[patch] webmention-sender-retry: already applied");
|
||||
} else {
|
||||
console.log(`[patch] webmention-sender-retry: patched ${filesPatched}/${filesChecked} file(s)`);
|
||||
}
|
||||
Reference in New Issue
Block a user