Worker 410 com liveness check

1/**
2 * worker.js — Cap. 22: "Aparecer ou Sumir"
3 *
4 * Cloudflare Worker (ES module) that returns HTTP 410 Gone at the edge for a
5 * curated list of dead URL paths — BUT only after confirming with the origin
6 * that the content is really gone (liveness check).
7 *
8 * Why 410 instead of 404? A 410 tells Google (and other crawlers) the resource
9 * is permanently gone, so it is dropped from the index faster than a 404, which
10 * is treated as "maybe temporary". Serving it at the edge means the origin is
11 * never even hit for these paths once the verdict is cached.
12 *
13 * The dead-URL list is NOT hardcoded in this file: it is LOADED from the bundled
14 * `urls-removidas.txt` (embedded below as URLS_REMOVIDAS_TXT and parsed at
15 * startup by parseRemovedUrls). Edit the .txt, redeploy, and the Worker picks up
16 * the new list — no code changes. Extra paths can still be appended at runtime
17 * via env.GONE_PATHS (comma-separated).
18 *
19 * The safety net — LIVENESS CHECK:
20 * Before serving 410, the Worker asks the origin (WordPress REST API) whether
21 * the content is *really* gone. If WordPress unexpectedly answers 200, the page
22 * was probably revived / republished, so we DO NOT serve 410 — we pass the
23 * request through to origin. This prevents us from nuking content that came
24 * back to life. The liveness result is cached briefly (via the Cache API) so we
25 * don't hammer the origin on every request.
26 *
27 * Configuration comes from `env` (see wrangler.toml.example):
28 *   env.WP_API      -> base URL of the WordPress REST API,
29 *                      e.g. "https://origin.exemplo.com"  (TODO: set this)
30 *   env.GONE_PATHS  -> (optional) comma-separated extra paths to treat as gone.
31 */ /**
32 * The removed-URL list, embedded verbatim from `urls-removidas.txt`.
33 * Keeping it inline means the Worker bundle is fully self-contained (Workers
34 * have no filesystem at runtime) while the .txt in the repo stays the single
35 * editable source of truth — the two are shipped together in the same module.
36 */ const URLS_REMOVIDAS_TXT = `# urls-removidas.txt — Cap. 22
37# Uma URL removida por linha (apenas o path, sem domínio).
38# Linhas em branco e linhas iniciadas por "#" são ignoradas.
39# Comparação é case-insensitive e ignora a barra final.
40# TODO: troque estes exemplos pelas suas próprias URLs aposentadas.
41
42/promo-black-friday-2019
43/produto/kit-antigo-descontinuado
44/blog/post-que-foi-despublicado
45/categoria/colecao-verao-2020
46/servicos/plano-legado
47/lp/campanha-encerrada
48/loja/estoque-liquidado
49/eventos/webinar-2021-03
50`;
51/** How long (seconds) to cache a liveness decision at the edge. */ const LIVENESS_TTL_SECONDS = 300; // 5 minutes
52/** Normalises a pathname for comparison: lowercase, no trailing slash. */ function normalisePath(pathname) {
53    let p = pathname.toLowerCase();
54    if (p.length > 1 && p.endsWith("/")) p = p.slice(0, -1);
55    return p;
56}
57/**
58 * Parses the contents of `urls-removidas.txt` into a Set of normalised paths.
59 * Ignores blank lines and comment lines (starting with "#"). This is the small
60 * embedded parser that lets the .txt drive the Worker's behaviour.
61 */ function parseRemovedUrls(text) {
62    const set = new Set();
63    for (const rawLine of text.split(/\r?\n/)){
64        const line = rawLine.trim();
65        if (!line || line.startsWith("#")) continue;
66        set.add(normalisePath(line));
67    }
68    return set;
69}
70/** Dead paths loaded once from the bundled list at module init. */ const DEFAULT_GONE_PATHS = parseRemovedUrls(URLS_REMOVIDAS_TXT);
71/**
72 * Builds the Set of gone paths, merging the list parsed from the .txt with any
73 * extra paths provided via env.GONE_PATHS (comma-separated).
74 */ function buildGoneSet(env) {
75    const set = new Set(DEFAULT_GONE_PATHS);
76    if (env && typeof env.GONE_PATHS === "string" && env.GONE_PATHS.trim()) {
77        for (const raw of env.GONE_PATHS.split(",")){
78            const p = normalisePath(raw.trim());
79            if (p) set.add(p);
80        }
81    }
82    return set;
83}
84/** The 410 response body + headers actually served to clients and crawlers. */ function goneResponse() {
85    const body = "<!doctype html><html lang=\"pt-BR\"><head><meta charset=\"utf-8\">" + "<title>410 - Conteúdo removido</title></head><body>" + "<h1>410 - Conteúdo removido permanentemente</h1>" + "<p>Esta página foi removida e não está mais disponível.</p>" + "</body></html>";
86    return new Response(body, {
87        status: 410,
88        statusText: "Gone",
89        headers: {
90            "Content-Type": "text/html; charset=UTF-8",
91            // Let crawlers cache the "gone" verdict; adjust to taste.
92            "Cache-Control": "public, max-age=3600",
93            // Belt-and-suspenders: keep these out of the index even if rendered.
94            "X-Robots-Tag": "noindex"
95        }
96    });
97}
98/**
99 * Liveness check: asks the origin WordPress REST API whether the content at
100 * `pathname` still exists. Returns true if the content appears GONE (safe to
101 * serve 410), false if it looks ALIVE (should pass through).
102 *
103 * Strategy: query the WP REST API search endpoint for the slug. If WordPress
104 * returns 404 for the mapped resource, or the search yields no live match, we
105 * treat the content as gone. If it returns 200 with a live match, it is alive.
106 *
107 * Results are cached at the edge for LIVENESS_TTL_SECONDS using the Cache API,
108 * keyed by a synthetic URL, so repeated hits don't re-query the origin.
109 *
110 * On any error (origin unreachable, timeout, malformed response) we FAIL SAFE
111 * by returning false (alive) — i.e. we do NOT serve 410 if we can't confirm the
112 * content is gone. Better to serve the page than to wrongly 410 live content.
113 */ async function isContentGone(pathname, env, ctx) {
114    // Without a configured origin we cannot verify; fail safe (treat as alive).
115    if (!env || !env.WP_API) {
116        // TODO: set env.WP_API in wrangler.toml so liveness checks can run.
117        return false;
118    }
119    const cache = caches.default;
120    // Synthetic cache key — never actually fetched, just used as a cache handle.
121    const cacheKey = new Request(`https://liveness.internal/${encodeURIComponent(pathname)}`, {
122        method: "GET"
123    });
124    // 1) Try the edge cache first.
125    const cached = await cache.match(cacheKey);
126    if (cached) {
127        const verdict = await cached.text();
128        return verdict === "gone";
129    }
130    let gone;
131    try {
132        // The slug is the last path segment; used to search WordPress.
133        const slug = pathname.split("/").filter(Boolean).pop() || "";
134        const apiUrl = `${env.WP_API.replace(/\/$/, "")}` + `/wp-json/wp/v2/search?search=${encodeURIComponent(slug)}&per_page=1`;
135        const originResponse = await fetch(apiUrl, {
136            headers: {
137                Accept: "application/json"
138            },
139            cf: {
140                cacheTtl: 0
141            }
142        });
143        if (originResponse.status === 404) {
144            // Endpoint/resource explicitly missing -> gone.
145            gone = true;
146        } else if (originResponse.status === 200) {
147            const results = await originResponse.json();
148            // If WordPress returns a live match whose URL ends with our path, the
149            // content is ALIVE (revived). Otherwise treat as gone.
150            const alive = Array.isArray(results) && results.some((item)=>{
151                if (!item || typeof item.url !== "string") return false;
152                try {
153                    return normalisePath(new URL(item.url).pathname) === normalisePath(pathname);
154                } catch  {
155                    return false;
156                }
157            });
158            gone = !alive;
159        } else {
160            // Any other status (5xx, etc.) is inconclusive -> fail safe as alive.
161            gone = false;
162        }
163    } catch (error) {
164        // Network/parse failure -> fail safe: do not 410 content we can't verify.
165        // eslint-disable-next-line no-console
166        console.error("liveness check failed:", error);
167        gone = false;
168    }
169    // 2) Store the verdict in the edge cache for a short while.
170    const verdictResponse = new Response(gone ? "gone" : "alive", {
171        headers: {
172            "Cache-Control": `max-age=${LIVENESS_TTL_SECONDS}`
173        }
174    });
175    // waitUntil so caching doesn't delay the response to the user.
176    if (ctx && typeof ctx.waitUntil === "function") {
177        ctx.waitUntil(cache.put(cacheKey, verdictResponse.clone()));
178    }
179    return gone;
180}
181export default {
182    /**
183   * @param {Request} request
184   * @param {Record<string, string>} env
185   * @param {{ waitUntil: (p: Promise<unknown>) => void }} ctx
186   */ async fetch (request, env, ctx) {
187        const url = new URL(request.url);
188        const path = normalisePath(url.pathname);
189        const goneSet = buildGoneSet(env);
190        // Not a curated dead path -> nothing to do, pass straight through.
191        if (!goneSet.has(path)) {
192            return fetch(request);
193        }
194        // Curated as dead. Confirm with the origin before serving 410.
195        const gone = await isContentGone(url.pathname, env, ctx);
196        if (gone) {
197            return goneResponse();
198        }
199        // Content unexpectedly alive (or unverifiable) -> pass through to origin
200        // so we never accidentally 410 revived content.
201        return fetch(request);
202    }
203};

1# urls-removidas.txt — Cap. 22
2# Uma URL removida por linha (apenas o path, sem domínio).
3# Linhas em branco e linhas iniciadas por "#" são ignoradas.
4# Comparação é case-insensitive e ignora a barra final.
5# TODO: troque estes exemplos pelas suas próprias URLs aposentadas.
6
7/promo-black-friday-2019
8/produto/kit-antigo-descontinuado
9/blog/post-que-foi-despublicado
10/categoria/colecao-verao-2020
11/servicos/plano-legado
12/lp/campanha-encerrada
13/loja/estoque-liquidado
14/eventos/webinar-2021-03

1# wrangler.toml.example — Cap. 22: "Aparecer ou Sumir"
2#
3# Copy this file to `wrangler.toml` and fill in your own values.
4# Deploy with:  npx wrangler deploy
5#
6# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/
7
8name = "worker-410-gone"
9main = "worker.js"
10compatibility_date = "2026-01-01"
11
12# TODO: set your Cloudflare account ID (Dashboard -> Workers -> right sidebar).
13account_id = "TODO_YOUR_CLOUDFLARE_ACCOUNT_ID"
14
15# Route(s) this Worker runs on.
16# TODO: replace with your real zone / hostname pattern.
17# routes = [
18#   { pattern = "exemplo.com/*", zone_name = "exemplo.com" }
19# ]
20
21[vars]
22# TODO: base URL of your WordPress origin (used by the liveness check).
23WP_API = "https://origin.exemplo.com"
24
25# TODO (optional): extra dead paths as a comma-separated list. These are merged
26# with the list parsed from urls-removidas.txt (embedded in worker.js).
27GONE_PATHS = "/promo-antiga,/outra-url-morta"