Worker 410 com liveness check

1/**
2 * worker.js — Cap. 22: "Aparecer ou Sumir"
3 *
4 * Cloudflare Worker (ES module) that returns HTTP 410 Gone at the edge for a
5 * curated list of dead URL paths — BUT only after confirming with the origin
6 * that the content is really gone (liveness check).
7 *
8 * Why 410 instead of 404? A 410 tells Google (and other crawlers) the resource
9 * is permanently gone, so it is dropped from the index faster than a 404, which
10 * is treated as "maybe temporary". Serving it at the edge means the origin is
11 * never even hit for these paths once the verdict is cached.
12 *
13 * The dead-URL list is NOT hardcoded in this file: it is LOADED from the bundled
14 * `urls-removidas.txt` (embedded below as URLS_REMOVIDAS_TXT and parsed at
15 * startup by parseRemovedUrls). Edit the .txt, redeploy, and the Worker picks up
16 * the new list — no code changes. Extra paths can still be appended at runtime
17 * via env.GONE_PATHS (comma-separated).
18 *
19 * The safety net — LIVENESS CHECK:
20 * Before serving 410, the Worker asks the origin (WordPress REST API) whether
21 * the content is *really* gone. If WordPress unexpectedly answers 200, the page
22 * was probably revived / republished, so we DO NOT serve 410 — we pass the
23 * request through to origin. This prevents us from nuking content that came
24 * back to life. The liveness result is cached briefly (via the Cache API) so we
25 * don't hammer the origin on every request.
26 *
27 * Configuration comes from `env` (see wrangler.toml.example):
28 *   env.WP_API      -> base URL of the WordPress REST API,
29 *                      e.g. "https://origin.exemplo.com"  (TODO: set this)
30 *   env.GONE_PATHS  -> (optional) comma-separated extra paths to treat as gone.
31 */
32
33/**
34 * The removed-URL list, embedded verbatim from `urls-removidas.txt`.
35 * Keeping it inline means the Worker bundle is fully self-contained (Workers
36 * have no filesystem at runtime) while the .txt in the repo stays the single
37 * editable source of truth — the two are shipped together in the same module.
38 */
39const URLS_REMOVIDAS_TXT = `# urls-removidas.txt — Cap. 22
40# Uma URL removida por linha (apenas o path, sem domínio).
41# Linhas em branco e linhas iniciadas por "#" são ignoradas.
42# Comparação é case-insensitive e ignora a barra final.
43# TODO: troque estes exemplos pelas suas próprias URLs aposentadas.
44
45/promo-black-friday-2019
46/produto/kit-antigo-descontinuado
47/blog/post-que-foi-despublicado
48/categoria/colecao-verao-2020
49/servicos/plano-legado
50/lp/campanha-encerrada
51/loja/estoque-liquidado
52/eventos/webinar-2021-03
53`;
54
55/** How long (seconds) to cache a liveness decision at the edge. */
56const LIVENESS_TTL_SECONDS = 300; // 5 minutes
57
58/** Normalises a pathname for comparison: lowercase, no trailing slash. */
59function normalisePath(pathname) {
60  let p = pathname.toLowerCase();
61  if (p.length > 1 && p.endsWith("/")) p = p.slice(0, -1);
62  return p;
63}
64
65/**
66 * Parses the contents of `urls-removidas.txt` into a Set of normalised paths.
67 * Ignores blank lines and comment lines (starting with "#"). This is the small
68 * embedded parser that lets the .txt drive the Worker's behaviour.
69 */
70function parseRemovedUrls(text) {
71  const set = new Set();
72  for (const rawLine of text.split(/\r?\n/)) {
73    const line = rawLine.trim();
74    if (!line || line.startsWith("#")) continue;
75    set.add(normalisePath(line));
76  }
77  return set;
78}
79
80/** Dead paths loaded once from the bundled list at module init. */
81const DEFAULT_GONE_PATHS = parseRemovedUrls(URLS_REMOVIDAS_TXT);
82
83/**
84 * Builds the Set of gone paths, merging the list parsed from the .txt with any
85 * extra paths provided via env.GONE_PATHS (comma-separated).
86 */
87function buildGoneSet(env) {
88  const set = new Set(DEFAULT_GONE_PATHS);
89  if (env && typeof env.GONE_PATHS === "string" && env.GONE_PATHS.trim()) {
90    for (const raw of env.GONE_PATHS.split(",")) {
91      const p = normalisePath(raw.trim());
92      if (p) set.add(p);
93    }
94  }
95  return set;
96}
97
98/** The 410 response body + headers actually served to clients and crawlers. */
99function goneResponse() {
100  const body =
101    "<!doctype html><html lang=\"pt-BR\"><head><meta charset=\"utf-8\">" +
102    "<title>410 - Conteúdo removido</title></head><body>" +
103    "<h1>410 - Conteúdo removido permanentemente</h1>" +
104    "<p>Esta página foi removida e não está mais disponível.</p>" +
105    "</body></html>";
106
107  return new Response(body, {
108    status: 410,
109    statusText: "Gone",
110    headers: {
111      "Content-Type": "text/html; charset=UTF-8",
112      // Let crawlers cache the "gone" verdict; adjust to taste.
113      "Cache-Control": "public, max-age=3600",
114      // Belt-and-suspenders: keep these out of the index even if rendered.
115      "X-Robots-Tag": "noindex",
116    },
117  });
118}
119
120/**
121 * Liveness check: asks the origin WordPress REST API whether the content at
122 * `pathname` still exists. Returns true if the content appears GONE (safe to
123 * serve 410), false if it looks ALIVE (should pass through).
124 *
125 * Strategy: query the WP REST API search endpoint for the slug. If WordPress
126 * returns 404 for the mapped resource, or the search yields no live match, we
127 * treat the content as gone. If it returns 200 with a live match, it is alive.
128 *
129 * Results are cached at the edge for LIVENESS_TTL_SECONDS using the Cache API,
130 * keyed by a synthetic URL, so repeated hits don't re-query the origin.
131 *
132 * On any error (origin unreachable, timeout, malformed response) we FAIL SAFE
133 * by returning false (alive) — i.e. we do NOT serve 410 if we can't confirm the
134 * content is gone. Better to serve the page than to wrongly 410 live content.
135 */
136async function isContentGone(pathname, env, ctx) {
137  // Without a configured origin we cannot verify; fail safe (treat as alive).
138  if (!env || !env.WP_API) {
139    // TODO: set env.WP_API in wrangler.toml so liveness checks can run.
140    return false;
141  }
142
143  const cache = caches.default;
144  // Synthetic cache key — never actually fetched, just used as a cache handle.
145  const cacheKey = new Request(
146    `https://liveness.internal/${encodeURIComponent(pathname)}`,
147    { method: "GET" },
148  );
149
150  // 1) Try the edge cache first.
151  const cached = await cache.match(cacheKey);
152  if (cached) {
153    const verdict = await cached.text();
154    return verdict === "gone";
155  }
156
157  let gone;
158  try {
159    // The slug is the last path segment; used to search WordPress.
160    const slug = pathname.split("/").filter(Boolean).pop() || "";
161    const apiUrl =
162      `${env.WP_API.replace(/\/$/, "")}` +
163      `/wp-json/wp/v2/search?search=${encodeURIComponent(slug)}&per_page=1`;
164
165    const originResponse = await fetch(apiUrl, {
166      headers: { Accept: "application/json" },
167      cf: { cacheTtl: 0 },
168    });
169
170    if (originResponse.status === 404) {
171      // Endpoint/resource explicitly missing -> gone.
172      gone = true;
173    } else if (originResponse.status === 200) {
174      const results = await originResponse.json();
175      // If WordPress returns a live match whose URL ends with our path, the
176      // content is ALIVE (revived). Otherwise treat as gone.
177      const alive =
178        Array.isArray(results) &&
179        results.some((item) => {
180          if (!item || typeof item.url !== "string") return false;
181          try {
182            return normalisePath(new URL(item.url).pathname) ===
183              normalisePath(pathname);
184          } catch {
185            return false;
186          }
187        });
188      gone = !alive;
189    } else {
190      // Any other status (5xx, etc.) is inconclusive -> fail safe as alive.
191      gone = false;
192    }
193  } catch (error) {
194    // Network/parse failure -> fail safe: do not 410 content we can't verify.
195    // eslint-disable-next-line no-console
196    console.error("liveness check failed:", error);
197    gone = false;
198  }
199
200  // 2) Store the verdict in the edge cache for a short while.
201  const verdictResponse = new Response(gone ? "gone" : "alive", {
202    headers: { "Cache-Control": `max-age=${LIVENESS_TTL_SECONDS}` },
203  });
204  // waitUntil so caching doesn't delay the response to the user.
205  if (ctx && typeof ctx.waitUntil === "function") {
206    ctx.waitUntil(cache.put(cacheKey, verdictResponse.clone()));
207  }
208
209  return gone;
210}
211
212export default {
213  /**
214   * @param {Request} request
215   * @param {Record<string, string>} env
216   * @param {{ waitUntil: (p: Promise<unknown>) => void }} ctx
217   */
218  async fetch(request, env, ctx) {
219    const url = new URL(request.url);
220    const path = normalisePath(url.pathname);
221    const goneSet = buildGoneSet(env);
222
223    // Not a curated dead path -> nothing to do, pass straight through.
224    if (!goneSet.has(path)) {
225      return fetch(request);
226    }
227
228    // Curated as dead. Confirm with the origin before serving 410.
229    const gone = await isContentGone(url.pathname, env, ctx);
230
231    if (gone) {
232      return goneResponse();
233    }
234
235    // Content unexpectedly alive (or unverifiable) -> pass through to origin
236    // so we never accidentally 410 revived content.
237    return fetch(request);
238  },
239};

1# urls-removidas.txt — Cap. 22
2# Uma URL removida por linha (apenas o path, sem domínio).
3# Linhas em branco e linhas iniciadas por "#" são ignoradas.
4# Comparação é case-insensitive e ignora a barra final.
5# TODO: troque estes exemplos pelas suas próprias URLs aposentadas.
6
7/promo-black-friday-2019
8/produto/kit-antigo-descontinuado
9/blog/post-que-foi-despublicado
10/categoria/colecao-verao-2020
11/servicos/plano-legado
12/lp/campanha-encerrada
13/loja/estoque-liquidado
14/eventos/webinar-2021-03

1# wrangler.toml.example — Cap. 22: "Aparecer ou Sumir"
2#
3# Copy this file to `wrangler.toml` and fill in your own values.
4# Deploy with:  npx wrangler deploy
5#
6# Docs: https://developers.cloudflare.com/workers/wrangler/configuration/
7
8name = "worker-410-gone"
9main = "worker.js"
10compatibility_date = "2026-01-01"
11
12# TODO: set your Cloudflare account ID (Dashboard -> Workers -> right sidebar).
13account_id = "TODO_YOUR_CLOUDFLARE_ACCOUNT_ID"
14
15# Route(s) this Worker runs on.
16# TODO: replace with your real zone / hostname pattern.
17# routes = [
18#   { pattern = "exemplo.com/*", zone_name = "exemplo.com" }
19# ]
20
21[vars]
22# TODO: base URL of your WordPress origin (used by the liveness check).
23WP_API = "https://origin.exemplo.com"
24
25# TODO (optional): extra dead paths as a comma-separated list. These are merged
26# with the list parsed from urls-removidas.txt (embedded in worker.js).
27GONE_PATHS = "/promo-antiga,/outra-url-morta"