Gerador de sitemaps por tipo

1#!/usr/bin/env python3
2"""Chapter 21 - Typed multi-sitemap generator with a sitemap index.
3
4Generates SEVERAL sitemaps split BY TYPE (pages, posts, products, localities,
5news) plus a top-level sitemap INDEX that references them all. Two production
6concerns are demonstrated:
7
8  * Per-type TTL cache: each sitemap type is only rebuilt when its cache TTL has
9    expired. Last-built timestamps live in a small JSON state file.
10  * News sitemap (Google News format): includes ONLY items published within the
11    last 48 hours, using the ``news:news`` namespace.
12
13The script ships with inline SAMPLE data so it runs with zero setup (standard
14library only - no pip install needed).
15
16Part of the "living code" companion repository for the book
17"Aparecer ou Sumir" / "Be Seen or Be Forgotten".
18"""
19
20from __future__ import annotations
21
22import argparse
23import json
24import os
25import sys
26from datetime import datetime, timedelta, timezone
27from pathlib import Path
28from typing import Dict, List, Optional
29from xml.sax.saxutils import escape
30
31# XML namespaces used across the generated files.
32NS_SITEMAP = "http://www.sitemaps.org/schemas/sitemap/0.9"
33NS_NEWS = "http://www.google.com/schemas/sitemap-news/0.9"
34
35# Name of the JSON file that records when each type's sitemap was last built.
36STATE_FILE_NAME = ".sitemap_state.json"
37
38# The 48-hour window is a Google News requirement: news sitemaps must only list
39# articles published in the last two days.
40NEWS_WINDOW = timedelta(hours=48)
41
42# Per-type cache TTLs. A type's sitemap is only rebuilt once its TTL elapses.
43# The news sitemap gets a short TTL because its content is time-sensitive.
44DEFAULT_TTL_SECONDS: Dict[str, int] = {
45    "pages": 24 * 3600,      # slow-changing: once a day
46    "posts": 6 * 3600,       # editorial content: a few times a day
47    "products": 3600,        # catalogue: hourly
48    "localities": 24 * 3600,  # geo pages: once a day
49    "news": 15 * 60,         # news: every 15 minutes
50}
51
52
53def now_utc() -> datetime:
54    """Return the current time as an aware UTC datetime (easy to mock in tests)."""
55    return datetime.now(timezone.utc)
56
57
58# ---------------------------------------------------------------------------
59# Sample data source. In a real project each of these would be a DB query.
60# Every item is a dict with at least "loc" and "lastmod". News items also carry
61# "title" and "published" (an ISO-8601 timestamp).
62# ---------------------------------------------------------------------------
63def fetch_items(item_type: str, base_url: str) -> List[Dict[str, str]]:
64    """Return the URL items for a given sitemap type.
65
66    TODO: replace this sample implementation with real queries against your CMS
67    / database. The shape of each returned dict must stay the same.
68    """
69    base = base_url.rstrip("/")
70    today = now_utc().strftime("%Y-%m-%d")
71    recent = now_utc().isoformat()
72    yesterday = (now_utc() - timedelta(hours=12)).isoformat()
73
74    samples: Dict[str, List[Dict[str, str]]] = {
75        "pages": [
76            {"loc": f"{base}/", "lastmod": today},
77            {"loc": f"{base}/sobre", "lastmod": today},
78            {"loc": f"{base}/contato", "lastmod": today},
79        ],
80        "posts": [
81            {"loc": f"{base}/blog/seo-local", "lastmod": today},
82            {"loc": f"{base}/blog/sitemaps-por-tipo", "lastmod": today},
83        ],
84        "products": [
85            {"loc": f"{base}/produtos/plano-pro", "lastmod": today},
86            {"loc": f"{base}/produtos/plano-basico", "lastmod": today},
87        ],
88        "localities": [
89            {"loc": f"{base}/localidades/vila-aurora", "lastmod": today},
90            {"loc": f"{base}/localidades/porto-sereno", "lastmod": today},
91        ],
92        "news": [
93            # Fresh item: published now -> inside the 48h window -> INCLUDED.
94            {
95                "loc": f"{base}/noticias/lancamento",
96                "lastmod": today,
97                "title": "Lancamento da nova ferramenta de SEO",
98                "published": recent,
99            },
100            # Fresh item: published 12h ago -> inside the 48h window -> INCLUDED.
101            {
102                "loc": f"{base}/noticias/atualizacao",
103                "lastmod": today,
104                "title": "Atualizacao do algoritmo de busca",
105                "published": yesterday,
106            },
107            # Stale item: published 5 days ago -> outside 48h -> FILTERED OUT.
108            {
109                "loc": f"{base}/noticias/antiga",
110                "lastmod": today,
111                "title": "Noticia antiga",
112                "published": (now_utc() - timedelta(days=5)).isoformat(),
113            },
114        ],
115    }
116    return samples.get(item_type, [])
117
118
119# ---------------------------------------------------------------------------
120# State / TTL handling
121# ---------------------------------------------------------------------------
122def load_state(state_path: Path) -> Dict[str, float]:
123    """Load {type: last_built_epoch_seconds} from the JSON state file."""
124    if state_path.exists():
125        with state_path.open(encoding="utf-8") as handle:
126            return json.load(handle)
127    return {}
128
129
130def save_state(state_path: Path, state: Dict[str, float]) -> None:
131    """Persist {type: last_built_epoch_seconds} to the JSON state file."""
132    with state_path.open("w", encoding="utf-8") as handle:
133        json.dump(state, handle, indent=2)
134
135
136def is_ttl_expired(state: Dict[str, float], item_type: str, ttl_seconds: int) -> bool:
137    """Return True when a type must be rebuilt because its TTL has expired.
138
139    TTL LOGIC: we compare "now" against the last-built timestamp. If more than
140    ``ttl_seconds`` have passed (or the type was never built), the cache is
141    considered stale and the sitemap is rebuilt.
142    """
143    last_built = state.get(item_type)
144    if last_built is None:
145        return True
146    age_seconds = now_utc().timestamp() - last_built
147    return age_seconds >= ttl_seconds
148
149
150# ---------------------------------------------------------------------------
151# XML builders
152# ---------------------------------------------------------------------------
153def build_urlset(items: List[Dict[str, str]]) -> str:
154    """Build a standard <urlset> sitemap document from URL items."""
155    lines = ['<?xml version="1.0" encoding="UTF-8"?>']
156    lines.append(f'<urlset xmlns="{NS_SITEMAP}">')
157    for item in items:
158        lines.append("  <url>")
159        lines.append(f"    <loc>{escape(item['loc'])}</loc>")
160        if item.get("lastmod"):
161            lines.append(f"    <lastmod>{escape(item['lastmod'])}</lastmod>")
162        lines.append("  </url>")
163    lines.append("</urlset>")
164    return "\n".join(lines) + "\n"
165
166
167def build_news_urlset(
168    items: List[Dict[str, str]],
169    publication_name: str,
170    language: str,
171) -> str:
172    """Build a Google News <urlset> including only items from the last 48h.
173
174    48h WINDOW: each item's ``published`` timestamp is parsed and compared
175    against ``now - 48h``. Anything older is skipped, per Google News rules.
176    """
177    cutoff = now_utc() - NEWS_WINDOW
178    lines = ['<?xml version="1.0" encoding="UTF-8"?>']
179    lines.append(
180        f'<urlset xmlns="{NS_SITEMAP}" xmlns:news="{NS_NEWS}">'
181    )
182    for item in items:
183        published = _parse_iso(item.get("published"))
184        if published is None or published < cutoff:
185            # Outside the 48-hour window -> excluded from the news sitemap.
186            continue
187        lines.append("  <url>")
188        lines.append(f"    <loc>{escape(item['loc'])}</loc>")
189        lines.append("    <news:news>")
190        lines.append("      <news:publication>")
191        lines.append(f"        <news:name>{escape(publication_name)}</news:name>")
192        lines.append(f"        <news:language>{escape(language)}</news:language>")
193        lines.append("      </news:publication>")
194        lines.append(
195            f"      <news:publication_date>{escape(item['published'])}"
196            "</news:publication_date>"
197        )
198        lines.append(f"      <news:title>{escape(item.get('title', ''))}</news:title>")
199        lines.append("    </news:news>")
200        lines.append("  </url>")
201    lines.append("</urlset>")
202    return "\n".join(lines) + "\n"
203
204
205def build_sitemap_index(sitemap_urls: List[str], lastmod: str) -> str:
206    """Build the top-level <sitemapindex> referencing every type sitemap."""
207    lines = ['<?xml version="1.0" encoding="UTF-8"?>']
208    lines.append(f'<sitemapindex xmlns="{NS_SITEMAP}">')
209    for url in sitemap_urls:
210        lines.append("  <sitemap>")
211        lines.append(f"    <loc>{escape(url)}</loc>")
212        lines.append(f"    <lastmod>{escape(lastmod)}</lastmod>")
213        lines.append("  </sitemap>")
214    lines.append("</sitemapindex>")
215    return "\n".join(lines) + "\n"
216
217
218def _parse_iso(value: Optional[str]) -> Optional[datetime]:
219    """Best-effort ISO-8601 parser that always returns an aware UTC datetime."""
220    if not value:
221        return None
222    try:
223        parsed = datetime.fromisoformat(value)
224    except ValueError:
225        return None
226    if parsed.tzinfo is None:
227        parsed = parsed.replace(tzinfo=timezone.utc)
228    return parsed.astimezone(timezone.utc)
229
230
231# ---------------------------------------------------------------------------
232# Orchestration
233# ---------------------------------------------------------------------------
234def generate(
235    base_url: str,
236    output_dir: Path,
237    types: List[str],
238    ttl_overrides: Dict[str, int],
239    publication_name: str,
240    language: str,
241    force: bool = False,
242) -> None:
243    """Generate the per-type sitemaps (honouring TTLs) and the sitemap index."""
244    output_dir.mkdir(parents=True, exist_ok=True)
245    state_path = output_dir / STATE_FILE_NAME
246    state = load_state(state_path)
247
248    sitemap_urls: List[str] = []
249    base = base_url.rstrip("/")
250
251    for item_type in types:
252        ttl = ttl_overrides.get(item_type, DEFAULT_TTL_SECONDS.get(item_type, 3600))
253        filename = f"sitemap-{item_type}.xml"
254        out_path = output_dir / filename
255        public_url = f"{base}/{filename}"
256        sitemap_urls.append(public_url)
257
258        # TTL cache decision: skip rebuild when still fresh (unless forced).
259        if not force and out_path.exists() and not is_ttl_expired(state, item_type, ttl):
260            print(f"[cached]    {filename} (TTL not expired)")
261            continue
262
263        items = fetch_items(item_type, base_url)
264        if item_type == "news":
265            xml = build_news_urlset(items, publication_name, language)
266            kept = xml.count("<url>")
267            print(
268                f"[built]     {filename} ({len(items)} source items, "
269                f"{kept} within 48h window)"
270            )
271        else:
272            xml = build_urlset(items)
273            print(f"[built]     {filename} ({len(items)} source items)")
274
275        out_path.write_text(xml, encoding="utf-8")
276        state[item_type] = now_utc().timestamp()
277
278    # The index is always rewritten so it references the current set of files.
279    index_xml = build_sitemap_index(
280        sitemap_urls, lastmod=now_utc().strftime("%Y-%m-%d")
281    )
282    (output_dir / "sitemap.xml").write_text(index_xml, encoding="utf-8")
283    print(f"[index]     sitemap.xml ({len(sitemap_urls)} sitemaps)")
284
285    save_state(state_path, state)
286
287
288def parse_args(argv: List[str]) -> argparse.Namespace:
289    parser = argparse.ArgumentParser(
290        description="Generate typed sitemaps + a sitemap index with TTL cache."
291    )
292    parser.add_argument(
293        "--base-url",
294        default=os.environ.get("SITE_BASE_URL", "https://www.example.com"),
295        help="Site base URL used to build absolute locations.",
296    )
297    parser.add_argument(
298        "--out",
299        dest="out",
300        default=os.environ.get("SITEMAP_OUTPUT", "out"),
301        help="Directory to write the sitemap files into.",
302    )
303    parser.add_argument(
304        "--types",
305        default="pages,posts,products,localities,news",
306        help="Comma-separated list of sitemap types to build.",
307    )
308    parser.add_argument(
309        "--publication-name",
310        default=os.environ.get("NEWS_PUBLICATION_NAME", "Sua Publicacao"),
311        help="Publication name used in the news sitemap.",
312    )
313    parser.add_argument(
314        "--language",
315        default=os.environ.get("NEWS_LANGUAGE", "pt"),
316        help="Language code used in the news sitemap.",
317    )
318    parser.add_argument(
319        "--force",
320        action="store_true",
321        help="Rebuild every sitemap regardless of TTL.",
322    )
323    return parser.parse_args(argv)
324
325
326def main(argv: List[str]) -> int:
327    args = parse_args(argv)
328    types = [t.strip() for t in args.types.split(",") if t.strip()]
329    generate(
330        base_url=args.base_url,
331        output_dir=Path(args.out),
332        types=types,
333        ttl_overrides={},  # TODO: load per-type TTL overrides here if desired.
334        publication_name=args.publication_name,
335        language=args.language,
336        force=args.force,
337    )
338    return 0
339
340
341if __name__ == "__main__":
342    raise SystemExit(main(sys.argv[1:]))