From b8851cb977cfdd608798d32bf5877f645b960831 Mon Sep 17 00:00:00 2001
From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com>
Date: Fri, 9 Oct 2026 21:46:05 +0800
Subject: [PATCH 1/2] feat(website): add search metadata and automatic sitemap
Prefer rea.tools in page canonicals and search listings. Generate the sitemap from current HTML indexing directives and render a shared PNG preview from SVG before verification and manual deployment.
Refs #1283
---
.github/workflows/website-check.yml | 22 +-
.github/workflows/website-pages.yml | 17 +-
.gitignore | 3 +
scripts/prepare-website.py | 77 +++++-
scripts/test-website.py | 130 ++++++++++
scripts/verify-website.py | 229 +++++++++++++++++-
website/evidence/guide-examples.md | 21 +-
website/public/assets/social-card.svg | 27 +++
website/public/examples/dino-lab/index.html | 42 +++-
.../public/examples/notes-electron/index.html | 5 +
website/public/examples/notes-web/index.html | 5 +
website/public/faq/index.html | 42 +++-
website/public/first-investigation/index.html | 42 +++-
website/public/get-started/index.html | 42 +++-
website/public/guides/browser/index.html | 36 ++-
website/public/guides/index.html | 36 ++-
website/public/guides/javascript/index.html | 42 +++-
website/public/guides/native/index.html | 36 ++-
website/public/index.html | 44 +++-
website/public/robots.txt | 4 +
website/public/showcase/aegis/index.html | 42 +++-
website/public/showcase/ctf/index.html | 42 +++-
website/public/showcase/dx-ball/index.html | 42 +++-
website/public/showcase/index.html | 42 +++-
website/public/showcase/notion/index.html | 42 +++-
website/public/showcase/th04/index.html | 42 +++-
website/requirements.txt | 2 +
website/wrangler.toml | 5 +
28 files changed, 1123 insertions(+), 38 deletions(-)
create mode 100644 scripts/test-website.py
create mode 100644 website/public/assets/social-card.svg
create mode 100644 website/public/robots.txt
create mode 100644 website/requirements.txt
diff --git a/.github/workflows/website-check.yml b/.github/workflows/website-check.yml
index 2fc7a6874..26270ce73 100644
--- a/.github/workflows/website-check.yml
+++ b/.github/workflows/website-check.yml
@@ -5,8 +5,7 @@ on:
branches: [main, "release/*"]
paths:
- "website/**"
- - "scripts/verify-website.py"
- - "scripts/prepare-website.py"
+ - "scripts/*website*.py"
- ".github/workflows/**"
workflow_dispatch:
@@ -21,11 +20,24 @@ jobs:
check:
name: Website checks
runs-on: ubuntu-latest
- timeout-minutes: 5
+ timeout-minutes: 10
steps:
- name: Check out the repository
uses: actions/checkout@v4
- - name: Prepare the example download
+ - name: Set up Python for website assets
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ cache: pip
+ cache-dependency-path: website/requirements.txt
+ - name: Install sharing-image dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install --no-install-recommends -y libcairo2 fonts-dejavu-core
+ python3 -m pip install -r website/requirements.txt
+ - name: Prepare website assets
run: python3 scripts/prepare-website.py
- - name: Check website links and publishing configuration
+ - name: Check website links, metadata and publishing configuration
run: python3 scripts/verify-website.py
+ - name: Check sitemap discovery and metadata regressions
+ run: python3 scripts/test-website.py
diff --git a/.github/workflows/website-pages.yml b/.github/workflows/website-pages.yml
index ec2b7153e..7dec1ea47 100644
--- a/.github/workflows/website-pages.yml
+++ b/.github/workflows/website-pages.yml
@@ -23,7 +23,7 @@ jobs:
prepare:
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
- timeout-minutes: 5
+ timeout-minutes: 10
permissions:
contents: read
pages: read
@@ -48,9 +48,20 @@ jobs:
fi
- name: Check out the repository
uses: actions/checkout@v4
- - name: Prepare the example download
+ - name: Set up Python for website assets
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ cache: pip
+ cache-dependency-path: website/requirements.txt
+ - name: Install sharing-image dependencies
+ run: |
+ sudo apt-get update
+ sudo apt-get install --no-install-recommends -y libcairo2 fonts-dejavu-core
+ python3 -m pip install -r website/requirements.txt
+ - name: Prepare website assets
run: python3 scripts/prepare-website.py
- - name: Check website links and publishing configuration
+ - name: Check website links, metadata and publishing configuration
run: python3 scripts/verify-website.py
- name: Record the published commit
run: |
diff --git a/.gitignore b/.gitignore
index c0d65e4e7..e88c33a87 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,6 +50,9 @@ Thumbs.db
# Generated website download
/website/public/examples/notes-example.zip
/website/public/deployment.json
+/website/public/sitemap.xml
+/website/public/assets/social-card.png
+/website/.venv/
# Local Cloudflare development state
/website/.wrangler/
diff --git a/scripts/prepare-website.py b/scripts/prepare-website.py
index 1abc9c39d..af4f2d9b9 100644
--- a/scripts/prepare-website.py
+++ b/scripts/prepare-website.py
@@ -1,10 +1,16 @@
-"""Package the public Notes example for the static website."""
+"""Prepare downloads, the sitemap and the sharing image for the static website."""
+from html.parser import HTMLParser
from io import BytesIO
from pathlib import Path
+import re
+import sys
+from urllib.parse import quote
+import xml.etree.ElementTree as ET
from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
+SITE_ORIGIN = "https://rea.tools"
EXAMPLE_FILES = (
"package.json",
"main.js",
@@ -15,6 +21,70 @@ EXAMPLE_FILES = (
)
+class IndexPolicy(HTMLParser):
+ """Read indexing directives from the authored HTML head."""
+
+ def __init__(self):
+ super().__init__()
+ self.in_head = False
+ self.noindex = False
+
+ def handle_starttag(self, tag, attrs):
+ if tag == "head":
+ self.in_head = True
+ if tag == "meta" and self.in_head:
+ attributes = dict(attrs)
+ if (attributes.get("name") or "").lower() in ("robots", "googlebot"):
+ directives = re.split(
+ r"[\s,]+", (attributes.get("content") or "").lower()
+ )
+ self.noindex |= bool(set(directives) & {"noindex", "none"})
+
+ def handle_endtag(self, tag):
+ if tag == "head":
+ self.in_head = False
+
+
+def prepare_sitemap(site):
+ """Discover HTML routes automatically, excluding pages marked noindex."""
+ namespace = "http://www.sitemaps.org/schemas/sitemap/0.9"
+ ET.register_namespace("", namespace)
+ sitemap = ET.Element(f"{{{namespace}}}urlset")
+ for path in sorted(site.rglob("*.html")):
+ policy = IndexPolicy()
+ policy.feed(path.read_text(encoding="utf-8"))
+ if policy.noindex:
+ continue
+ route = path.relative_to(site).as_posix()
+ if path.name == "index.html":
+ route = route.removesuffix("index.html")
+ entry = ET.SubElement(sitemap, f"{{{namespace}}}url")
+ ET.SubElement(
+ entry, f"{{{namespace}}}loc"
+ ).text = f"{SITE_ORIGIN}/{quote(route, safe='/')}"
+ ET.indent(sitemap, space=" ")
+ ET.ElementTree(sitemap).write(
+ site / "sitemap.xml", encoding="utf-8", xml_declaration=True
+ )
+ print(f"Prepared sitemap.xml ({len(sitemap)} indexable pages).")
+
+
+def prepare_sharing_image(site):
+ """Rasterize the maintained SVG for clients that need a PNG preview."""
+ try:
+ import cairosvg
+ except (ImportError, OSError) as error:
+ sys.exit(
+ f"Cannot prepare sharing image: {error}. Install website/requirements.txt and native Cairo; see website/README.md."
+ )
+ source = site / "assets/social-card.svg"
+ destination = site / "assets/social-card.png"
+ cairosvg.svg2png(
+ url=str(source), write_to=str(destination), output_width=1200, output_height=630
+ )
+ print("Prepared assets/social-card.png (1200 × 630).")
+
+
def prepare_notes_example(site):
"""Create a ZIP with public source only and reproducible metadata."""
source = site / "examples/notes-electron"
@@ -37,4 +107,7 @@ def prepare_notes_example(site):
if __name__ == "__main__":
root = Path(__file__).resolve().parent.parent
- prepare_notes_example(root / "website/public")
+ site = root / "website/public"
+ prepare_sharing_image(site)
+ prepare_sitemap(site)
+ prepare_notes_example(site)
diff --git a/scripts/test-website.py b/scripts/test-website.py
new file mode 100644
index 000000000..121dec8ae
--- /dev/null
+++ b/scripts/test-website.py
@@ -0,0 +1,130 @@
+"""Exercise automatic route discovery and mistakes that break search metadata."""
+
+import contextlib
+import importlib.util
+import io
+from pathlib import Path
+import shutil
+import tempfile
+import unittest
+import xml.etree.ElementTree as ET
+
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def load_script(name):
+ """Load a repository script without executing its command-line entrypoint."""
+ spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / f"{name}.py")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+PREPARE = load_script("prepare-website")
+VERIFY = load_script("verify-website")
+
+
+class WebsiteSearchChecks(unittest.TestCase):
+ """Use authored pages and temporary fixtures to check the publishing boundary."""
+
+ def test_sitemap_tracks_added_moved_removed_and_nonindexable_pages(self):
+ with tempfile.TemporaryDirectory() as directory:
+ site = Path(directory)
+
+ def page(name, head="", body=""):
+ path = site / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(f"{head}{body}")
+ return path
+
+ page("index.html")
+ nested = page("guides/new/index.html")
+ page(
+ "examples/fixture/index.html",
+ '',
+ )
+ page("draft.html", '')
+ page("code & notes.html")
+ page("visible/index.html", body='')
+ (site / "asset.json").write_text("{}")
+
+ def urls():
+ with contextlib.redirect_stdout(io.StringIO()):
+ PREPARE.prepare_sitemap(site)
+ sitemap = ET.parse(site / "sitemap.xml")
+ return {node.text for node in sitemap.findall(".//{*}loc")}
+
+ self.assertEqual(
+ urls(),
+ {
+ "https://rea.tools/",
+ "https://rea.tools/guides/new/",
+ "https://rea.tools/code%20%26%20notes.html",
+ "https://rea.tools/visible/",
+ },
+ )
+ nested.rename(site / "moved.html")
+ self.assertIn("https://rea.tools/moved.html", urls())
+ self.assertNotIn("https://rea.tools/guides/new/", urls())
+ (site / "moved.html").unlink()
+ self.assertNotIn("https://rea.tools/moved.html", urls())
+
+ def test_verifier_rejects_wrong_canonicals_stale_sitemaps_and_missing_previews(
+ self,
+ ):
+ cases = (
+ (
+ "index.html",
+ 'rel="canonical" href="https://rea.tools/"',
+ 'rel="canonical" href="https://morluto.github.io/rea/"',
+ "head canonical",
+ ),
+ (
+ "showcase/aegis/index.html",
+ 'rel="canonical" href="https://rea.tools/showcase/aegis/"',
+ 'rel="canonical" href="https://rea.tools/"',
+ "head canonical",
+ ),
+ (
+ "examples/notes-electron/index.html",
+ 'content="noindex"',
+ 'content="index"',
+ "must have a head noindex",
+ ),
+ (
+ "sitemap.xml",
+ "https://rea.tools/faq/",
+ "https://rea.tools/deleted/",
+ "each indexable page once",
+ ),
+ (
+ "faq/index.html",
+ 'name="twitter:card" content="summary_large_image"',
+ 'name="twitter:card" content="summary"',
+ "twitter:card",
+ ),
+ )
+ for name, old, new, message in cases:
+ with self.subTest(name=name), tempfile.TemporaryDirectory() as directory:
+ site = Path(directory) / "public"
+ shutil.copytree(ROOT / "website/public", site)
+ path = site / name
+ source = path.read_text()
+ self.assertIn(old, source)
+ path.write_text(source.replace(old, new, 1))
+ self.assertTrue(
+ any(message in error for error in VERIFY.check_site(site)[3])
+ )
+ with tempfile.TemporaryDirectory() as directory:
+ site = Path(directory) / "public"
+ shutil.copytree(ROOT / "website/public", site)
+ (site / "assets/social-card.png").unlink()
+ self.assertIn(
+ "Social preview PNG is missing; run scripts/prepare-website.py",
+ VERIFY.check_site(site)[3],
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/scripts/verify-website.py b/scripts/verify-website.py
index d6f4a6be0..d67a89284 100644
--- a/scripts/verify-website.py
+++ b/scripts/verify-website.py
@@ -1,14 +1,19 @@
-"""Check local website references and the sole GitHub Pages publisher."""
+"""Check website references, search metadata and the sole production publisher."""
from html.parser import HTMLParser
from pathlib import Path
import re
+import struct
import sys
-from urllib.parse import unquote, urlsplit
+from urllib.parse import quote, unquote, urlsplit
import xml.etree.ElementTree as ET
from zipfile import BadZipFile, ZipFile
+SITE_ORIGIN = "https://rea.tools"
+NOTES_FIXTURES = {"examples/notes-electron/index.html", "examples/notes-web/index.html"}
+
+
class HtmlReferences(HTMLParser):
"""Collect document IDs and references without fetching external URLs."""
@@ -17,9 +22,42 @@ class HtmlReferences(HTMLParser):
self.ids = set()
self.references = []
self.duplicate_ids = []
+ self.in_head = False
+ self.in_title = False
+ self.titles = []
+ self.meta = {}
+ self.properties = {}
+ self.canonicals = []
+ self.lang = None
+ self.h1_count = 0
def handle_starttag(self, tag, attrs):
attributes = dict(attrs)
+ if tag == "html":
+ self.lang = attributes.get("lang")
+ if tag == "h1":
+ self.h1_count += 1
+ if tag == "head":
+ self.in_head = True
+ if self.in_head:
+ if tag == "title":
+ self.titles.append("")
+ self.in_title = True
+ if (
+ tag == "link"
+ and "canonical" in (attributes.get("rel") or "").lower().split()
+ ):
+ self.canonicals.append(attributes.get("href") or "")
+ if tag == "meta":
+ for attribute, collection in (
+ ("name", self.meta),
+ ("property", self.properties),
+ ):
+ key = attributes.get(attribute)
+ if key:
+ collection.setdefault(key.lower(), []).append(
+ attributes.get("content") or ""
+ )
identifier = attributes.get("id")
if identifier is not None:
if identifier in self.ids:
@@ -30,6 +68,154 @@ class HtmlReferences(HTMLParser):
if reference is not None:
self.references.append(reference)
+ def handle_endtag(self, tag):
+ if tag == "head":
+ self.in_head = False
+ if tag == "title":
+ self.in_title = False
+
+ def handle_data(self, data):
+ if self.in_title:
+ self.titles[-1] += data
+
+ def noindex(self):
+ """Recognize directives that exclude this page from Google's index."""
+ directives = ",".join(
+ self.meta.get("robots", []) + self.meta.get("googlebot", [])
+ ).lower()
+ return bool(set(re.split(r"[\s,]+", directives)) & {"noindex", "none"})
+
+
+def check_search_metadata(site, pages):
+ """Verify route-specific canonicals, sharing assets and the generated sitemap."""
+ errors = []
+ indexable = set()
+ titles = set()
+ for path, page in pages.items():
+ label = path.relative_to(site).as_posix()
+ title = page.titles[0].strip() if len(page.titles) == 1 else ""
+ if not title:
+ errors.append(f"{label}: needs exactly one nonempty head title")
+ if page.lang != "en" or page.h1_count != 1:
+ errors.append(f"{label}: needs lang=en and exactly one h1")
+ if label in NOTES_FIXTURES and not page.noindex():
+ errors.append(
+ f"{label}: example application must have a head noindex directive"
+ )
+ if page.noindex():
+ continue
+ route = (
+ path.relative_to(site).parent.as_posix()
+ if path.name == "index.html"
+ else label
+ )
+ route = "" if route == "." else route
+ url = f"{SITE_ORIGIN}/{quote(route, safe='/')}" + (
+ "/" if route and path.name == "index.html" else ""
+ )
+ indexable.add(url)
+ if page.canonicals != [url]:
+ errors.append(f"{label}: needs exactly one head canonical for {url}")
+ if title in titles:
+ errors.append(f"{label}: duplicate search title {title!r}")
+ titles.add(title)
+ descriptions = page.meta.get("description", [])
+ description = descriptions[0].strip() if len(descriptions) == 1 else ""
+ if not description:
+ errors.append(f"{label}: needs exactly one nonempty head description")
+ expected_og = {
+ "og:type": "website",
+ "og:site_name": "REA",
+ "og:title": title,
+ "og:description": description,
+ "og:url": url,
+ "og:image": f"{SITE_ORIGIN}/assets/social-card.png",
+ "og:image:type": "image/png",
+ "og:image:width": "1200",
+ "og:image:height": "630",
+ }
+ expected_twitter = {
+ "twitter:card": "summary_large_image",
+ "twitter:title": title,
+ "twitter:description": description,
+ "twitter:image": expected_og["og:image"],
+ }
+ for collection, expected in (
+ (page.properties, expected_og),
+ (page.meta, expected_twitter),
+ ):
+ for key, value in expected.items():
+ if collection.get(key) != [value]:
+ errors.append(
+ f"{label}: {key} must appear once and match {value!r}"
+ )
+ for collection, key in (
+ (page.properties, "og:image:alt"),
+ (page.meta, "twitter:image:alt"),
+ ):
+ values = collection.get(key, [])
+ if len(values) != 1 or not values[0].strip():
+ errors.append(f"{label}: needs exactly one nonempty {key}")
+
+ sitemap_path = site / "sitemap.xml"
+ try:
+ sitemap = ET.parse(sitemap_path).getroot()
+ namespace = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
+ urls = [entry.findtext(f"{namespace}loc") for entry in sitemap]
+ if sitemap.tag != f"{namespace}urlset" or any(
+ entry.tag != f"{namespace}url" for entry in sitemap
+ ):
+ errors.append("sitemap.xml: needs a sitemap-protocol urlset")
+ if len(urls) != len(set(urls)) or set(urls) != indexable:
+ errors.append(
+ "sitemap.xml: must contain each indexable page once, using its rea.tools URL"
+ )
+ if any(
+ len(entry) != 1 or entry[0].tag != f"{namespace}loc" for entry in sitemap
+ ):
+ errors.append(
+ "sitemap.xml: keep entries to loc; do not invent lastmod or add priority/changefreq"
+ )
+ except (OSError, ET.ParseError) as error:
+ errors.append(
+ f"sitemap.xml: cannot read sitemap ({error}); run scripts/prepare-website.py"
+ )
+
+ robots_path = site / "robots.txt"
+ if not robots_path.is_file():
+ errors.append("robots.txt: authored crawl rules are missing")
+ else:
+ directives = []
+ for line in robots_path.read_text(encoding="utf-8").splitlines():
+ value = line.split("#", 1)[0].strip()
+ if value:
+ key, separator, content = value.partition(":")
+ directives.append(
+ (key.lower(), content.strip()) if separator else (value, "")
+ )
+ expected = [
+ ("user-agent", "*"),
+ ("allow", "/"),
+ ("sitemap", f"{SITE_ORIGIN}/sitemap.xml"),
+ ]
+ if directives != expected:
+ errors.append(
+ "robots.txt: expected crawl access and one absolute rea.tools sitemap directive"
+ )
+
+ image = site / "assets/social-card.png"
+ if not image.is_file():
+ errors.append("Social preview PNG is missing; run scripts/prepare-website.py")
+ else:
+ header = image.read_bytes()[:24]
+ if (
+ len(header) != 24
+ or header[:16] != b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR"
+ or struct.unpack(">II", header[16:24]) != (1200, 630)
+ ):
+ errors.append("Social preview must be a 1200 × 630 PNG")
+ return len(indexable), errors
+
def check_site(site):
"""Check assets, HTML fragment destinations and SVG syntax."""
@@ -63,7 +249,11 @@ def check_site(site):
target /= "index.html"
if not target.is_file():
errors.append(f"{label}: missing local target: {reference}")
- elif target in pages and url.fragment and unquote(url.fragment) not in pages[target].ids:
+ elif (
+ target in pages
+ and url.fragment
+ and unquote(url.fragment) not in pages[target].ids
+ ):
errors.append(f"{label}: missing HTML fragment: {reference}")
for path in sorted(site.rglob("*.svg")):
@@ -71,18 +261,23 @@ def check_site(site):
ET.parse(path)
except ET.ParseError as error:
errors.append(f"{path.relative_to(site)}: invalid SVG XML: {error}")
- return len(pages), references, errors
+ indexable, metadata_errors = check_search_metadata(site, pages)
+ errors.extend(metadata_errors)
+ return len(pages), references, indexable, errors
def check_publisher(root):
"""Detect a second Pages action that could replace the website."""
expected = root / ".github/workflows/website-pages.yml"
workflows = root / ".github/workflows"
- deploy_action = re.compile(r"^\s*-?\s*uses:\s*['\"]?actions/deploy-pages@", re.MULTILINE)
+ deploy_action = re.compile(
+ r"^\s*-?\s*uses:\s*['\"]?actions/deploy-pages@", re.MULTILINE
+ )
publishers = [
path
for path in sorted(workflows.iterdir())
- if path.suffix in (".yml", ".yaml") and deploy_action.search(path.read_text(encoding="utf-8"))
+ if path.suffix in (".yml", ".yaml")
+ and deploy_action.search(path.read_text(encoding="utf-8"))
]
if publishers != [expected]:
actual = ", ".join(str(path.relative_to(root)) for path in publishers) or "none"
@@ -95,7 +290,14 @@ def check_example_archive(site):
source = site / "examples/notes-electron"
expected = {
f"notes-example/{name}": source / name
- for name in ("package.json", "main.js", "preload.js", "renderer.js", "csv.js", "index.html")
+ for name in (
+ "package.json",
+ "main.js",
+ "preload.js",
+ "renderer.js",
+ "csv.js",
+ "index.html",
+ )
}
path = site / "examples/notes-example.zip"
if not path.is_file():
@@ -103,7 +305,9 @@ def check_example_archive(site):
try:
with ZipFile(path) as archive:
if sorted(archive.namelist()) != sorted(expected):
- return ["Notes example ZIP must contain exactly the six files in notes-example/."]
+ return [
+ "Notes example ZIP must contain exactly the six files in notes-example/."
+ ]
return [
f"Notes example ZIP has stale source: {name}"
for name, original in expected.items()
@@ -116,13 +320,18 @@ def check_example_archive(site):
def main():
"""Run the same checks locally, on website PRs and before publishing."""
root = Path(__file__).resolve().parent.parent
- pages, references, errors = check_site(root / "website/public")
+ pages, references, indexable, errors = check_site(root / "website/public")
errors.extend(check_example_archive(root / "website/public"))
errors.extend(check_publisher(root))
if errors:
print("Website checks failed:\n" + "\n".join(errors), file=sys.stderr)
return 1
- print(f"Verified {pages} HTML pages, {references} local references, SVG XML and the example ZIP.")
+ print(
+ f"Verified {pages} HTML pages, {references} local references, SVG XML and the example ZIP."
+ )
+ print(
+ f"Verified search/sharing metadata, robots.txt and the sitemap for {indexable} indexable pages."
+ )
print("website-pages.yml is the sole GitHub Pages publisher.")
return 0
diff --git a/website/evidence/guide-examples.md b/website/evidence/guide-examples.md
index 7a6626447..5be6e80e7 100644
--- a/website/evidence/guide-examples.md
+++ b/website/evidence/guide-examples.md
@@ -33,8 +33,8 @@ when the example files change.
The guide's `examples/notes-example.zip` is generated from exactly these six
source files, under the `notes-example/` folder. Packaging changes no file
contents. Website verification compares every ZIP entry to its source file and
-rejects extra entries. Existing source digests and static findings remain valid
-for the unzipped example.
+rejects extra entries. The recorded digests below identify the original analysis
+checkpoint; subsequent metadata changes and checks are recorded separately.
## First-investigation formatter check
@@ -82,6 +82,23 @@ The guide's follow-up CLI commands were also exercised against the real page:
source. Discovery Evidence: `ev_31abb33e7707d47625ac909ab9256c705d03a31ee6bcd643fa96730e72469162`. Script capture
Evidence: `ev_aeb777df405c09dc1bbdd4c185a5af2452abaa3cb493683c296c5afa97efbc11`.
+## SEO metadata check — 9 October 2026
+
+Both Notes HTML heads gained descriptions and `noindex` directives. The Electron
+fixture was analyzed again with `rea-agents` 4.1.0 and Node.js 24.18.0 on Linux
+x64. REA still reports four parsed JavaScript files, zero parse failures, one
+exposed API member, one literal IPC channel, one renderer transmission, one main
+handler and one paired transmission, with zero ambiguous transmissions.
+
+- Evidence ID: `ev_3958d870ecd27c202e5decc97161f2ea19db8cc88f9f39ed0449bf1581df2f2a`.
+- Root artifact SHA-256: `0ee325d4eaf7f94b62b2962044995fe1efe9f1d31411eb035118b19b1f56f7bb`.
+- Current Electron HTML SHA-256: `457b22543aa956fe2cd005fc6bda42d2798eb05040b75c4cb4957e3b66d0ea09`.
+- Current browser HTML SHA-256: `3b24b3307ded48055ce5a48f907544195bfde3285f82df919c54a6415ac11070`.
+
+The JavaScript and data files retain their original digests. The generated ZIP
+contains the updated HTML and the same JavaScript. Local browser checks exercise
+the Notes CSV download at both root and `/rea/` paths.
+
Full local outputs and the browser verification harness are under
`/tmp/rea-website-guides/` in the development environment, not website assets.
diff --git a/website/public/assets/social-card.svg b/website/public/assets/social-card.svg
new file mode 100644
index 000000000..faab21bd2
--- /dev/null
+++ b/website/public/assets/social-card.svg
@@ -0,0 +1,27 @@
+
diff --git a/website/public/examples/dino-lab/index.html b/website/public/examples/dino-lab/index.html
index feb3df2e7..bfae79460 100644
--- a/website/public/examples/dino-lab/index.html
+++ b/website/public/examples/dino-lab/index.html
@@ -3,11 +3,51 @@
- Dinosaur speed lab — REA
+ Chrome Dinosaur Game: Rebuild Its Speed Rule — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/examples/notes-electron/index.html b/website/public/examples/notes-electron/index.html
index fc4e3a62d..244b08541 100644
--- a/website/public/examples/notes-electron/index.html
+++ b/website/public/examples/notes-electron/index.html
@@ -2,6 +2,11 @@
+
+
Notes — Electron analysis example
diff --git a/website/public/examples/notes-web/index.html b/website/public/examples/notes-web/index.html
index 7f420d16d..92980ef55 100644
--- a/website/public/examples/notes-web/index.html
+++ b/website/public/examples/notes-web/index.html
@@ -3,6 +3,11 @@
+
+
Notes — REA browser example
diff --git a/website/public/faq/index.html b/website/public/faq/index.html
index 6a551fa40..40e5f616a 100644
--- a/website/public/faq/index.html
+++ b/website/public/faq/index.html
@@ -3,11 +3,51 @@
- FAQ — REA
+ REA FAQ: Setup, Updates and Troubleshooting
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/first-investigation/index.html b/website/public/first-investigation/index.html
index 1ebbd474b..349ece913 100644
--- a/website/public/first-investigation/index.html
+++ b/website/public/first-investigation/index.html
@@ -3,11 +3,51 @@
- Your first investigation — REA
+ Your First Reverse Engineering Investigation — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/get-started/index.html b/website/public/get-started/index.html
index fbb68dfcd..b86987e9e 100644
--- a/website/public/get-started/index.html
+++ b/website/public/get-started/index.html
@@ -3,11 +3,51 @@
- Get started — REA
+ Install REA and Set Up Your Coding Agent
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/guides/browser/index.html b/website/public/guides/browser/index.html
index 7b904f7ae..95a0aaaae 100644
--- a/website/public/guides/browser/index.html
+++ b/website/public/guides/browser/index.html
@@ -3,11 +3,45 @@
- Browser observation — REA
+ Browser and Network Analysis — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/guides/index.html b/website/public/guides/index.html
index ee5c8c484..babd58bdf 100644
--- a/website/public/guides/index.html
+++ b/website/public/guides/index.html
@@ -3,11 +3,45 @@
- Guides — REA
+ Reverse Engineering Guides — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/guides/javascript/index.html b/website/public/guides/javascript/index.html
index d5f865470..5212b579d 100644
--- a/website/public/guides/javascript/index.html
+++ b/website/public/guides/javascript/index.html
@@ -3,11 +3,51 @@
- JavaScript & Electron — REA
+ JavaScript and Electron Analysis — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/guides/native/index.html b/website/public/guides/native/index.html
index d034f5cc7..a0e19caf5 100644
--- a/website/public/guides/native/index.html
+++ b/website/public/guides/native/index.html
@@ -3,11 +3,45 @@
- Native analysis — REA
+ Native Binary Analysis — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/index.html b/website/public/index.html
index 8d895b266..0282d64de 100644
--- a/website/public/index.html
+++ b/website/public/index.html
@@ -3,10 +3,50 @@
- REA — Reverse Engineer Anything
+ REA — Reverse Engineering for Your Coding Agent
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/robots.txt b/website/public/robots.txt
new file mode 100644
index 000000000..4f2eb4ed8
--- /dev/null
+++ b/website/public/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://rea.tools/sitemap.xml
diff --git a/website/public/showcase/aegis/index.html b/website/public/showcase/aegis/index.html
index 619eacc85..05a230501 100644
--- a/website/public/showcase/aegis/index.html
+++ b/website/public/showcase/aegis/index.html
@@ -3,11 +3,51 @@
- How an authenticator generates a login code — REA
+ Aegis APK Reverse Engineering: Login Codes — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/showcase/ctf/index.html b/website/public/showcase/ctf/index.html
index 051697de5..ed2002924 100644
--- a/website/public/showcase/ctf/index.html
+++ b/website/public/showcase/ctf/index.html
@@ -3,11 +3,51 @@
- Solve a CTF flag checker — REA
+ CTF Reverse Engineering: DownUnderCTF Flag Checker — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/showcase/dx-ball/index.html b/website/public/showcase/dx-ball/index.html
index 1b0d16a3f..d47449978 100644
--- a/website/public/showcase/dx-ball/index.html
+++ b/website/public/showcase/dx-ball/index.html
@@ -3,11 +3,51 @@
- Rebuilding DX-Ball — REA
+ DX-Ball Reverse Engineering: Sound Panning — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/showcase/index.html b/website/public/showcase/index.html
index a993fa5a3..f0a537f69 100644
--- a/website/public/showcase/index.html
+++ b/website/public/showcase/index.html
@@ -3,11 +3,51 @@
- Case studies — REA
+ Reverse Engineering Case Studies — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/showcase/notion/index.html b/website/public/showcase/notion/index.html
index 4e3a24b06..5120d8ca6 100644
--- a/website/public/showcase/notion/index.html
+++ b/website/public/showcase/notion/index.html
@@ -3,11 +3,51 @@
- Following Notion's clipboard through Electron — REA
+ Notion Electron Reverse Engineering: Clipboard — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/public/showcase/th04/index.html b/website/public/showcase/th04/index.html
index fa4bce783..e08f8f2a3 100644
--- a/website/public/showcase/th04/index.html
+++ b/website/public/showcase/th04/index.html
@@ -3,11 +3,51 @@
- TH04 bullet patterns — REA
+ TH04 Reverse Engineering: DOS Bullet Patterns — REA
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/website/requirements.txt b/website/requirements.txt
new file mode 100644
index 000000000..cc77ae509
--- /dev/null
+++ b/website/requirements.txt
@@ -0,0 +1,2 @@
+# SVG-to-PNG conversion for the generated social preview.
+CairoSVG==2.9.1
diff --git a/website/wrangler.toml b/website/wrangler.toml
index 06be8ad53..7ebf92a64 100644
--- a/website/wrangler.toml
+++ b/website/wrangler.toml
@@ -20,3 +20,8 @@ name = "rea-website"
workers_dev = true
preview_urls = false
routes = [{ pattern = "rea.tools", custom_domain = true }]
+
+# Production receives the already prepared and verified shared Pages artifact.
+# Override the preview build so deployment does not regenerate local assets.
+[env.production.build]
+command = ""
From bb8ff2cfba638ee3a77f951c01e98340c757f525 Mon Sep 17 00:00:00 2001
From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com>
Date: Fri, 9 Oct 2026 21:47:08 +0800
Subject: [PATCH 2/2] docs(website): explain SEO maintenance and indexing
follow-up
Document the Python preview environment, automatic sitemap updates, canonical URLs, generated sharing card and Search Console checks after publication. Apply the website's clear, concise writing standard to search and sharing metadata.
---
website/AGENTS.md | 8 +++-
website/README.md | 91 ++++++++++++++++++++++++++++++++++++++----
website/figures.md | 9 +++++
website/style-guide.md | 7 ++++
4 files changed, 107 insertions(+), 8 deletions(-)
diff --git a/website/AGENTS.md b/website/AGENTS.md
index da4f38b27..d96ab2b62 100644
--- a/website/AGENTS.md
+++ b/website/AGENTS.md
@@ -16,9 +16,15 @@ for asset notes and the source of case-study claims.
target identity and verification scope.
- Keep personal paths, account data, credentials and raw captures outside the
public assets. Use generic inputs and label shortened display paths.
+- Follow the same clear writing standard in search titles and descriptions.
+ Give each content page its own absolute `https://rea.tools/` canonical and
+ matching sharing metadata; keep navigation relative. The sitemap is generated
+ from HTML automatically. Mark example applications `noindex` in the head and
+ keep them crawlable. Maintain the SVG sharing source, not the generated PNG.
- Check both local root and `/rea/` paths, mobile layouts, expanded details,
copying and downloads. Run `scripts/prepare-website.py` followed by
- `scripts/verify-website.py`.
+ `scripts/verify-website.py` and `scripts/test-website.py`, using the website
+ Python environment described in the README.
- Keep `website-pages.yml` as the sole, manual production publisher for
Cloudflare and Pages. Default to `both`: both hosts receive the same verified
artifact and pass the published-commit check. Use `pages` only when explicitly
diff --git a/website/README.md b/website/README.md
index 33c6e1fad..2005c633d 100644
--- a/website/README.md
+++ b/website/README.md
@@ -7,8 +7,8 @@
An English static website with explanatory figures, worked guides and Aegis, DX-Ball, Notion, TH04 and CTF investigations.
The public files are in `website/public/`. The site uses HTML, CSS and small
scripts for copying code, following code comparisons and playing the dinosaur
-speed reconstruction. Python
-packages the downloadable example; there is no frontend bundler or npm dependency.
+speed reconstruction. Python prepares the example ZIP, sitemap and sharing
+image; there is no frontend bundler or npm dependency.
[style-guide.md](style-guide.md) explains the writing, page structure, figures,
visual system and review process. Read it before adding or revising a page.
@@ -18,11 +18,23 @@ visual system and review process. Read it before adding or revising a page.
From the repository root:
```sh
+python3 -m venv website/.venv
+source website/.venv/bin/activate
+python3 -m pip install -r website/requirements.txt
python3 scripts/prepare-website.py
python3 -m http.server 4173 --bind 127.0.0.1 --directory website/public
```
Open . Refresh the browser after editing a file.
+Keep the virtual environment active when preparing assets. CairoSVG needs the
+native Cairo library; the sharing card uses DejaVu Sans. On Debian or Ubuntu,
+install them with `sudo apt-get install libcairo2 fonts-dejavu-core`. For other
+systems, see [CairoSVG's installation instructions](https://cairosvg.org/documentation/#installation).
+CI installs both packages explicitly before rendering the card.
+
+After adding, moving or removing a page, or changing its indexing policy, rerun
+`python3 scripts/prepare-website.py`. It regenerates the sitemap from the current
+HTML files. No URL list needs to be maintained.
## Pages
@@ -44,6 +56,69 @@ Open . Refresh the browser after editing a file.
Navigation and assets use relative paths, so the same files work at the local
root and a GitHub Pages project path such as `/rea/`.
+## Search and sharing metadata
+
+`https://rea.tools/` is the preferred public origin. Each content page has one
+absolute canonical URL for its own route, including its trailing slash. The
+same HTML on GitHub Pages points to the corresponding rea.tools page. Ordinary
+navigation and asset references remain relative so both hosts and local previews
+keep working.
+
+Maintain a descriptive `` and a short meta description in each page's
+`<head>`. The title names the page's subject; visible headings keep the wording
+that best explains it to a reader. Open Graph and Twitter titles/descriptions
+match those fields. `og:url` matches the canonical, and sharing images use
+absolute rea.tools URLs. Search results and link previews should follow the same
+clear, concise writing standard as the page.
+
+`scripts/prepare-website.py` scans `public/**/*.html` and generates
+`public/sitemap.xml` with absolute rea.tools URLs. An `index.html` maps to its
+directory route; other HTML filenames retain their extension. Pages with
+`noindex` or `none` in a head `robots` or `googlebot` meta directive are excluded.
+The two Notes example applications use `noindex`; their guides and the dinosaur
+lab remain indexable. Downloads, scripts and other non-HTML files are not listed.
+The sitemap has no guessed modification dates, priority or change frequency.
+It is ignored by Git and regenerated before checks and publication.
+
+`public/robots.txt` allows crawling and names the canonical sitemap. Keep example
+applications crawlable so search engines can read their `noindex` directives.
+Cloudflare may prepend managed rules or comments to the served file; inspect the
+live response after publication. The mirrored `/rea/robots.txt` is not the
+hostname-root robots file for `morluto.github.io`; canonical HTML tags establish
+the mirror's preference.
+
+`public/assets/social-card.svg` is the maintained 1200 × 630 preview source.
+CairoSVG generates `social-card.png` for sharing clients. Commit the SVG, not the
+PNG. Both hosts receive the generated image in the same publishing artifact.
+The preview uses the site's typography, palette and direct headline.
+
+Verification checks route-specific canonicals, unique titles, descriptions,
+sharing fields, image dimensions, indexing policy and complete sitemap coverage.
+`scripts/test-website.py` exercises automatic route changes and rejects incorrect
+canonicals, lost fixture exclusions, stale sitemap entries and missing previews.
+Run it after preparing assets:
+
+```sh
+python3 scripts/verify-website.py
+python3 scripts/test-website.py
+```
+
+### After publishing an SEO change
+
+1. Check the served `robots.txt`, `sitemap.xml`, HTML head and preview PNG at
+ rea.tools. Confirm that both hosts contain the same page-specific canonicals.
+2. In the domain's Google Search Console property, submit
+ `https://rea.tools/sitemap.xml`.
+3. Inspect the homepage, a guide and a case study. Check crawl access, indexing
+ and Google's selected canonical; inspect a GitHub Pages URL as well if that
+ property is available. Track indexing and search performance there afterward.
+
+These Search Console checks require the property owner's access. Local checks
+verify the website inputs; Google's indexing reports show how they were used.
+See Google's [canonical guidance](https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls)
+and [sitemap guidance](https://developers.google.com/search/docs/crawling-indexing/sitemaps/build-sitemap),
+and Cloudflare's [managed robots behavior](https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/).
+
## Content
Keep the copy direct and specific. Explain the task and the result before listing
@@ -293,16 +368,18 @@ a release. Cloudflare deploys first; GitHub Pages deploys after that step succee
Preparation adds `deployment.json` with the workflow's commit SHA. Cloudflare
receives the extracted Pages artifact through Wrangler's explicit assets path;
-the local custom-build output is not the uploaded directory. The workflow then
+the production environment disables Wrangler's preview custom build, so deployment
+uses the prepared artifact directly. The workflow then
checks the selected public version markers and reports success only when they
match that commit. A failed publication remains failed; fix its cause and rerun the
workflow to complete both deployments. The two hosts can update at different
times while a run is in progress.
-Website checks run only for pull requests that change `website/`, the verification
-script or workflow definitions. They check local links, HTML fragments, SVG XML
-and the single Pages publisher without installing npm dependencies. The same
-checks run before each manual deployment. You can also run them locally:
+Website checks run only for pull requests that change `website/`, the website
+scripts or workflow definitions. They check local links, HTML fragments, SVG XML,
+search/sharing metadata and the single Pages publisher without installing npm
+dependencies. The same asset preparation and verification run before each manual
+deployment. With the preview environment active, run them locally:
```sh
python3 scripts/prepare-website.py
diff --git a/website/figures.md b/website/figures.md
index 5cf50ce29..72a32ac61 100644
--- a/website/figures.md
+++ b/website/figures.md
@@ -43,6 +43,15 @@ branches when changing these figures. Match progress claims to the pinned
reconstruction checkpoint. Validate the SVG as XML and inspect its browser
rendering at desktop and mobile widths.
+## Sharing preview
+
+`public/assets/social-card.svg` is the 1200 × 630 preview source. It uses the
+homepage's direct headline, the site's white/charcoal/slate/blue palette, and a
+short agent → REA → evidence flow. CairoSVG renders the ignored PNG during asset
+preparation. Check the rendered image for readable text and spacing before
+publishing. The build uses DejaVu Sans for consistent typography; only the SVG
+source is committed. Metadata and build instructions are in the website README.
+
## Beginner homepage and dinosaur reconstruction
The manual/agent comparison and Calculator percentage controls are semantic
diff --git a/website/style-guide.md b/website/style-guide.md
index a69683795..836e0f488 100644
--- a/website/style-guide.md
+++ b/website/style-guide.md
@@ -55,6 +55,13 @@ comes from, or what a check covers. Scope a claim positively and precisely:
“The compiled function matches 63 bytes” is more useful than a broad claim
followed by several qualifications.
+Search titles and sharing previews follow this standard too. Name the software
+or task so someone can understand the page before opening it. Keep titles and
+descriptions accurate and concise; use technical terms when they identify the
+actual topic. Preserve a clear visible headline rather than repeating search
+terms throughout the page. See [README.md](README.md#search-and-sharing-metadata)
+for canonical URLs, automatic sitemap generation and sharing assets.
+
## Build a page around one question
### Introductions and first exercises