From 5aa30b8b867836df578639e824d8b47bdc54f861 Mon Sep 17 00:00:00 2001
From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com>
Date: Sat, 10 Oct 2026 19:56:37 +0800
Subject: [PATCH] chore(website): simplify maintenance and detect documentation
drift
---
.github/workflows/website-check.yml | 8 +-
.github/workflows/website-pages.yml | 6 +-
scripts/audit-website-links.py | 135 ++++++++++
scripts/prepare-website.py | 37 +--
scripts/test-website.py | 116 +++++++++
scripts/verify-website.py | 121 +++++++--
scripts/website.py | 245 ++++++++++++++++++
website/AGENTS.md | 16 +-
website/README.md | 98 +++++--
website/public/blog/index.html | 10 +-
.../blog/touhou-reconstruction/index.html | 26 +-
website/public/examples/dino-lab/index.html | 19 +-
website/public/faq/index.html | 16 +-
website/public/first-investigation/index.html | 19 +-
website/public/get-started/index.html | 51 +++-
website/public/guides/browser/index.html | 23 +-
website/public/guides/index.html | 23 +-
website/public/guides/javascript/index.html | 13 +-
website/public/guides/native/index.html | 23 +-
website/public/index.html | 16 +-
website/public/showcase/aegis/index.html | 16 +-
website/public/showcase/ctf/index.html | 20 +-
website/public/showcase/dx-ball/index.html | 10 +-
website/public/showcase/index.html | 10 +-
website/public/showcase/notion/index.html | 14 +-
website/public/showcase/th04/index.html | 14 +-
website/style-guide.md | 6 +-
website/templates/footer.html | 16 ++
website/templates/header.html | 71 +++++
website/templates/page.html | 36 +++
website/templates/sharing.html | 40 +++
website/wrangler.toml | 2 +-
32 files changed, 1108 insertions(+), 168 deletions(-)
create mode 100644 scripts/audit-website-links.py
create mode 100644 scripts/website.py
create mode 100644 website/templates/footer.html
create mode 100644 website/templates/header.html
create mode 100644 website/templates/page.html
create mode 100644 website/templates/sharing.html
diff --git a/.github/workflows/website-check.yml b/.github/workflows/website-check.yml
index 26270ce73..676a9c83a 100644
--- a/.github/workflows/website-check.yml
+++ b/.github/workflows/website-check.yml
@@ -35,9 +35,5 @@ jobs:
sudo apt-get update
sudo apt-get install --no-install-recommends -y libcairo2 fonts-dejavu-core
python3 -m pip install -r website/requirements.txt
- - name: Prepare website assets
- run: python3 scripts/prepare-website.py
- - name: Check website links, metadata and publishing configuration
- run: python3 scripts/verify-website.py
- - name: Check sitemap discovery and metadata regressions
- run: python3 scripts/test-website.py
+ - name: Check shared HTML, links, assets and contributor workflows
+ run: python3 scripts/website.py check
diff --git a/.github/workflows/website-pages.yml b/.github/workflows/website-pages.yml
index 7dec1ea47..3d20f7ea1 100644
--- a/.github/workflows/website-pages.yml
+++ b/.github/workflows/website-pages.yml
@@ -59,10 +59,8 @@ jobs:
sudo apt-get update
sudo apt-get install --no-install-recommends -y libcairo2 fonts-dejavu-core
python3 -m pip install -r website/requirements.txt
- - name: Prepare website assets
- run: python3 scripts/prepare-website.py
- - name: Check website links, metadata and publishing configuration
- run: python3 scripts/verify-website.py
+ - name: Check shared HTML, links, assets and contributor workflows
+ run: python3 scripts/website.py check
- name: Record the published commit
run: |
python3 - <<'PY'
diff --git a/scripts/audit-website-links.py b/scripts/audit-website-links.py
new file mode 100644
index 000000000..cff49da0b
--- /dev/null
+++ b/scripts/audit-website-links.py
@@ -0,0 +1,135 @@
+"""Optionally audit external website links; network failures stay distinct from missing targets."""
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache
+from html.parser import HTMLParser
+import json
+from pathlib import Path
+import re
+import subprocess
+import sys
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote, unquote, urlsplit
+from urllib.request import Request, urlopen
+
+from website import authored_markdown
+
+
+@lru_cache(maxsize=None)
+def github_api(endpoint):
+ """Read GitHub evidence through the caller's gh installation and authentication."""
+ result = subprocess.run(["gh", "api", endpoint], capture_output=True, text=True, timeout=20)
+ if result.returncode:
+ raise ValueError(result.stderr.strip())
+ return json.loads(result.stdout)
+
+
+def github_target(url):
+ """Resolve repository URLs, including branch names containing slashes."""
+ parts = [unquote(part) for part in url.path.strip("/").split("/")]
+ if len(parts) < 2:
+ return None
+ repo = f"repos/{parts[0]}/{parts[1]}"
+ if url.hostname == "raw.githubusercontent.com" and len(parts) >= 4:
+ return f"{repo}/contents/{'/'.join(parts[3:])}?ref={quote(parts[2], safe='')}"
+ if len(parts) == 2:
+ return repo
+ kind = parts[2]
+ if kind in ("blob", "tree") and len(parts) >= 4:
+ ref = parts[3]
+ # GitHub uses the longest matching branch name before the source path.
+ if not re.fullmatch(r"[0-9a-f]{40}", ref):
+ references = github_api(f"{repo}/git/matching-refs/heads/{quote(ref, safe='')}")
+ references += github_api(f"{repo}/git/matching-refs/tags/{quote(ref, safe='')}")
+ suffix = "/".join(parts[3:])
+ matches = [item["ref"].split("/", 2)[2] for item in references]
+ matches = [name for name in matches if suffix == name or suffix.startswith(name + "/")]
+ if matches:
+ ref = max(matches, key=len)
+ path = "/".join(parts[3:])[len(ref):].lstrip("/")
+ return f"{repo}/contents/{quote(path, safe='/')}?ref={quote(ref, safe='')}"
+ if kind in ("issues", "pull") and len(parts) == 4 and parts[3].isdigit():
+ return f"{repo}/{'pulls' if kind == 'pull' else 'issues'}/{parts[3]}"
+ if kind == "commit" and len(parts) == 4:
+ return f"{repo}/commits/{parts[3]}"
+ if kind == "releases" and len(parts) >= 5 and parts[3] in ("tag", "download"):
+ return f"{repo}/releases/tags/{quote(parts[4], safe='')}"
+ return None
+
+
+def audit_url(item):
+ """Classify confirmed missing links separately from blocked or unsupported probes."""
+ value, sources = item
+ result = {"url": value, "sources": sorted(set(sources))}
+ url = urlsplit(value)
+ try:
+ if url.hostname in ("github.com", "raw.githubusercontent.com"):
+ endpoint = github_target(url)
+ if endpoint is None:
+ return dict(result, status="unknown", reason="GitHub UI route; check in a browser")
+ data = github_api(endpoint)
+ parts = url.path.strip("/").split("/")
+ if len(parts) >= 6 and parts[2:4] == ["releases", "download"]:
+ if unquote(parts[5]) not in [asset["name"] for asset in data.get("assets", [])]:
+ return dict(result, status="broken", reason="Release asset is missing")
+ return dict(result, status="ok", via="gh api", endpoint=endpoint)
+ request = Request(value.split("#", 1)[0], headers={
+ "User-Agent": "REA-Website-Verifier/1.0 (+https://github.com/morluto/rea)",
+ })
+ with urlopen(request, timeout=15) as response:
+ return dict(result, status="ok", http_status=response.status, final_url=response.url)
+ except HTTPError as error:
+ return dict(result, status="broken" if error.code in (404, 410) else "unknown", reason=str(error))
+ except ValueError as error:
+ return dict(result, status="broken" if "HTTP 404" in str(error) else "unknown", reason=str(error))
+ except (OSError, URLError, TimeoutError, subprocess.TimeoutExpired) as error:
+ return dict(result, status="unknown", reason=str(error))
+
+
+def external_links(root):
+ """Discover authored external links, excluding locally checked site and preview URLs."""
+ references = {}
+
+ def add(value, source):
+ url = urlsplit(value)
+ if url.scheme in ("http", "https") and url.hostname not in ("rea.tools", "localhost", "127.0.0.1", "::1"):
+ references.setdefault(value, []).append(str(source.relative_to(root)))
+
+ class Links(HTMLParser):
+ def handle_starttag(self, tag, attrs):
+ for key, value in attrs:
+ if key in ("href", "src") and value:
+ add(value, path)
+
+ for path in sorted((root / "website/public").rglob("*.html")):
+ Links().feed(path.read_text(encoding="utf-8"))
+ for path in authored_markdown(root):
+ for value in re.findall(r"(?:\]\(|<)(https?://[^\s<>\)]+)", path.read_text(encoding="utf-8")):
+ add(value, path)
+ return references
+
+
+def main():
+ """Run an explicit network audit without making PR checks depend on third-party uptime."""
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--report", type=Path, help="Write the complete JSON report outside public assets")
+ args = parser.parse_args()
+ root = Path(__file__).resolve().parent.parent
+ with ThreadPoolExecutor(max_workers=4) as pool:
+ results = list(pool.map(audit_url, external_links(root).items()))
+ if args.report:
+ if args.report.resolve().is_relative_to(root / "website/public"):
+ parser.error("Keep audit reports outside website/public.")
+ args.report.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
+ counts = {status: sum(item["status"] == status for item in results) for status in ("ok", "broken", "unknown")}
+ print(f"External links: {counts['ok']} reachable, {counts['broken']} missing, {counts['unknown']} need manual verification.")
+ print("GitHub checks establish target existence; external fragment IDs are not checked.")
+ for item in results:
+ if item["status"] != "ok":
+ print(f"{item['status']}: {item['url']} — {item['reason']}")
+ return 1 if counts["broken"] else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/prepare-website.py b/scripts/prepare-website.py
index af4f2d9b9..848b6fbc4 100644
--- a/scripts/prepare-website.py
+++ b/scripts/prepare-website.py
@@ -1,16 +1,14 @@
"""Prepare downloads, the sitemap and the sharing image for the static website."""
-from html.parser import HTMLParser
from io import BytesIO
from pathlib import Path
-import re
import sys
-from urllib.parse import quote
import xml.etree.ElementTree as ET
from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
+from website import PageMetadata, page_url
+
-SITE_ORIGIN = "https://rea.tools"
EXAMPLE_FILES = (
"package.json",
"main.js",
@@ -21,47 +19,20 @@ EXAMPLE_FILES = (
)
-class IndexPolicy(HTMLParser):
- """Read indexing directives from the authored HTML head."""
-
- def __init__(self):
- super().__init__()
- self.in_head = False
- self.noindex = False
-
- def handle_starttag(self, tag, attrs):
- if tag == "head":
- self.in_head = True
- if tag == "meta" and self.in_head:
- attributes = dict(attrs)
- if (attributes.get("name") or "").lower() in ("robots", "googlebot"):
- directives = re.split(
- r"[\s,]+", (attributes.get("content") or "").lower()
- )
- self.noindex |= bool(set(directives) & {"noindex", "none"})
-
- def handle_endtag(self, tag):
- if tag == "head":
- self.in_head = False
-
-
def prepare_sitemap(site):
"""Discover HTML routes automatically, excluding pages marked noindex."""
namespace = "http://www.sitemaps.org/schemas/sitemap/0.9"
ET.register_namespace("", namespace)
sitemap = ET.Element(f"{{{namespace}}}urlset")
for path in sorted(site.rglob("*.html")):
- policy = IndexPolicy()
+ policy = PageMetadata()
policy.feed(path.read_text(encoding="utf-8"))
if policy.noindex:
continue
- route = path.relative_to(site).as_posix()
- if path.name == "index.html":
- route = route.removesuffix("index.html")
entry = ET.SubElement(sitemap, f"{{{namespace}}}url")
ET.SubElement(
entry, f"{{{namespace}}}loc"
- ).text = f"{SITE_ORIGIN}/{quote(route, safe='/')}"
+ ).text = page_url(site, path)
ET.indent(sitemap, space=" ")
ET.ElementTree(sitemap).write(
site / "sitemap.xml", encoding="utf-8", xml_declaration=True
diff --git a/scripts/test-website.py b/scripts/test-website.py
index 121dec8ae..43d913548 100644
--- a/scripts/test-website.py
+++ b/scripts/test-website.py
@@ -6,7 +6,11 @@ import io
from pathlib import Path
import shutil
import tempfile
+from threading import Thread
import unittest
+from http.server import ThreadingHTTPServer
+from urllib.error import HTTPError
+from urllib.request import urlopen
import xml.etree.ElementTree as ET
@@ -23,6 +27,7 @@ def load_script(name):
PREPARE = load_script("prepare-website")
VERIFY = load_script("verify-website")
+WEBSITE = load_script("website")
class WebsiteSearchChecks(unittest.TestCase):
@@ -126,5 +131,116 @@ class WebsiteSearchChecks(unittest.TestCase):
)
+class WebsiteMaintenanceChecks(unittest.TestCase):
+ """Exercise contributor edits and the static preview through real consumers."""
+
+ def test_shared_edits_preserve_body_and_catch_drift_before_rewriting(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ for name in ("public", "templates"):
+ shutil.copytree(ROOT / "website" / name, root / "website" / name)
+ page = root / "website/public/blog/touhou-reconstruction/index.html"
+ original = page.read_text()
+ body = original.split('", 1)[0]
+ template = root / "website/templates/header.html"
+ template.write_text(template.read_text().replace("Reverse Engineer Anything", "REA test navigation"))
+ with self.assertRaisesRegex(ValueError, "Shared HTML is stale"):
+ WEBSITE.sync_pages(root, check=True)
+ self.assertEqual(page.read_text(), original)
+ WEBSITE.sync_pages(root)
+ updated = page.read_text()
+ self.assertIn("REA test navigation", updated)
+ self.assertEqual(updated.split('", 1)[0], body)
+ self.assertEqual(WEBSITE.sync_pages(root), [])
+ page.write_text(updated.replace(' ', '', 1))
+ with self.assertRaisesRegex(ValueError, "needs one header region"):
+ WEBSITE.sync_pages(root)
+
+ def test_new_page_escapes_metadata_stays_a_draft_and_joins_sitemap_when_ready(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ for name in ("public", "templates"):
+ shutil.copytree(ROOT / "website" / name, root / "website" / name)
+ site = root / "website/public"
+ page = WEBSITE.new_page(root, "blog/new-example", 'A & "result"', 'Check "quotes" & .')
+ source = page.read_text()
+ self.assertIn('href="../../blog/" aria-current="page"', source)
+ self.assertIn('href="../../assets/styles.css"', source)
+ self.assertIn("A <test> & "result"", source)
+ with self.assertRaisesRegex(ValueError, "Page already exists"):
+ WEBSITE.new_page(root, "blog/new-example", "Another page", "Description")
+ with self.assertRaises(ValueError):
+ WEBSITE.new_page(root, "../outside", "Title", "Description")
+ WEBSITE.sync_pages(root, check=True)
+ PREPARE.prepare_sitemap(site)
+ self.assertNotIn("https://rea.tools/blog/new-example/", (site / "sitemap.xml").read_text())
+ page.write_text(source.replace(' \n', ''))
+ PREPARE.prepare_sitemap(site)
+ self.assertIn("https://rea.tools/blog/new-example/", (site / "sitemap.xml").read_text())
+ self.assertEqual(VERIFY.check_site(site)[3], [])
+
+ def test_links_in_prompts_fragments_and_copy_controls_are_verified(self):
+ with tempfile.TemporaryDirectory() as directory:
+ site = Path(directory) / "public"
+ shutil.copytree(ROOT / "website/public", site)
+ page = site / "faq/index.html"
+ original = page.read_text()
+ cases = (
+ ('Broken', "missing local target"),
+ ('https://rea.tools/guides/native/#deleted', "missing HTML fragment"),
+ ('https://morluto.github.io/rea/examples/notes-web/', "obsolete website URL"),
+ ('', "missing copy target"),
+ ('Malformed', "invalid URL"),
+ )
+ for inserted, message in cases:
+ with self.subTest(message=message):
+ page.write_text(original.replace("", inserted + "", 1))
+ self.assertTrue(any(message in error for error in VERIFY.check_site(site)[3]))
+
+ def test_repository_links_reject_missing_files_and_old_markdown_anchors(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ (root / "website/public").mkdir(parents=True)
+ (root / "docs").mkdir()
+ (root / "docs/guide.md").write_text("# Guide\n## Set up REA\n")
+ page = root / "website/README.md"
+ page.write_text("[Guide](../docs/guide.md#set-up-rea)\n")
+ (root / "website/.venv").mkdir()
+ (root / "website/.venv/vendor.md").write_text("[File](missing.md)")
+ self.assertEqual(VERIFY.check_repository_links(root), [])
+ page.write_text("[Guide](../docs/guide.md#removed)\n[File](../docs/removed.md)\n")
+ errors = VERIFY.check_repository_links(root)
+ self.assertTrue(any("missing Markdown fragment" in error for error in errors))
+ self.assertTrue(any("missing repository target" in error for error in errors))
+
+ def test_preview_preserves_pages_prefix_for_assets_and_directory_redirects(self):
+ with tempfile.TemporaryDirectory() as directory:
+ site = Path(directory)
+ (site / "guide").mkdir()
+ (site / "guide/index.html").write_text("
Guide
")
+ (site / "asset.css").write_text("body { color: black; }")
+ for prefix in ("/", "/rea/"):
+ with self.subTest(prefix=prefix), ThreadingHTTPServer(("127.0.0.1", 0), WEBSITE.preview_handler(site, prefix)) as server:
+ worker = Thread(target=server.serve_forever, daemon=True)
+ worker.start()
+ try:
+ base = f"http://127.0.0.1:{server.server_port}"
+ with urlopen(base + prefix + "guide?test=1") as response:
+ self.assertEqual(response.url, base + prefix + "guide/?test=1")
+ self.assertEqual(response.read(), b"
Guide
")
+ with urlopen(base + prefix + "asset.css") as response:
+ self.assertIn(b"color: black", response.read())
+ with self.assertRaises(HTTPError) as error:
+ urlopen(base + prefix + "missing")
+ self.assertEqual(error.exception.code, 404)
+ if prefix != "/":
+ with self.assertRaises(HTTPError) as error:
+ urlopen(base + "/guide/")
+ self.assertEqual(error.exception.code, 404)
+ finally:
+ server.shutdown()
+ worker.join()
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/scripts/verify-website.py b/scripts/verify-website.py
index d67a89284..889d77394 100644
--- a/scripts/verify-website.py
+++ b/scripts/verify-website.py
@@ -5,13 +5,79 @@ from pathlib import Path
import re
import struct
import sys
-from urllib.parse import quote, unquote, urlsplit
+import unicodedata
+from urllib.parse import unquote, urlsplit
import xml.etree.ElementTree as ET
from zipfile import BadZipFile, ZipFile
+from website import NOTES_FIXTURES, SITE_ORIGIN, authored_markdown, page_url, sync_pages
-SITE_ORIGIN = "https://rea.tools"
-NOTES_FIXTURES = {"examples/notes-electron/index.html", "examples/notes-web/index.html"}
+LEGACY_ORIGIN = "morluto.github.io"
+SITE_TEXT_URL = re.compile(
+ r"https?://(?:rea\.tools|morluto\.github\.io)/[^\s<>\"'`]+", re.IGNORECASE
+)
+
+
+def markdown_ids(path):
+ """Collect GitHub heading anchors and explicit HTML IDs in local Markdown."""
+ source = path.read_text(encoding="utf-8")
+ parser = HtmlReferences()
+ parser.feed(source)
+ identifiers = set(parser.ids)
+ counts = {}
+ fenced = False
+ for line in source.splitlines():
+ if line.lstrip().startswith(("```", "~~~")):
+ fenced = not fenced
+ if fenced:
+ continue
+ heading = re.match(r"^ {0,3}#{1,6}\s+(.+?)(?:\s+#+)?\s*$", line)
+ if not heading:
+ continue
+ text = re.sub(r"<[^>]*>", "", heading[1]).lower()
+ slug = "".join(c for c in text if c in "-_ " or unicodedata.category(c)[0] in "LN").replace(" ", "-")
+ count = counts.get(slug, 0)
+ identifiers.add(f"{slug}-{count}" if count else slug)
+ counts[slug] = count + 1
+ return identifiers
+
+
+def check_repository_links(root):
+ """Check local links in website Markdown and current-repository source links."""
+ errors = []
+ sources = {}
+ for path in authored_markdown(root):
+ text = re.sub(r"(?ms)^\s*(```|~~~).*?^\s*\1[^\n]*$", "", path.read_text(encoding="utf-8"))
+ text = re.sub(r"`[^`]*`", "", text)
+ sources[path] = re.findall(r"\[[^\]\n]*\]\(([^\s)]+)\)", text)
+ for path in sorted((root / "website/public").rglob("*.html")):
+ parser = HtmlReferences()
+ parser.feed(path.read_text(encoding="utf-8"))
+ sources[path] = parser.references
+ for path, references in sources.items():
+ for reference in references:
+ try:
+ url = urlsplit(reference)
+ except ValueError as error:
+ errors.append(f"{path.relative_to(root)}: invalid URL {reference!r}: {error}")
+ continue
+ if url.hostname == LEGACY_ORIGIN and (url.path == "/rea" or url.path.startswith("/rea/")):
+ errors.append(f"{path.relative_to(root)}: obsolete website URL: {reference}")
+ continue
+ if url.hostname == "github.com":
+ match = re.match(r"^/morluto/rea/(?:blob|tree)/main/(.+)$", url.path)
+ if not match:
+ continue
+ target = (root / unquote(match[1])).resolve()
+ elif url.scheme or url.netloc or path.suffix == ".html":
+ continue
+ else:
+ target = (path.parent / unquote(url.path)).resolve() if url.path else path
+ if not target.is_relative_to(root) or not target.exists():
+ errors.append(f"{path.relative_to(root)}: missing repository target: {reference}")
+ elif target.suffix == ".md" and url.fragment and unquote(url.fragment) not in markdown_ids(target):
+ errors.append(f"{path.relative_to(root)}: missing Markdown fragment: {reference}")
+ return errors
class HtmlReferences(HTMLParser):
@@ -30,6 +96,8 @@ class HtmlReferences(HTMLParser):
self.canonicals = []
self.lang = None
self.h1_count = 0
+ self.text = []
+ self.copy_targets = []
def handle_starttag(self, tag, attrs):
attributes = dict(attrs)
@@ -67,6 +135,8 @@ class HtmlReferences(HTMLParser):
reference = attributes.get(attribute)
if reference is not None:
self.references.append(reference)
+ if attributes.get("data-copy"):
+ self.copy_targets.append(attributes["data-copy"])
def handle_endtag(self, tag):
if tag == "head":
@@ -75,6 +145,7 @@ class HtmlReferences(HTMLParser):
self.in_title = False
def handle_data(self, data):
+ self.text.append(data)
if self.in_title:
self.titles[-1] += data
@@ -104,15 +175,7 @@ def check_search_metadata(site, pages):
)
if page.noindex():
continue
- route = (
- path.relative_to(site).parent.as_posix()
- if path.name == "index.html"
- else label
- )
- route = "" if route == "." else route
- url = f"{SITE_ORIGIN}/{quote(route, safe='/')}" + (
- "/" if route and path.name == "index.html" else ""
- )
+ url = page_url(site, path)
indexable.add(url)
if page.canonicals != [url]:
errors.append(f"{label}: needs exactly one head canonical for {url}")
@@ -233,15 +296,32 @@ def check_site(site):
label = path.relative_to(site)
for identifier in page.duplicate_ids:
errors.append(f"{label}: duplicate ID {identifier!r}")
- for reference in page.references:
- url = urlsplit(reference)
- if url.scheme or url.netloc:
+ for identifier in page.copy_targets:
+ if identifier not in page.ids:
+ errors.append(f"{label}: missing copy target: {identifier}")
+ text_urls = [match.group().rstrip(".,;)") for match in SITE_TEXT_URL.finditer(" ".join(page.text))]
+ for reference in page.references + text_urls:
+ try:
+ url = urlsplit(reference)
+ except ValueError as error:
+ errors.append(f"{label}: invalid URL {reference!r}: {error}")
continue
- references += 1
- if url.path.startswith("/"):
+ if url.hostname == LEGACY_ORIGIN and (url.path == "/rea" or url.path.startswith("/rea/")):
+ errors.append(f"{label}: obsolete website URL: {reference}; use rea.tools")
+ continue
+ if url.hostname == "rea.tools":
+ if url.scheme != "https" or url.netloc != "rea.tools":
+ errors.append(f"{label}: website URL must use {SITE_ORIGIN}: {reference}")
+ continue
+ target = (site / unquote(url.path).lstrip("/")).resolve()
+ elif url.scheme or url.netloc:
+ continue
+ elif url.path.startswith("/"):
errors.append(f"{label}: use a relative site reference: {reference}")
continue
- target = (path.parent / unquote(url.path)).resolve() if url.path else path
+ else:
+ target = (path.parent / unquote(url.path)).resolve() if url.path else path
+ references += 1
if not target.is_relative_to(site):
errors.append(f"{label}: reference leaves the website: {reference}")
continue
@@ -323,6 +403,11 @@ def main():
pages, references, indexable, errors = check_site(root / "website/public")
errors.extend(check_example_archive(root / "website/public"))
errors.extend(check_publisher(root))
+ errors.extend(check_repository_links(root))
+ try:
+ sync_pages(root, check=True)
+ except (OSError, ValueError, KeyError) as error:
+ errors.append(str(error))
if errors:
print("Website checks failed:\n" + "\n".join(errors), file=sys.stderr)
return 1
diff --git a/scripts/website.py b/scripts/website.py
new file mode 100644
index 000000000..77099d5bf
--- /dev/null
+++ b/scripts/website.py
@@ -0,0 +1,245 @@
+"""Maintain shared HTML and run the static website's contributor commands."""
+
+import argparse
+from functools import partial
+from html import escape
+from html.parser import HTMLParser
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path, PurePosixPath
+from string import Template
+import subprocess
+import sys
+from urllib.parse import quote, urlsplit
+
+
+SITE_ORIGIN = "https://rea.tools"
+NOTES_FIXTURES = {"examples/notes-electron/index.html", "examples/notes-web/index.html"}
+NAVIGATION = (
+ ("Showcases", "showcase/"),
+ ("Blog", "blog/"),
+ ("Get started", "get-started/"),
+ ("Guides", "guides/"),
+ ("FAQ", "faq/"),
+)
+
+
+def authored_markdown(root):
+ """Discover website documentation while excluding local environments and vendors."""
+ site = root / "website"
+ return [
+ path for path in sorted(site.rglob("*.md"))
+ if not any(part.startswith(".") or part in ("node_modules", "__pycache__")
+ for part in path.relative_to(site).parts)
+ ]
+
+
+class PageMetadata(HTMLParser):
+ """Read the authored title, description and indexing policy from the head."""
+
+ def __init__(self):
+ super().__init__()
+ self.in_head = False
+ self.in_title = False
+ self.titles = []
+ self.descriptions = []
+ self.noindex = False
+
+ def handle_starttag(self, tag, attrs):
+ if tag == "head":
+ self.in_head = True
+ if not self.in_head:
+ return
+ if tag == "title":
+ self.in_title = True
+ self.titles.append("")
+ if tag == "meta":
+ attributes = dict(attrs)
+ name = (attributes.get("name") or "").lower()
+ content = attributes.get("content") or ""
+ if name == "description":
+ self.descriptions.append(content)
+ if name in ("robots", "googlebot"):
+ self.noindex |= bool(
+ set(content.lower().replace(",", " ").split()) & {"noindex", "none"}
+ )
+
+ def handle_endtag(self, tag):
+ if tag == "head":
+ self.in_head = False
+ if tag == "title":
+ self.in_title = False
+
+ def handle_data(self, data):
+ if self.in_title:
+ self.titles[-1] += data
+
+
+def page_route(site, path):
+ """Map an HTML file to its public route without a separate page registry."""
+ route = path.relative_to(site).as_posix()
+ return route.removesuffix("index.html") if path.name == "index.html" else route
+
+
+def page_url(site, path):
+ """Return the canonical public URL of an authored HTML file."""
+ return f"{SITE_ORIGIN}/{quote(page_route(site, path), safe='/')}"
+
+
+def shared_blocks(root, path, source):
+ """Render shared regions while keeping the page's body and metadata authored."""
+ site = root / "website/public"
+ metadata = PageMetadata()
+ metadata.feed(source)
+ for name, values in (("title", metadata.titles), ("description", metadata.descriptions)):
+ if len(values) != 1 or not values[0].strip():
+ raise ValueError(f"{path.relative_to(site)}: needs one nonempty head {name}")
+ depth = len(path.relative_to(site).parent.parts)
+ base = "../" * depth or "./"
+ route = page_route(site, path)
+ links = []
+ for label, destination in NAVIGATION:
+ attributes = ' class="nav-start"' if destination == "get-started/" else ""
+ if route.startswith(destination):
+ attributes += ' aria-current="page"'
+ links.append(f' {label}')
+ values = {
+ "base": base,
+ "navigation": "\n".join(links),
+ "canonical": escape(page_url(site, path), quote=True),
+ "origin": SITE_ORIGIN,
+ "title": escape(metadata.titles[0].strip(), quote=True),
+ "description": escape(metadata.descriptions[0].strip(), quote=True),
+ }
+ templates = root / "website/templates"
+ return {
+ name: Template((templates / f"{name}.html").read_text(encoding="utf-8")).substitute(values)
+ for name in ("sharing", "header", "footer")
+ }
+
+
+def sync_pages(root, check=False):
+ """Update shared regions, or reject drift without changing any authored file."""
+ site = root / "website/public"
+ changes = []
+ for path in sorted(site.rglob("*.html")):
+ if path.relative_to(site).as_posix() in NOTES_FIXTURES:
+ continue
+ original = path.read_text(encoding="utf-8")
+ source = original
+ for name, block in shared_blocks(root, path, source).items():
+ start = f" \n"
+ end = f" "
+ if source.count(start) != 1 or source.count(end) != 1:
+ raise ValueError(f"{path.relative_to(site)}: needs one {name} region; use new-page for a new reading page")
+ before, _, remaining = source.partition(start)
+ _, found, after = remaining.partition(end)
+ if not found:
+ raise ValueError(f"{path.relative_to(site)}: {name} region is out of order")
+ source = before + start + block + end + after
+ if source != original:
+ changes.append((path, source))
+ if check and changes:
+ names = ", ".join(str(path.relative_to(site)) for path, _ in changes)
+ raise ValueError(f"Shared HTML is stale: {names}. Run python3 scripts/website.py sync and commit the updated pages.")
+ if not check:
+ for path, source in changes:
+ path.write_text(source, encoding="utf-8")
+ print(f"{'Checked' if check else 'Synced'} shared HTML ({len(changes)} pages changed).")
+ return [path for path, _ in changes]
+
+
+def new_page(root, route, title, description):
+ """Create a draft reading page with correct depth, metadata and shared regions."""
+ parts = PurePosixPath(route)
+ if not route or parts.is_absolute() or ".." in parts.parts or "\\" in route or parts.suffix:
+ raise ValueError("Use a relative directory route, such as blog/my-article.")
+ site = root / "website/public"
+ path = site.joinpath(*parts.parts, "index.html")
+ if not path.resolve().is_relative_to(site.resolve()):
+ raise ValueError("The new page must stay inside website/public.")
+ if path.exists():
+ raise ValueError(f"Page already exists: {path.relative_to(site)}")
+ base = "../" * len(parts.parts) or "./"
+ source = Template((root / "website/templates/page.html").read_text(encoding="utf-8")).substitute(
+ title=escape(title, quote=True), description=escape(description, quote=True), base=base,
+ )
+ # Render before writing, so invalid input cannot leave an incomplete page.
+ blocks = shared_blocks(root, path, source)
+ for name, block in blocks.items():
+ source = source.replace(f" \n", f" \n{block}")
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(source, encoding="utf-8")
+ print(f"Created {path.relative_to(root)}. Write the page, add its index link, then remove the Draft label and noindex directive when ready.")
+ return path
+
+
+def run_script(root, name, arguments=()):
+ """Use the active Python environment for the existing preparation and checks."""
+ subprocess.run([sys.executable, str(root / "scripts" / name), *arguments], cwd=root, check=True)
+
+
+def preview_handler(site, prefix):
+ """Serve one static directory at root or a Pages prefix, preserving redirects."""
+ class Handler(SimpleHTTPRequestHandler):
+ def send_head(self):
+ if not urlsplit(self.path).path.startswith(prefix):
+ self.send_error(404)
+ return None
+ return super().send_head()
+
+ def translate_path(self, path):
+ return super().translate_path("/" + path[len(prefix):])
+
+ return partial(Handler, directory=str(site))
+
+
+def main():
+ """Expose one small command for synchronizing, checking and previewing the site."""
+ parser = argparse.ArgumentParser(description=__doc__)
+ commands = parser.add_subparsers(dest="command", required=True)
+ sync = commands.add_parser("sync", help="Update shared HTML after editing templates or page metadata")
+ sync.add_argument("--check", action="store_true", help="Report drift without rewriting pages")
+ commands.add_parser("check", help="Check shared HTML, prepare assets, verify links and run regression tests")
+ links = commands.add_parser("links", help="Audit external targets using gh and HTTP; separate from offline checks")
+ links.add_argument("--report", help="Write the full external-link JSON report")
+ create = commands.add_parser("new-page", help="Create a draft reading page")
+ create.add_argument("route")
+ create.add_argument("--title", required=True)
+ create.add_argument("--description", required=True)
+ serve = commands.add_parser("serve", help="Prepare and preview the static site")
+ serve.add_argument("--port", type=int, default=4173)
+ serve.add_argument("--base-path", choices=("/", "/rea/"), default="/")
+ args = parser.parse_args()
+ root = Path(__file__).resolve().parent.parent
+ try:
+ if args.command == "new-page":
+ new_page(root, args.route, args.title, args.description)
+ elif args.command == "sync":
+ sync_pages(root, check=args.check)
+ elif args.command == "check":
+ sync_pages(root, check=True)
+ for name in ("prepare-website.py", "verify-website.py", "test-website.py"):
+ run_script(root, name)
+ elif args.command == "links":
+ run_script(root, "audit-website-links.py", ["--report", args.report] if args.report else [])
+ else:
+ sync_pages(root)
+ run_script(root, "prepare-website.py")
+ prefix = args.base_path
+
+ handler = preview_handler(root / "website/public", prefix)
+ with ThreadingHTTPServer(("127.0.0.1", args.port), handler) as server:
+ print(f"Preview: http://127.0.0.1:{server.server_port}{prefix}", flush=True)
+ server.serve_forever()
+ except (OSError, ValueError, KeyError) as error:
+ print(str(error), file=sys.stderr)
+ return 1
+ except subprocess.CalledProcessError as error:
+ return error.returncode
+ except KeyboardInterrupt:
+ return 0
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/website/AGENTS.md b/website/AGENTS.md
index e97181ec6..92824a294 100644
--- a/website/AGENTS.md
+++ b/website/AGENTS.md
@@ -12,6 +12,12 @@ for asset notes and the source of case-study claims.
should explain a new fact or action.
- Reuse the shared styles, prompts, figures and code-step components. Keep a
case study focused on one question, with further examples in native details.
+- Edit shared header, footer and sharing markup in `templates/`; run
+ `python3 scripts/website.py sync` and commit the rendered regions with their
+ templates. Keep page bodies outside the marked regions. Author each title and
+ description once; derive sharing fields and canonicals from them. Use
+ `new-page` for a reading-page scaffold, then add its index link. Keep a draft's
+ visible label and `noindex` until the content is ready.
- Show what REA returns and how the agent uses it. Preserve source attribution,
target identity and verification scope.
- Blog articles teach methods, decisions and workflows through concrete cases.
@@ -27,9 +33,13 @@ for asset notes and the source of case-study claims.
from HTML automatically. Mark example applications `noindex` in the head and
keep them crawlable. Maintain the SVG sharing source, not the generated PNG.
- Check both local root and `/rea/` paths, mobile layouts, expanded details,
- copying and downloads. Run `scripts/prepare-website.py` followed by
- `scripts/verify-website.py` and `scripts/test-website.py`, using the website
- Python environment described in the README.
+ copying and downloads. Run `python3 scripts/website.py check`, using the Python
+ environment described in the README. It rejects drift rather than silently
+ rewriting shared HTML. Use `website.py links` for an optional network audit;
+ distinguish missing targets from access denials and unverified fragments.
+- Check setup and support statements against the current installation/provider
+ guides and released package. Preserve historical case-study checkpoints and
+ versions unless a new investigation supplies replacement evidence.
- Keep `website-pages.yml` as the sole, manual production publisher for
Cloudflare and Pages. Default to `both`: both hosts receive the same verified
artifact and pass the published-commit check. Use `pages` only when explicitly
diff --git a/website/README.md b/website/README.md
index 298bc4b80..d5fbf121d 100644
--- a/website/README.md
+++ b/website/README.md
@@ -7,8 +7,9 @@
An English static website with explanatory figures, worked guides and Aegis, DX-Ball, Notion, TH04 and CTF investigations.
The public files are in `website/public/`. The site uses HTML, CSS and small
scripts for copying code, following code comparisons and playing the dinosaur
-speed reconstruction. Python prepares the example ZIP, sitemap and sharing
-image; there is no frontend bundler or npm dependency.
+speed reconstruction. Small Python commands maintain shared HTML, prepare
+downloads and check the site. There is no frontend framework, bundler or npm
+dependency. The checked-in HTML works as ordinary static files.
[style-guide.md](style-guide.md) explains the writing, page structure, figures,
visual system and review process. Read it before adding or revising a page.
@@ -21,8 +22,7 @@ From the repository root:
python3 -m venv website/.venv
source website/.venv/bin/activate
python3 -m pip install -r website/requirements.txt
-python3 scripts/prepare-website.py
-python3 -m http.server 4173 --bind 127.0.0.1 --directory website/public
+python3 scripts/website.py serve
```
Open . Refresh the browser after editing a file.
@@ -32,9 +32,77 @@ install them with `sudo apt-get install libcairo2 fonts-dejavu-core`. For other
systems, see [CairoSVG's installation instructions](https://cairosvg.org/documentation/#installation).
CI installs both packages explicitly before rendering the card.
-After adding, moving or removing a page, or changing its indexing policy, rerun
-`python3 scripts/prepare-website.py`. It regenerates the sitemap from the current
-HTML files. No URL list needs to be maintained.
+To test the GitHub Pages prefix, use
+`python3 scripts/website.py serve --port 4174 --base-path /rea/` and open
+.
+
+## Contribute a page or correction
+
+Edit the page's HTML in `public/`. Keep the question, evidence and result easy to
+follow. The writing and design guidance in [style-guide.md](style-guide.md)
+applies to every page.
+
+| Change | Edit | Then run |
+| ------------------------------ | ----------------------------------------------------------- | ----------------------------------------------- |
+| Page text, examples or figures | The page's HTML outside the marked shared regions | `python3 scripts/website.py check` |
+| Search title or description | The page's `` and first description meta tag | `python3 scripts/website.py sync`, then `check` |
+| Main navigation | `NAVIGATION` in `scripts/website.py` | `sync`, then `check` |
+| Header, social links or footer | `templates/header.html` or `templates/footer.html` | `sync`, then `check` |
+| Shared link previews | `templates/sharing.html` or `public/assets/social-card.svg` | `sync`, then `check` |
+
+Create a reading page with:
+
+```sh
+python3 scripts/website.py new-page blog/my-article \
+ --title "My article title" \
+ --description "One sentence explaining the article."
+```
+
+The command creates `public/blog/my-article/index.html` with the correct relative
+paths, navigation and sharing metadata. It starts as a visible draft with
+`noindex`. Write the article, add its link to the Blog index, then remove the
+Draft label and robots meta tag when it is ready. The same command accepts
+routes under `guides/` and `showcase/`. New pages still need their own content;
+the scaffold supplies the common structure.
+
+The regions marked `website:sharing`, `website:header` and `website:footer` are
+generated from the templates. Edit the template rather than a generated copy.
+`sync` updates only those regions; page bodies, figures and scripts remain
+authored HTML. Commit the updated pages alongside the template. Titles and
+descriptions are written once per page; their sharing fields are derived from
+them. Routes come from file paths, including the sitemap, so there is no second
+page list to register.
+
+Before opening a PR:
+
+```sh
+python3 scripts/website.py check
+```
+
+This checks shared HTML without rewriting it, prepares generated assets, and
+checks page links, copied-text targets, repository documentation links, SEO
+metadata and the publishing configuration. It also runs the maintenance
+regressions. A stale shared region fails with the command needed to fix it.
+PR checks and the manual publisher run this same command. The checks work
+offline after the Python dependencies are installed.
+
+For an optional external-link audit, with GitHub CLI available:
+
+```sh
+python3 scripts/website.py links --report /tmp/rea-website-links.json
+```
+
+GitHub source links are checked through `gh`, including pinned commits and
+branch names containing slashes. Other external links use HTTP. Missing targets
+fail the command; access denials, timeouts and GitHub UI routes are reported as
+unknown and need manual verification. External fragment IDs are not checked.
+This network audit runs separately so third-party uptime does not block every PR.
+Keep its report outside public assets.
+
+Links are only part of a content review. When updating setup or support claims,
+compare them with [installation.md](../docs/installation.md), the relevant
+provider guide and the released package. A case study's pinned version and date
+describe its recorded investigation; update them only when repeating that work.
## Pages
@@ -67,11 +135,11 @@ navigation and asset references remain relative so both hosts and local previews
keep working.
Maintain a descriptive `` and a short meta description in each page's
-``. The title names the page's subject; visible headings keep the wording
-that best explains it to a reader. Open Graph and Twitter titles/descriptions
-match those fields. `og:url` matches the canonical, and sharing images use
-absolute rea.tools URLs. Search results and link previews should follow the same
-clear, concise writing standard as the page.
+``, then run `sync`. The title names the page's subject; visible headings
+keep the wording that best explains it to a reader. The script derives Open
+Graph and Twitter fields and the canonical from those inputs and the file's
+route. Search results and link previews should follow the same clear, concise
+writing standard as the page.
`scripts/prepare-website.py` scans `public/**/*.html` and generates
`public/sitemap.xml` with absolute rea.tools URLs. An `index.html` maps to its
@@ -101,8 +169,7 @@ canonicals, lost fixture exclusions, stale sitemap entries and missing previews.
Run it after preparing assets:
```sh
-python3 scripts/verify-website.py
-python3 scripts/test-website.py
+python3 scripts/website.py check
```
### After publishing an SEO change
@@ -404,8 +471,7 @@ dependencies. The same asset preparation and verification run before each manual
deployment. With the preview environment active, run them locally:
```sh
-python3 scripts/prepare-website.py
-python3 scripts/verify-website.py
+python3 scripts/website.py check
```
`.github/workflows/pages.yml` is a separate, manual-only VitePress build. It
diff --git a/website/public/blog/index.html b/website/public/blog/index.html
index 90bf245ad..e56e26449 100644
--- a/website/public/blog/index.html
+++ b/website/public/blog/index.html
@@ -8,6 +8,7 @@
name="description"
content="Thoughts on reverse engineering, drawing on practical experience with REA and AI agents."
/>
+
@@ -48,6 +49,7 @@
name="twitter:image:alt"
content="REA — Find out how software works. Reverse engineering with your coding agent."
/>
+
@@ -55,6 +57,7 @@
Skip to content
+