diff --git a/.github/workflows/progress.yml b/.github/workflows/progress.yml index 93e9f0600..a45defac9 100644 --- a/.github/workflows/progress.yml +++ b/.github/workflows/progress.yml @@ -22,7 +22,10 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - run: python3 tools/progress.py out + - run: python3 tools/progress_history.py out - uses: actions/upload-pages-artifact@v5 with: path: out diff --git a/CMakeLists.txt b/CMakeLists.txt index c0b526313..a551fe53a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,8 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_dependencies.py) add_test(NAME pr_overlap_tools COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap.py) + add_test(NAME progress_history COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_history.py) add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py) add_test(NAME progress_encoding COMMAND ${Python3_EXECUTABLE} diff --git a/README.md b/README.md index 1779d66e2..76dd74c31 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Includes a [relinker](core/relinker) that converts executable to the target syst [![progress map](https://boykopovar.github.io/AnyPS5/progress.svg)](https://boykopovar.github.io/AnyPS5/) +[![progress history](https://boykopovar.github.io/AnyPS5/history.svg)](https://boykopovar.github.io/AnyPS5/history.html) + * System libraries: percentage of the functions known to the project so far (declared in [core/libs/prx](core/libs/prx)), not of every PS5 system function. The total grows as more functions are declared. Unsupported or unexpected states strictly throw `std::runtime_error`. `what()` is printed to stderr and the process terminates. diff --git a/docs/dev/PROGRESS.md b/docs/dev/PROGRESS.md index ca20cad09..8b532eeb6 100644 --- a/docs/dev/PROGRESS.md +++ b/docs/dev/PROGRESS.md @@ -12,6 +12,10 @@ For these wrappers, the JSON group adds `shared_sources` with the source library A library with no scanned definitions and no shared sources, such as one whose `Export.cpp` only has `APS5_DUMMY_FUN` because its functions are implemented in another library, is listed as having no exports instead of `0/0`. It is left out of the map. +## History + +[`tools/progress_history.py`](../../tools/progress_history.py) measures the first-parent history of `main` with the same rules as `progress.py` and writes `history.json`, `history.svg` (shown in the README) and `history.html` (the chart with a tooltip per day plus a table, linked from the end of the progress page). It takes one point per day, at the last commit of the day, and skips commits whose tree lacks the scanned paths. The chart compacts as days accumulate: up to 31 days it labels every day, up to 150 days one label per Monday, beyond that one per month; points (and their tooltips) are drawn up to 45 days, after which the table and `history.json` keep every day. Old trees are measured with today's counting rules and instruction list, so the series is comparable but not what the page showed on that day. The workflow checks out the full history for it. + ## Comparisons Comparisons identify functions by library and name. When a head group becomes a shared-source wrapper, its previous definitions are compared under the source library's identity. Removing a duplicate implementation is reported as sharing sources, without reporting the retained functions as removed. @@ -25,6 +29,7 @@ Implemented definitions are also classified as behaviourally empty when their bo ```sh python3 tools/tests/test_progress.py python3 tools/progress.py /tmp/anyps5-progress +python3 tools/tests/test_progress_history.py ``` The regressions check that adding test callbacks leaves counts and comparisons unchanged, including when a callback shares a pending function's name or a wrapper uses shared sources. They also create a source library and a copied wrapper, move the wrapper to a shared builder, check output labels and totals, then verify real regressions and removals are still reported. With `BUILD_TESTING=ON` and Python available, CTest registers them as `progress_report`. diff --git a/tools/progress.py b/tools/progress.py index d2222e720..5139d1ec4 100644 --- a/tools/progress.py +++ b/tools/progress.py @@ -363,6 +363,7 @@ def summary(libraries, shaders): f'

The total is the AMD RDNA 1 + RDNA 2 instruction list (tools/rdna_isa.txt). ' f'An instruction is implemented when the decoder recognizes it. ' f"Decoded opcodes not present in AMD's public list are not counted: {extra}.

", + '

Progress history (one point per day).

', "", ]) + "\n" diff --git a/tools/progress_history.py b/tools/progress_history.py new file mode 100644 index 000000000..26e1ae7b4 --- /dev/null +++ b/tools/progress_history.py @@ -0,0 +1,139 @@ +import argparse +import io +import json +import subprocess +import tarfile +import tempfile +from datetime import date +from html import escape +from pathlib import Path + +import progress + +SERIES = (("libraries", "System libraries", "#2f81f7"), ("shaders", "GPU shader instructions", "#2ea043")) +SHORT = {"libraries": "Libraries", "shaders": "Shaders"} +WIDTH, HEIGHT, LEFT, RIGHT, TOP, BOTTOM = 900, 380, 50, 130, 20, 60 +MAX_DOTS = 45 +BACKGROUND, GRID, TEXT = "#0d1117", "#30363d", "#c9d1d9" + + +def git(*args): + return subprocess.run(["git", "-C", str(progress.ROOT), *args], check=True, capture_output=True).stdout + + +def daily_commits(ref): + seen = {} + for line in git("log", "--first-parent", "--format=%H %cs", ref).decode().splitlines(): + sha, day = line.split() + seen.setdefault(day, sha) + return sorted(seen.items()) + + +def measure(sha): + paths = [progress.PRX.relative_to(progress.ROOT).as_posix(), progress.OPCODES.relative_to(progress.ROOT).as_posix()] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + try: + tarfile.open(fileobj=io.BytesIO(git("archive", sha, "--", *paths))).extractall(root, filter="data") + except subprocess.CalledProcessError: + return None + saved = progress.PRX, progress.OPCODES + progress.PRX, progress.OPCODES = root / paths[0], root / paths[1] + try: + libraries, shaders = progress.collect_libraries(), progress.collect_shaders() + except (OSError, AttributeError): + return None + finally: + progress.PRX, progress.OPCODES = saved + return {name: {key: data[key] for key in ("done", "total", "percent")} + for name, data in (("libraries", libraries), ("shaders", shaders))} + + +def collect(ref): + points = [] + for day, sha in daily_commits(ref): + counts = measure(sha) + if counts: + points.append({"date": day, "commit": sha, **counts}) + return points + + +def labelled(day, span): + if span <= 31: + return day.strftime("%m-%d") + if span <= 150: + return day.strftime("%m-%d") if day.weekday() == 0 else "" + return day.strftime("%Y-%m") if day.day <= 7 and day.weekday() == 0 else "" + + +def chart(points): + days = [date.fromisoformat(p["date"]).toordinal() for p in points] + first, span = days[0], max(days[-1] - days[0], 1) + top = max(10, int(-(-max(p[name]["percent"] for p in points for name, _, _ in SERIES) // 10) * 10)) + px = lambda d: LEFT + (d - first) / span * (WIDTH - LEFT - RIGHT) + py = lambda v: TOP + (1 - v / top) * (HEIGHT - TOP - BOTTOM) + base = py(0) + out = [f'', + f''] + for tick in range(0, top + 1, 10): + out.append(f'' + f'{tick}%') + for d, p in zip(days, points): + x = px(d) + label = labelled(date.fromordinal(d), span) + if label: + out.append(f'' + f'{label}') + for name, label, color in SERIES: + coords = [(px(d), py(p[name]["percent"])) for d, p in zip(days, points)] + line = " ".join(f"{x:.1f},{y:.1f}" for x, y in coords) + out.append(f'' + f'') + for (x, y), p in zip(coords, points) if len(points) <= MAX_DOTS else (): + out.append(f'{p["date"]}: {label} ' + f'{p[name]["percent"]}% ({p[name]["done"]}/{p[name]["total"]})') + last = points[-1][name] + out.append(f'' + f'{SHORT[name]} {last["percent"]}%') + out.append("") + return "\n".join(out) + + +def page(points): + rows = "\n".join(f'{p["date"]}' + "".join( + f'{p[name]["done"]}/{p[name]["total"]}{p[name]["percent"]}%' for name, _, _ in SERIES) + "" + for p in reversed(points)) + return "\n".join([ + '', + '', + 'AnyPS5 progress history', + '', + '', + '

Progress history

', + chart(points), + f'

One point per day with commits on main, taken at the last commit of the day. Every point is measured ' + f'with the current tools/progress.py rules and instruction list, ' + 'so the series is comparable across days. Hover a point for the exact numbers. Raw numbers: history.json. ' + 'Back to the current progress.

', + '' + "".join(f'' for _, label, _ in SERIES) + '', + rows, + '
Date{escape(label)}%
', + '', + ]) + "\n" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("output", type=Path) + parser.add_argument("--ref", default="HEAD", help="branch or commit whose first-parent history is measured") + args = parser.parse_args() + points = collect(args.ref) + if not points: + parser.error("no commit with a measurable tree") + args.output.mkdir(parents=True, exist_ok=True) + (args.output / "history.json").write_text(json.dumps(points, indent=2)) + (args.output / "history.svg").write_text(chart(points) + "\n") + (args.output / "history.html").write_text(page(points)) + print(f"{len(points)} days, {points[0]['date']} .. {points[-1]['date']}") diff --git a/tools/tests/test_progress_history.py b/tools/tests/test_progress_history.py new file mode 100644 index 000000000..c8da6a22c --- /dev/null +++ b/tools/tests/test_progress_history.py @@ -0,0 +1,59 @@ +import os +import subprocess +import sys +import tempfile +import unittest +from datetime import date +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import progress +import progress_history + +OPCODES = Path("core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp") + + +def commit(root, day, message, body): + library = root / "core/libs/prx/libSceExample" + library.mkdir(parents=True, exist_ok=True) + (library / "Export.cpp").write_text(body) + (root / OPCODES).parent.mkdir(parents=True, exist_ok=True) + (root / OPCODES).write_text("enum class RdnaOpcode : int {\n SEndpgm,\n};\n") + env = {**os.environ, "GIT_AUTHOR_DATE": f"{day}T12:00:00Z", "GIT_COMMITTER_DATE": f"{day}T12:00:00Z"} + subprocess.run(["git", "-C", str(root), "add", "-A"], check=True) + subprocess.run(["git", "-C", str(root), "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", message], + check=True, env=env) + + +class ProgressHistoryTests(unittest.TestCase): + def test_one_point_per_day_at_the_last_commit(self): + stub = "static void Stub() { NotImplemented_nid_no_patch(__func__); }\n" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + subprocess.run(["git", "-C", directory, "init", "-q"], check=True) + commit(root, "2026-01-01", "first", "int APS5_VABI A() { Stub(); return 0; }\n" + stub) + commit(root, "2026-01-02", "second", "int APS5_VABI A() { return 0; }\nint APS5_VABI B() { return 0; }\n") + commit(root, "2026-01-02", "third", "int APS5_VABI A() { return 0; }\nint APS5_VABI B() { Stub(); return 0; }\n" + stub) + with patch.multiple(progress, ROOT=root, PRX=root / "core/libs/prx", OPCODES=root / OPCODES): + points = progress_history.collect("HEAD") + html = progress_history.page(points) + self.assertEqual([p["date"] for p in points], ["2026-01-01", "2026-01-02"]) + self.assertEqual([(p["libraries"]["done"], p["libraries"]["total"]) for p in points], [(0, 1), (1, 2)]) + self.assertIn("