Merge pull request #4707 from adriadam10/feat/progress-history
feat(progress): publish a progress history chart
This commit is contained in:
Adrià
GitHub
commit
8027c749e5
|
|
@ -22,7 +22,10 @@ jobs:
|
|||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- run: python3 tools/progress.py out
|
||||
- run: python3 tools/progress_history.py out
|
||||
- uses: actions/upload-pages-artifact@v5
|
||||
with:
|
||||
path: out
|
||||
|
|
|
|||
|
|
@ -62,6 +62,8 @@ if(BUILD_TESTING)
|
|||
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_dependencies.py)
|
||||
add_test(NAME pr_overlap_tools COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap.py)
|
||||
add_test(NAME progress_history COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_history.py)
|
||||
add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py)
|
||||
add_test(NAME progress_encoding COMMAND ${Python3_EXECUTABLE}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ Includes a [relinker](core/relinker) that converts executable to the target syst
|
|||
|
||||
[](https://boykopovar.github.io/AnyPS5/)
|
||||
|
||||
[](https://boykopovar.github.io/AnyPS5/history.html)
|
||||
|
||||
<sub>* System libraries: percentage of the functions known to the project so far (declared in [core/libs/prx](core/libs/prx)), not of every PS5 system function. The total grows as more functions are declared.</sub>
|
||||
|
||||
Unsupported or unexpected states strictly throw `std::runtime_error`. `what()` is printed to stderr and the process terminates.
|
||||
|
|
|
|||
|
|
@ -12,6 +12,10 @@ For these wrappers, the JSON group adds `shared_sources` with the source library
|
|||
|
||||
A library with no scanned definitions and no shared sources, such as one whose `Export.cpp` only has `APS5_DUMMY_FUN` because its functions are implemented in another library, is listed as having no exports instead of `0/0`. It is left out of the map.
|
||||
|
||||
## History
|
||||
|
||||
[`tools/progress_history.py`](../../tools/progress_history.py) measures the first-parent history of `main` with the same rules as `progress.py` and writes `history.json`, `history.svg` (shown in the README) and `history.html` (the chart with a tooltip per day plus a table, linked from the end of the progress page). It takes one point per day, at the last commit of the day, and skips commits whose tree lacks the scanned paths. The chart compacts as days accumulate: up to 31 days it labels every day, up to 150 days one label per Monday, beyond that one per month; points (and their tooltips) are drawn up to 45 days, after which the table and `history.json` keep every day. Old trees are measured with today's counting rules and instruction list, so the series is comparable but not what the page showed on that day. The workflow checks out the full history for it.
|
||||
|
||||
## Comparisons
|
||||
|
||||
Comparisons identify functions by library and name. When a head group becomes a shared-source wrapper, its previous definitions are compared under the source library's identity. Removing a duplicate implementation is reported as sharing sources, without reporting the retained functions as removed.
|
||||
|
|
@ -25,6 +29,7 @@ Implemented definitions are also classified as behaviourally empty when their bo
|
|||
```sh
|
||||
python3 tools/tests/test_progress.py
|
||||
python3 tools/progress.py /tmp/anyps5-progress
|
||||
python3 tools/tests/test_progress_history.py
|
||||
```
|
||||
|
||||
The regressions check that adding test callbacks leaves counts and comparisons unchanged, including when a callback shares a pending function's name or a wrapper uses shared sources. They also create a source library and a copied wrapper, move the wrapper to a shared builder, check output labels and totals, then verify real regressions and removals are still reported. With `BUILD_TESTING=ON` and Python available, CTest registers them as `progress_report`.
|
||||
|
|
|
|||
|
|
@ -363,6 +363,7 @@ def summary(libraries, shaders):
|
|||
f'<p>The total is the AMD RDNA 1 + RDNA 2 instruction list (<a href="{SOURCE}/tools/rdna_isa.txt">tools/rdna_isa.txt</a>). '
|
||||
f'An instruction is implemented when the <a href="{SOURCE}/core/shader/recompiler/RdnaDecoder">decoder</a> recognizes it. '
|
||||
f"Decoded opcodes not present in AMD's public list are not counted: {extra}.</p>",
|
||||
'<p><a href="history.html">Progress history</a> (one point per day).</p>',
|
||||
"</body></html>",
|
||||
]) + "\n"
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,139 @@
|
|||
import argparse
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
from datetime import date
|
||||
from html import escape
|
||||
from pathlib import Path
|
||||
|
||||
import progress
|
||||
|
||||
SERIES = (("libraries", "System libraries", "#2f81f7"), ("shaders", "GPU shader instructions", "#2ea043"))
|
||||
SHORT = {"libraries": "Libraries", "shaders": "Shaders"}
|
||||
WIDTH, HEIGHT, LEFT, RIGHT, TOP, BOTTOM = 900, 380, 50, 130, 20, 60
|
||||
MAX_DOTS = 45
|
||||
BACKGROUND, GRID, TEXT = "#0d1117", "#30363d", "#c9d1d9"
|
||||
|
||||
|
||||
def git(*args):
|
||||
return subprocess.run(["git", "-C", str(progress.ROOT), *args], check=True, capture_output=True).stdout
|
||||
|
||||
|
||||
def daily_commits(ref):
|
||||
seen = {}
|
||||
for line in git("log", "--first-parent", "--format=%H %cs", ref).decode().splitlines():
|
||||
sha, day = line.split()
|
||||
seen.setdefault(day, sha)
|
||||
return sorted(seen.items())
|
||||
|
||||
|
||||
def measure(sha):
|
||||
paths = [progress.PRX.relative_to(progress.ROOT).as_posix(), progress.OPCODES.relative_to(progress.ROOT).as_posix()]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
try:
|
||||
tarfile.open(fileobj=io.BytesIO(git("archive", sha, "--", *paths))).extractall(root, filter="data")
|
||||
except subprocess.CalledProcessError:
|
||||
return None
|
||||
saved = progress.PRX, progress.OPCODES
|
||||
progress.PRX, progress.OPCODES = root / paths[0], root / paths[1]
|
||||
try:
|
||||
libraries, shaders = progress.collect_libraries(), progress.collect_shaders()
|
||||
except (OSError, AttributeError):
|
||||
return None
|
||||
finally:
|
||||
progress.PRX, progress.OPCODES = saved
|
||||
return {name: {key: data[key] for key in ("done", "total", "percent")}
|
||||
for name, data in (("libraries", libraries), ("shaders", shaders))}
|
||||
|
||||
|
||||
def collect(ref):
|
||||
points = []
|
||||
for day, sha in daily_commits(ref):
|
||||
counts = measure(sha)
|
||||
if counts:
|
||||
points.append({"date": day, "commit": sha, **counts})
|
||||
return points
|
||||
|
||||
|
||||
def labelled(day, span):
|
||||
if span <= 31:
|
||||
return day.strftime("%m-%d")
|
||||
if span <= 150:
|
||||
return day.strftime("%m-%d") if day.weekday() == 0 else ""
|
||||
return day.strftime("%Y-%m") if day.day <= 7 and day.weekday() == 0 else ""
|
||||
|
||||
|
||||
def chart(points):
|
||||
days = [date.fromisoformat(p["date"]).toordinal() for p in points]
|
||||
first, span = days[0], max(days[-1] - days[0], 1)
|
||||
top = max(10, int(-(-max(p[name]["percent"] for p in points for name, _, _ in SERIES) // 10) * 10))
|
||||
px = lambda d: LEFT + (d - first) / span * (WIDTH - LEFT - RIGHT)
|
||||
py = lambda v: TOP + (1 - v / top) * (HEIGHT - TOP - BOTTOM)
|
||||
base = py(0)
|
||||
out = [f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 {WIDTH} {HEIGHT}" role="img" '
|
||||
f'aria-label="Progress over time" font-family="sans-serif" font-size="12" fill="{TEXT}">',
|
||||
f'<rect width="{WIDTH}" height="{HEIGHT}" rx="6" fill="{BACKGROUND}"/>']
|
||||
for tick in range(0, top + 1, 10):
|
||||
out.append(f'<line x1="{LEFT}" x2="{WIDTH - RIGHT}" y1="{py(tick):.1f}" y2="{py(tick):.1f}" stroke="{GRID}"/>'
|
||||
f'<text x="{LEFT - 6}" y="{py(tick) + 4:.1f}" text-anchor="end">{tick}%</text>')
|
||||
for d, p in zip(days, points):
|
||||
x = px(d)
|
||||
label = labelled(date.fromordinal(d), span)
|
||||
if label:
|
||||
out.append(f'<line x1="{x:.1f}" x2="{x:.1f}" y1="{TOP}" y2="{base:.1f}" stroke="{GRID}" stroke-width="0.5"/>'
|
||||
f'<text transform="translate({x + 4:.1f} {base + 8:.1f}) rotate(60)" font-size="10">{label}</text>')
|
||||
for name, label, color in SERIES:
|
||||
coords = [(px(d), py(p[name]["percent"])) for d, p in zip(days, points)]
|
||||
line = " ".join(f"{x:.1f},{y:.1f}" for x, y in coords)
|
||||
out.append(f'<polygon points="{coords[0][0]:.1f},{base:.1f} {line} {coords[-1][0]:.1f},{base:.1f}" fill="{color}" opacity="0.15"/>'
|
||||
f'<polyline points="{line}" fill="none" stroke="{color}" stroke-width="2"/>')
|
||||
for (x, y), p in zip(coords, points) if len(points) <= MAX_DOTS else ():
|
||||
out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="3" fill="{color}"><title>{p["date"]}: {label} '
|
||||
f'{p[name]["percent"]}% ({p[name]["done"]}/{p[name]["total"]})</title></circle>')
|
||||
last = points[-1][name]
|
||||
out.append(f'<text x="{WIDTH - RIGHT + 8}" y="{coords[-1][1] + 4:.1f}" fill="{color}" font-weight="bold">'
|
||||
f'{SHORT[name]} {last["percent"]}%</text>')
|
||||
out.append("</svg>")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def page(points):
|
||||
rows = "\n".join(f'<tr><td>{p["date"]}</td>' + "".join(
|
||||
f'<td>{p[name]["done"]}/{p[name]["total"]}</td><td>{p[name]["percent"]}%</td>' for name, _, _ in SERIES) + "</tr>"
|
||||
for p in reversed(points))
|
||||
return "\n".join([
|
||||
'<!DOCTYPE html>',
|
||||
'<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">',
|
||||
'<title>AnyPS5 progress history</title>',
|
||||
'<style>body{font-family:sans-serif;max-width:1000px;margin:auto;padding:16px}svg{max-width:100%;height:auto}'
|
||||
'table{border-collapse:collapse}th,td{border:1px solid #ccc;padding:2px 8px}td+td,th+th{text-align:right}</style>',
|
||||
'</head><body>',
|
||||
'<h1>Progress history</h1>',
|
||||
chart(points),
|
||||
f'<p>One point per day with commits on <code>main</code>, taken at the last commit of the day. Every point is measured '
|
||||
f'with the current <a href="{progress.SOURCE}/tools/progress.py">tools/progress.py</a> rules and instruction list, '
|
||||
'so the series is comparable across days. Hover a point for the exact numbers. Raw numbers: <a href="history.json">history.json</a>. '
|
||||
'<a href="./">Back to the current progress</a>.</p>',
|
||||
'<table><tr><th>Date</th>' + "".join(f'<th>{escape(label)}</th><th>%</th>' for _, label, _ in SERIES) + '</tr>',
|
||||
rows,
|
||||
'</table>',
|
||||
'</body></html>',
|
||||
]) + "\n"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("--ref", default="HEAD", help="branch or commit whose first-parent history is measured")
|
||||
args = parser.parse_args()
|
||||
points = collect(args.ref)
|
||||
if not points:
|
||||
parser.error("no commit with a measurable tree")
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
(args.output / "history.json").write_text(json.dumps(points, indent=2))
|
||||
(args.output / "history.svg").write_text(chart(points) + "\n")
|
||||
(args.output / "history.html").write_text(page(points))
|
||||
print(f"{len(points)} days, {points[0]['date']} .. {points[-1]['date']}")
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import progress
|
||||
import progress_history
|
||||
|
||||
OPCODES = Path("core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp")
|
||||
|
||||
|
||||
def commit(root, day, message, body):
|
||||
library = root / "core/libs/prx/libSceExample"
|
||||
library.mkdir(parents=True, exist_ok=True)
|
||||
(library / "Export.cpp").write_text(body)
|
||||
(root / OPCODES).parent.mkdir(parents=True, exist_ok=True)
|
||||
(root / OPCODES).write_text("enum class RdnaOpcode : int {\n SEndpgm,\n};\n")
|
||||
env = {**os.environ, "GIT_AUTHOR_DATE": f"{day}T12:00:00Z", "GIT_COMMITTER_DATE": f"{day}T12:00:00Z"}
|
||||
subprocess.run(["git", "-C", str(root), "add", "-A"], check=True)
|
||||
subprocess.run(["git", "-C", str(root), "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", message],
|
||||
check=True, env=env)
|
||||
|
||||
|
||||
class ProgressHistoryTests(unittest.TestCase):
|
||||
def test_one_point_per_day_at_the_last_commit(self):
|
||||
stub = "static void Stub() { NotImplemented_nid_no_patch(__func__); }\n"
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
subprocess.run(["git", "-C", directory, "init", "-q"], check=True)
|
||||
commit(root, "2026-01-01", "first", "int APS5_VABI A() { Stub(); return 0; }\n" + stub)
|
||||
commit(root, "2026-01-02", "second", "int APS5_VABI A() { return 0; }\nint APS5_VABI B() { return 0; }\n")
|
||||
commit(root, "2026-01-02", "third", "int APS5_VABI A() { return 0; }\nint APS5_VABI B() { Stub(); return 0; }\n" + stub)
|
||||
with patch.multiple(progress, ROOT=root, PRX=root / "core/libs/prx", OPCODES=root / OPCODES):
|
||||
points = progress_history.collect("HEAD")
|
||||
html = progress_history.page(points)
|
||||
self.assertEqual([p["date"] for p in points], ["2026-01-01", "2026-01-02"])
|
||||
self.assertEqual([(p["libraries"]["done"], p["libraries"]["total"]) for p in points], [(0, 1), (1, 2)])
|
||||
self.assertIn("<svg", html)
|
||||
self.assertIn("2026-01-02", html)
|
||||
|
||||
def test_long_series_keeps_the_chart_readable(self):
|
||||
def series(count):
|
||||
start = date(2026, 1, 1).toordinal()
|
||||
return [{"date": date.fromordinal(start + i).isoformat(),
|
||||
**{name: {"done": i, "total": count, "percent": 100 * i / count} for name in ("libraries", "shaders")}}
|
||||
for i in range(count)]
|
||||
short, medium, long = (progress_history.chart(series(count)) for count in (20, 100, 400))
|
||||
self.assertEqual((short.count("rotate(60)"), short.count("<circle")), (20, 40))
|
||||
self.assertEqual((medium.count("rotate(60)"), medium.count("<circle")), (14, 0))
|
||||
self.assertEqual((long.count("rotate(60)"), long.count("<circle")), (14, 0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue