shifathosn
/
AnyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3675 from MrMha/fix/oracle-cache-current 

fix(hw-oracle): invalidate cached builds when inputs change
This commit is contained in:
Dean 2026-10-12 18:21:47 +13:00 committed by GitHub
parent f9984863e6 cf1195ad20
commit 6e653bef51
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 133 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
tools/hw-oracle/hw_oracle.py
View File

@ -1,5 +1,7 @@
import argparse
import hashlib
import os
import shlex
import shutil
import struct
import subprocess
@ -33,15 +35,23 @@ def tool(name):
def oracle():
binary = CACHE / "oracle"
source = HERE / "oracle.c"
if binary.exists() and binary.stat().st_mtime >= source.stat().st_mtime:
compiler = shlex.split(os.environ.get("CC", "cc"))
if not compiler:
raise ValueError("CC must name a compiler command")
root = rocm_root()
if root is not None:
root = root.resolve()
configuration = repr((compiler, str(root))).encode("utf-8")
key = hashlib.sha256(source.read_bytes() + configuration).hexdigest()
directory = CACHE / key
binary = directory / "oracle"
if binary.is_file():
return binary
CACHE.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=CACHE) as tmp:
directory.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=directory) as tmp:
output = Path(tmp) / "oracle"
command = [os.environ.get("CC", "cc"), "-O1", str(source), "-o", str(output)]
root = rocm_root()
command = [*compiler, "-O1", str(source), "-o", str(output)]
if root:
lib = root / "lib"
command += [f"-I{root / 'include'}", f"-L{lib}", f"-Wl,-rpath,{lib}"]
@ -50,6 +60,8 @@ def oracle():
return binary
@cache
def target():
return os.environ.get("HW_ORACLE_TARGET") or subprocess.run([oracle(), "--target"], capture_output=True, text=True, check=True).stdout.strip()

123
tools/hw-oracle/test_hw_oracle.py
View File

@ -1,7 +1,10 @@
import contextlib
import io
import os
import shlex
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
@ -38,24 +41,26 @@ class OracleCacheTests(unittest.TestCase):
with self.assertRaises(subprocess.CalledProcessError):
hw_oracle.oracle()
self.assertEqual(binary.read_bytes() if binary.exists() else None, previous)
self.assertEqual(list(cache.iterdir()), [binary] if previous is not None else [])
self.assertEqual([file for file in cache.rglob("*") if file.is_file()], [binary] if previous is not None else [])
compiler.side_effect = succeed
self.assertEqual(hw_oracle.oracle(), binary)
self.assertEqual(binary.read_bytes(), b"complete executable")
self.assertEqual(hw_oracle.oracle(), binary)
built = hw_oracle.oracle()
self.assertEqual(built.read_bytes(), b"complete executable")
self.assertEqual(hw_oracle.oracle(), built)
self.assertEqual(compiler.call_count, 2)
self.assertEqual(list(cache.iterdir()), [binary])
self.assertEqual(set(file for file in cache.rglob("*") if file.is_file()), {built, binary} if previous is not None else {built})
if previous is not None:
self.assertEqual(binary.read_bytes(), previous)
def test_overlapping_builds_do_not_expose_partial_output(self):
with tempfile.TemporaryDirectory() as tmp:
cache = Path(tmp) / "cache"
binary = cache / "oracle"
outputs = []
def compile(command, *, check):
self.assertTrue(check)
output = Path(command[command.index("-o") + 1])
outputs.append(output)
binary = output.parent.parent / "oracle"
output.write_bytes(b"partial executable")
if len(outputs) == 1:
self.assertFalse(binary.exists())
@ -64,11 +69,11 @@ class OracleCacheTests(unittest.TestCase):
with patch.object(hw_oracle, "CACHE", cache), patch.object(hw_oracle, "rocm_root", return_value=None), \
patch.object(hw_oracle.subprocess, "run", side_effect=compile) as compiler:
self.assertEqual(hw_oracle.oracle(), binary)
binary = hw_oracle.oracle()
self.assertEqual(binary.read_bytes(), b"complete executable")
self.assertEqual(compiler.call_count, 2)
self.assertNotEqual(outputs[0], outputs[1])
self.assertEqual(list(cache.iterdir()), [binary])
self.assertEqual([file for file in cache.rglob("*") if file.is_file()], [binary])
class FloatModeTests(unittest.TestCase):
@ -122,6 +127,108 @@ class FloatModeTests(unittest.TestCase):
self.assertEqual(assemble.call_args.kwargs, MODES | {"lds": 4096})
class OracleConfigurationTests(unittest.TestCase):
@unittest.skipUnless(sys.platform.startswith("linux"), "Linux compiler and shared-library fixture")
def test_cache_and_compiler_command_with_host_toolchain(self):
compiler = shutil.which("cc")
if compiler is None:
self.skipTest("C compiler not found")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
runtime = root / "runtime"
(runtime / "lib").mkdir(parents=True)
stub = root / "runtime.c"
stub.write_text("void runtime_stub(void) {}\n", encoding="utf-8")
subprocess.run([compiler, "-shared", "-fPIC", str(stub), "-o", str(runtime / "lib/libhsa-runtime64.so")], check=True)
source = root / "source"
source.mkdir()
source_file = source / "oracle.c"
source_file.write_text('#include <stdio.h>\nint main(void) { printf("%d", VALUE); }\n', encoding="utf-8")
os.utime(source_file, (0, 0))
wrapper = root / "compiler wrapper"
wrapper.write_text("#!/bin/sh\nexec " + shlex.quote(compiler) + ' "$@"\n', encoding="utf-8")
wrapper.chmod(0o755)
with patch.object(hw_oracle, "HERE", source), patch.object(hw_oracle, "CACHE", root / "cache"), \
patch.object(hw_oracle, "rocm_root", return_value=runtime):
with patch.dict(os.environ, {"CC": shlex.quote(str(wrapper)) + " -DVALUE=7"}):
first = hw_oracle.oracle()
self.assertEqual(subprocess.check_output([first]), b"7")
self.assertEqual(hw_oracle.oracle(), first)
with patch.dict(os.environ, {"CC": shlex.quote(str(wrapper)) + " -DVALUE=8"}):
second = hw_oracle.oracle()
self.assertNotEqual(second, first)
self.assertEqual(subprocess.check_output([second]), b"8")
source_file.write_text('#include <stdio.h>\nint main(void) { printf("%d", VALUE + 1); }\n', encoding="utf-8")
os.utime(source_file, (0, 0))
third = hw_oracle.oracle()
self.assertNotEqual(third, second)
self.assertEqual(subprocess.check_output([third]), b"9")
def test_compiler_command_supports_wrapper_and_flags(self):
with tempfile.TemporaryDirectory() as tmp:
def compile(command, *, check):
self.assertEqual(command[:3], ["ccache", "cc", "-fno-omit-frame-pointer"])
Path(command[command.index("-o") + 1]).write_bytes(b"executable")
with patch.object(hw_oracle, "CACHE", Path(tmp)), patch.object(hw_oracle, "rocm_root", return_value=None), \
patch.dict(os.environ, {"CC": "ccache cc -fno-omit-frame-pointer"}), \
patch.object(hw_oracle.subprocess, "run", side_effect=compile):
self.assertEqual(hw_oracle.oracle().read_bytes(), b"executable")
def test_source_change_with_preserved_timestamp_invalidates_cache(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
source = root / "source"
source.mkdir()
source_file = source / "oracle.c"
source_file.write_text("first", encoding="utf-8")
os.utime(source_file, (0, 0))
def compile(command, *, check):
Path(command[command.index("-o") + 1]).write_bytes(source_file.read_bytes())
with patch.object(hw_oracle, "HERE", source), patch.object(hw_oracle, "CACHE", root / "cache"), \
patch.object(hw_oracle, "rocm_root", return_value=None), \
patch.object(hw_oracle.subprocess, "run", side_effect=compile) as compiler:
self.assertEqual(hw_oracle.oracle().read_bytes(), b"first")
source_file.write_text("second", encoding="utf-8")
os.utime(source_file, (0, 0))
self.assertEqual(hw_oracle.oracle().read_bytes(), b"second")
self.assertEqual(compiler.call_count, 2)
def test_compiler_change_invalidates_cache(self):
with tempfile.TemporaryDirectory() as tmp:
def compile(command, *, check):
Path(command[command.index("-o") + 1]).write_bytes(command[0].encode())
with patch.object(hw_oracle, "CACHE", Path(tmp)), patch.object(hw_oracle, "rocm_root", return_value=None), \
patch.object(hw_oracle.subprocess, "run", side_effect=compile) as compiler:
with patch.dict(os.environ, {"CC": "cc"}):
self.assertEqual(hw_oracle.oracle().read_bytes(), b"cc")
with patch.dict(os.environ, {"CC": "clang"}):
self.assertEqual(hw_oracle.oracle().read_bytes(), b"clang")
self.assertEqual(compiler.call_count, 2)
def test_runtime_path_change_invalidates_cache(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
def compile(command, *, check):
library_path = next(option for option in command if option.startswith("-L"))
Path(command[command.index("-o") + 1]).write_bytes(library_path.encode())
with patch.object(hw_oracle, "CACHE", root / "cache"), \
patch.object(hw_oracle.subprocess, "run", side_effect=compile) as compiler:
with patch.object(hw_oracle, "rocm_root", return_value=root / "one"):
first = hw_oracle.oracle()
self.assertEqual(first.read_bytes(), ("-L" + str((root / "one").resolve() / "lib")).encode())
with patch.object(hw_oracle, "rocm_root", return_value=root / "two"):
second = hw_oracle.oracle()
self.assertNotEqual(second, first)
self.assertEqual(second.read_bytes(), ("-L" + str((root / "two").resolve() / "lib")).encode())
self.assertEqual(compiler.call_count, 2)
class GroupSegmentTests(unittest.TestCase):
def test_wave64_dispatches_at_most_512_rows(self):
rows = [(index, 0, 0, 0) for index in range(1025)]