From 1c32fd93c8a810ca4af8643dbd284621e672ab72 Mon Sep 17 00:00:00 2001
From: GalaxyGamingFR <153150716+GalaxyGamingFR@users.noreply.github.com>
Date: Sun, 11 Oct 2026 11:59:49 -0700
Subject: [PATCH] fix(progress): read cmake and shader tables as utf-8

scan_library and collect_shaders read CMakeLists.txt, the ISA table and the opcode header with the locale codec. On Windows that is cp1252, so a UTF-8 file containing U+2010 or U+2192 raised UnicodeDecodeError. These reads now pass encoding="utf-8" and errors="ignore", and a subprocess test with an ASCII default codec covers it.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V9UU71SP1UKGR51Sowbgmq
---
 CMakeLists.txt                             |  2 ++
 tools/progress.py                          |  6 ++--
 tools/tests/test_progress_read_encoding.py | 41 ++++++++++++++++++++++
 3 files changed, 46 insertions(+), 3 deletions(-)
 create mode 100644 tools/tests/test_progress_read_encoding.py

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c0ac81795..536bf71fc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -86,6 +86,8 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_render_sliver.py)
         add_test(NAME progress_wrappers COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_wrappers.py)
+        add_test(NAME progress_read_encoding COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_read_encoding.py)
         add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
         add_test(NAME conventions_function_bodies COMMAND ${Python3_EXECUTABLE}
diff --git a/tools/progress.py b/tools/progress.py
index 5bde11b6a..c27199b01 100644
--- a/tools/progress.py
+++ b/tools/progress.py
@@ -172,7 +172,7 @@ def scan_library(path):
              "done_names": sorted(done), "todo_names": sorted(todo), "inert_names": sorted(inert)}
     if not (done or todo):
         cmake = path / "CMakeLists.txt"
-        texts = [(CMAKE_SHARED, cmake.read_text())] if cmake.is_file() else []
+        texts = [(CMAKE_SHARED, cmake.read_text(encoding="utf-8", errors="ignore"))] if cmake.is_file() else []
         texts += [(SOURCE_SHARED, source.read_text(errors="ignore")) for source in sorted(path.glob("*.cpp"))]
         owners = [m[1] for pattern, content in texts for m in pattern.finditer(content)]
         owners = [o for o in owners if o != path.name and (path.parent / o).is_dir()]
@@ -197,12 +197,12 @@ def camel(name):
 
 def collect_shaders():
     isa = {}
-    for line in ISA.read_text().splitlines():
+    for line in ISA.read_text(encoding="utf-8", errors="ignore").splitlines():
         if line and not line.startswith("#"):
             name, encoding = line.split()
             isa[name] = encoding
     by_camel = {camel(name): name for name in isa}
-    enum = re.search(r"enum class RdnaOpcode[^{]*\{(.*?)\};", OPCODES.read_text(), re.S).group(1)
+    enum = re.search(r"enum class RdnaOpcode[^{]*\{(.*?)\};", OPCODES.read_text(encoding="utf-8", errors="ignore"), re.S).group(1)
     opcodes = [o for o in re.findall(r"^\s*([A-Z]\w*)\s*[,=]", enum, re.M) if o not in OPCODE_SENTINELS]
     supported, extra = set(), []
     for opcode in opcodes:
diff --git a/tools/tests/test_progress_read_encoding.py b/tools/tests/test_progress_read_encoding.py
new file mode 100644
index 000000000..1c01153b5
--- /dev/null
+++ b/tools/tests/test_progress_read_encoding.py
@@ -0,0 +1,41 @@
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+TOOLS = Path(__file__).resolve().parents[1]
+
+CODE = """
+import sys
+from pathlib import Path
+sys.path.insert(0, sys.argv[1])
+import progress
+group = progress.scan_library(Path(sys.argv[2]) / "libSceExample")
+print(group.get("shared_sources"))
+"""
+
+INCLUDE = 'include("${CMAKE_CURRENT_SOURCE_DIR}/../libSceOther/libSceOther.cmake")\n'
+
+
+class ProgressReadEncodingTests(unittest.TestCase):
+    def test_utf8_cmake_with_ascii_default_codec(self):
+        with tempfile.TemporaryDirectory(prefix="progress read ") as directory:
+            root = Path(directory)
+            (root / "libSceOther").mkdir()
+            library = root / "libSceExample"
+            library.mkdir()
+            (library / "CMakeLists.txt").write_bytes(("# \u2010 \u2192 shared\n" + INCLUDE).encode("utf-8"))
+            env = {key: value for key, value in os.environ.items() if key != "PYTHONIOENCODING"}
+            env.update({"PYTHONUTF8": "0", "LC_ALL": "C", "LANG": "C"})
+            result = subprocess.run([sys.executable, "-I", "-X", "utf8=0", "-c", CODE, str(TOOLS), str(root)],
+                                    env=env, capture_output=True, timeout=20)
+        stderr = result.stderr.decode("utf-8", errors="replace")
+        self.assertNotIn("UnicodeDecodeError", stderr)
+        self.assertEqual(result.returncode, 0, stderr)
+        self.assertEqual(result.stdout.decode("ascii").strip(), "libSceOther")
+
+
+if __name__ == "__main__":
+    unittest.main()