Merge pull request #3907 from GalaxyGamingFR/fix/progress-read-encoding

fix(progress): read cmake and shader tables as utf-8

CMakeLists.txt

@@ -86,6 +86,8 @@ if(BUILD_TESTING)
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_render_sliver.py)
         add_test(NAME progress_wrappers COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_wrappers.py)
+        add_test(NAME progress_read_encoding COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_read_encoding.py)
         add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
         add_test(NAME conventions_function_bodies COMMAND ${Python3_EXECUTABLE}

tools/progress.py

@@ -172,7 +172,7 @@ def scan_library(path):
              "done_names": sorted(done), "todo_names": sorted(todo), "inert_names": sorted(inert)}
     if not (done or todo):
         cmake = path / "CMakeLists.txt"
-        texts = [(CMAKE_SHARED, cmake.read_text())] if cmake.is_file() else []
+        texts = [(CMAKE_SHARED, cmake.read_text(encoding="utf-8", errors="ignore"))] if cmake.is_file() else []
         texts += [(SOURCE_SHARED, source.read_text(errors="ignore")) for source in sorted(path.glob("*.cpp"))]
         owners = [m[1] for pattern, content in texts for m in pattern.finditer(content)]
         owners = [o for o in owners if o != path.name and (path.parent / o).is_dir()]
@@ -197,12 +197,12 @@ def camel(name):
 
 def collect_shaders():
     isa = {}
-    for line in ISA.read_text().splitlines():
+    for line in ISA.read_text(encoding="utf-8", errors="ignore").splitlines():
         if line and not line.startswith("#"):
             name, encoding = line.split()
             isa[name] = encoding
     by_camel = {camel(name): name for name in isa}
-    enum = re.search(r"enum class RdnaOpcode[^{]*\{(.*?)\};", OPCODES.read_text(), re.S).group(1)
+    enum = re.search(r"enum class RdnaOpcode[^{]*\{(.*?)\};", OPCODES.read_text(encoding="utf-8", errors="ignore"), re.S).group(1)
     opcodes = [o for o in re.findall(r"^\s*([A-Z]\w*)\s*[,=]", enum, re.M) if o not in OPCODE_SENTINELS]
     supported, extra = set(), []
     for opcode in opcodes:

tools/tests/test_progress_read_encoding.py

@@ -0,0 +1,41 @@
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+TOOLS = Path(__file__).resolve().parents[1]
+
+CODE = """
+import sys
+from pathlib import Path
+sys.path.insert(0, sys.argv[1])
+import progress
+group = progress.scan_library(Path(sys.argv[2]) / "libSceExample")
+print(group.get("shared_sources"))
+"""
+
+INCLUDE = 'include("${CMAKE_CURRENT_SOURCE_DIR}/../libSceOther/libSceOther.cmake")\n'
+
+
+class ProgressReadEncodingTests(unittest.TestCase):
+    def test_utf8_cmake_with_ascii_default_codec(self):
+        with tempfile.TemporaryDirectory(prefix="progress read ") as directory:
+            root = Path(directory)
+            (root / "libSceOther").mkdir()
+            library = root / "libSceExample"
+            library.mkdir()
+            (library / "CMakeLists.txt").write_bytes(("# \u2010 \u2192 shared\n" + INCLUDE).encode("utf-8"))
+            env = {key: value for key, value in os.environ.items() if key != "PYTHONIOENCODING"}
+            env.update({"PYTHONUTF8": "0", "LC_ALL": "C", "LANG": "C"})
+            result = subprocess.run([sys.executable, "-I", "-X", "utf8=0", "-c", CODE, str(TOOLS), str(root)],
+                                    env=env, capture_output=True, timeout=20)
+        stderr = result.stderr.decode("utf-8", errors="replace")
+        self.assertNotIn("UnicodeDecodeError", stderr)
+        self.assertEqual(result.returncode, 0, stderr)
+        self.assertEqual(result.stdout.decode("ascii").strip(), "libSceOther")
+
+
+if __name__ == "__main__":
+    unittest.main()