Merge pull request #1989 from yeahboi233/fix/pr-overlap-utf8

fix(tools): decode overlap command output as UTF-8

CMakeLists.txt

@@ -37,6 +37,8 @@ endif()
 if(BUILD_TESTING)
     find_package(Python3 QUIET COMPONENTS Interpreter)
     if(Python3_Interpreter_FOUND)
+        add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
+            ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
         add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE}
             ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py)
         add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE}

tools/pr_overlap.py

@@ -14,11 +14,11 @@ MARKER = "<!-- pr-overlap -->"
 
 
 def git(*args):
-    return subprocess.run(["git", *args], capture_output=True, text=True, check=True).stdout
+    return subprocess.run(["git", *args], capture_output=True, text=True, encoding="utf-8", check=True).stdout
 
 
 def gh(*args):
-    return subprocess.run(["gh", *args], capture_output=True, text=True, check=True).stdout
+    return subprocess.run(["gh", *args], capture_output=True, text=True, encoding="utf-8", check=True).stdout
 
 
 def upstream():
@@ -46,7 +46,7 @@ def fetch(base, prs):
 
 def merge(*args):
     result = subprocess.run(["git", "merge-tree", "--write-tree", "--name-only", "--no-messages", *args],
-                            capture_output=True, text=True)
+                            capture_output=True, text=True, encoding="utf-8")
     lines = result.stdout.splitlines()
     if result.returncode not in (0, 1) or not lines:
         raise RuntimeError(f"git merge-tree {' '.join(args)} failed: {result.stderr.strip()}")

tools/tests/test_pr_overlap_encoding.py

@@ -0,0 +1,78 @@
+from contextlib import contextmanager
+import json
+from pathlib import Path
+import subprocess
+import sys
+import unittest
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import pr_overlap
+
+RUN = subprocess.run
+UNICODE = "Quoted \u201cmetadata\u201d, Y\u014dtei and \U0001f3ae"
+TREE = "a" * 40
+
+
+@contextmanager
+def command_output(stdout, stderr="", returncode=0):
+    code = ("import os, sys\n"
+            f"os.write(1, {stdout.encode('utf-8')!r})\n"
+            f"os.write(2, {stderr.encode('utf-8')!r})\n"
+            f"sys.exit({returncode})\n")
+
+    def run(command, **kwargs):
+        return RUN([sys.executable, "-c", code], **kwargs)
+
+    with patch.object(subprocess, "_text_encoding", return_value="cp1252"), \
+            patch.object(subprocess, "run", side_effect=run):
+        yield
+
+
+class EncodingTests(unittest.TestCase):
+    def test_gh_json_preserves_unicode(self):
+        response = [{"number": 1, "body": UNICODE}]
+        with command_output(json.dumps(response, ensure_ascii=False)):
+            self.assertEqual(json.loads(pr_overlap.gh("pr", "list", "--json", "number,body")), response)
+
+    def test_ascii_gh_json(self):
+        response = [{"number": 1, "body": "Ordinary metadata"}]
+        with command_output(json.dumps(response)):
+            self.assertEqual(json.loads(pr_overlap.gh("pr", "list", "--json", "number,body")), response)
+
+    def test_gh_error_preserves_unicode(self):
+        with command_output("", UNICODE, 1):
+            with self.assertRaises(subprocess.CalledProcessError) as raised:
+                pr_overlap.gh("pr", "list")
+        self.assertEqual(raised.exception.returncode, 1)
+        self.assertEqual(raised.exception.stderr, UNICODE)
+
+    def test_git_diff_preserves_unicode(self):
+        diff = "@@ -1 +1 @@\n-old\n+" + UNICODE + "\n"
+        with command_output(diff):
+            self.assertEqual(pr_overlap.git("diff", "--no-color"), diff)
+
+    def test_git_error_preserves_unicode(self):
+        with command_output("", UNICODE, 128):
+            with self.assertRaises(subprocess.CalledProcessError) as raised:
+                pr_overlap.git("show", "HEAD")
+        self.assertEqual(raised.exception.returncode, 128)
+        self.assertEqual(raised.exception.stderr, UNICODE)
+
+    def test_merge_conflict_preserves_unicode_paths(self):
+        path = UNICODE + ".txt"
+        with command_output(TREE + "\n" + path + "\n", returncode=1):
+            self.assertEqual(pr_overlap.merge("main", "topic"), (TREE, {path}))
+
+    def test_merge_success(self):
+        with command_output(TREE + "\n"):
+            self.assertEqual(pr_overlap.merge("main", "topic"), (TREE, set()))
+
+    def test_merge_error_preserves_unicode(self):
+        with command_output("", UNICODE, 128):
+            with self.assertRaisesRegex(RuntimeError, UNICODE):
+                pr_overlap.merge("main", "topic")
+
+
+if __name__ == "__main__":
+    unittest.main()