Re: [PATCH v9 0/6] Rework iotests/check

[Kevin Wolf]

Am 26.01.2021 um 14:19 hat Vladimir Sementsov-Ogievskiy geschrieben:
> 26.01.2021 15:53, Kevin Wolf wrote:
> > Am 25.01.2021 um 19:50 hat Vladimir Sementsov-Ogievskiy geschrieben:
> > > v9:
> > > 01: new, one more whitespace-fixing patch
> > > testenv: allow case when we don't have system-arch emulator, but have 
> > > several for another architectures
> > >           change direct os.access(..., os.X_OK) calls to new helper 
> > > function which also check that path is a file
> > > testrunner: s/fail/not run/ for 'No qualified output'
> > >              drop elapsed time arg for one of 'not run' results (now no 
> > > elapsed time for any 'not run' result)
> > 
> > More CI fun:
> > 
> > Traceback (most recent call last):
> >    File "./check", line 117, in <module>
> >      testfinder = TestFinder(test_dir=env.source_iotests)
> >    File "/builds/.../qemu/tests/qemu-iotests/findtests.py", line 53, in 
> > __init__
> >      for line in f:
> >    File "/usr/lib64/python3.6/encodings/ascii.py", line 26, in decode
> >      return codecs.ascii_decode(input, self.errors)[0]
> > UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 1591: 
> > ordinal not in range(128)
> 
> Can be solved by adding encoding='utf8' to the open().. But.. Adding it 
> everywhere is not fun.
> 
> So, what about just define PYTHONUTF8=1 for running check in check-block.sh?
> 
> https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUTF8

That looked nice, but we both missed the important line:

"New in version 3.7: See PEP 540 for more details."

So I'm back to explicitly requesting utf-8 encoding everywhere and that
seems to finally make it pass in the CI.

Kevin


diff --git a/tests/qemu-iotests/findtests.py b/tests/qemu-iotests/findtests.py
index d0c72efd6a..dd77b453b8 100644
--- a/tests/qemu-iotests/findtests.py
+++ b/tests/qemu-iotests/findtests.py
@@ -49,7 +49,7 @@ class TestFinder:
                                os.path.isfile(f + '.out')]
 
             for t in self.all_tests:
-                with open(t) as f:
+                with open(t, encoding="utf-8") as f:
                     for line in f:
                         if line.startswith('# group: '):
                             for g in line.split()[2:]:
@@ -57,7 +57,7 @@ class TestFinder:
                             break
 
     def add_group_file(self, fname: str) -> None:
-        with open(fname) as f:
+        with open(fname, encoding="utf-8") as f:
             for line in f:
                 line = line.strip()
 
diff --git a/tests/qemu-iotests/testrunner.py b/tests/qemu-iotests/testrunner.py
index 046f9ce38f..a581be6a29 100644
--- a/tests/qemu-iotests/testrunner.py
+++ b/tests/qemu-iotests/testrunner.py
@@ -41,7 +41,8 @@ def silent_unlink(path: Path) -> None:
 
 
 def file_diff(file1: str, file2: str) -> List[str]:
-    with open(file1) as f1, open(file2) as f2:
+    with open(file1, encoding="utf-8") as f1, \
+         open(file2, encoding="utf-8") as f2:
         # We want to ignore spaces at line ends. There are a lot of mess about
         # it in iotests.
         # TODO: fix all tests to not produce extra spaces, fix all .out files
@@ -81,7 +82,7 @@ class LastElapsedTime(ContextManager['LastElapsedTime']):
         self.cache: Dict[str, Dict[str, Dict[str, float]]]
 
         try:
-            with open(cache_file) as f:
+            with open(cache_file, encoding="utf-8") as f:
                 self.cache = json.load(f)
         except (OSError, ValueError):
             self.cache = {}
@@ -102,7 +103,7 @@ class LastElapsedTime(ContextManager['LastElapsedTime']):
         d.setdefault(self.env.imgproto, {})[self.env.imgfmt] = elapsed
 
     def save(self) -> None:
-        with open(self.cache_file, 'w') as f:
+        with open(self.cache_file, 'w', encoding="utf-8") as f:
             json.dump(self.cache, f)
 
     def __enter__(self) -> 'LastElapsedTime':
@@ -245,7 +246,7 @@ class TestRunner(ContextManager['TestRunner']):
         if self.env.debug:
             args.append('-d')
 
-        with f_test.open() as f:
+        with f_test.open(encoding="utf-8") as f:
             try:
                 if f.readline() == '#!/usr/bin/env python3':
                     args.insert(0, self.env.python)
@@ -256,7 +257,7 @@ class TestRunner(ContextManager['TestRunner']):
         env.update(self.test_run_env)
 
         t0 = time.time()
-        with f_bad.open('w') as f:
+        with f_bad.open('w', encoding="utf-8") as f:
             proc = subprocess.Popen(args, cwd=str(f_test.parent), env=env,
                                     stdout=f, stderr=subprocess.STDOUT)
             try: