fix: Always explicitly set encoding for text files (fixes #8263)

ci/ciimage/build.py

@@ -18,7 +18,7 @@
 class ImageDef:
     def __init__(self, image_dir: Path) -> None:
         path = image_dir / image_def_file
-        data = json.loads(path.read_text())
+        data = json.loads(path.read_text(encoding='utf-8'))
 
         assert isinstance(data, dict)
         assert all([x in data for x in ['base_image', 'env']])
@@ -74,7 +74,7 @@ def gen_bashrc(self) -> None:
         # Also add /ci to PATH
         out_data += 'export PATH="/ci:$PATH"\n'
 
-        out_file.write_text(out_data)
+        out_file.write_text(out_data, encoding='utf-8')
 
         # make it executable
         mode = out_file.stat().st_mode
@@ -91,7 +91,7 @@ def gen_dockerfile(self) -> None:
             RUN /ci/install.sh
         ''')
 
-        out_file.write_text(out_data)
+        out_file.write_text(out_data, encoding='utf-8')
 
     def do_build(self) -> None:
         # copy files
@@ -131,7 +131,7 @@ def gen_dockerfile(self) -> None:
             ADD meson /meson
         ''')
 
-        out_file.write_text(out_data)
+        out_file.write_text(out_data, encoding='utf-8')
 
     def copy_meson(self) -> None:
         shutil.copytree(

docs/genrelnotes.py

@@ -36,10 +36,10 @@ def add_to_sitemap(from_version, to_version):
        Adds release note entry to sitemap.txt.
     '''
     sitemapfile = '../sitemap.txt'
-    s_f = open(sitemapfile)
+    s_f = open(sitemapfile, encoding='utf-8')
     lines = s_f.readlines()
     s_f.close()
-    with open(sitemapfile, 'w') as s_f:
+    with open(sitemapfile, 'w', encoding='utf-8') as s_f:
         for line in lines:
             if 'Release-notes' in line and from_version in line:
                 new_line = line.replace(from_version, to_version)
@@ -51,10 +51,10 @@ def generate(from_version, to_version):
        Generate notes for Meson build next release.
     '''
     ofilename = f'Release-notes-for-{to_version}.md'
-    with open(ofilename, 'w') as ofile:
+    with open(ofilename, 'w', encoding='utf-8') as ofile:
         ofile.write(RELNOTE_TEMPLATE.format(to_version, to_version))
         for snippetfile in glob('snippets/*.md'):
-            snippet = open(snippetfile).read()
+            snippet = open(snippetfile, encoding='utf-8').read()
             ofile.write(snippet)
             if not snippet.endswith('\n'):
                 ofile.write('\n')

mesonbuild/ast/interpreter.py

@@ -179,7 +179,7 @@ def func_subdir(self, node: BaseNode, args: T.List[TYPE_nvar], kwargs: T.Dict[st
         if not os.path.isfile(absname):
             sys.stderr.write(f'Unable to find build file {buildfilename} --> Skipping\n')
             return
-        with open(absname, encoding='utf8') as f:
+        with open(absname, encoding='utf-8') as f:
             code = f.read()
         assert(isinstance(code, str))
         try:

mesonbuild/backend/backends.py

@@ -374,7 +374,7 @@ def init_language_file(suffix, unity_file_number):
             if not os.path.exists(outfileabs_tmp_dir):
                 os.makedirs(outfileabs_tmp_dir)
             result.append(unity_src)
-            return open(outfileabs_tmp, 'w')
+            return open(outfileabs_tmp, 'w', encoding='utf-8')
 
         # For each language, generate unity source files and return the list
         for comp, srcs in compsrcs.items():
@@ -765,7 +765,7 @@ def create_msvc_pch_implementation(self, target, lang, pch_header):
 
         content = f'#include "{os.path.basename(pch_header)}"'
         pch_file_tmp = pch_file + '.tmp'
-        with open(pch_file_tmp, 'w') as f:
+        with open(pch_file_tmp, 'w', encoding='utf-8') as f:
             f.write(content)
         mesonlib.replace_if_different(pch_file, pch_file_tmp)
         return pch_rel_to_build
@@ -1019,7 +1019,7 @@ def generate_depmf_install(self, d: InstallData) -> None:
         ifilename = os.path.join(self.environment.get_build_dir(), 'depmf.json')
         ofilename = os.path.join(self.environment.get_prefix(), self.build.dep_manifest_name)
         mfobj = {'type': 'dependency manifest', 'version': '1.0', 'projects': self.build.dep_manifest}
-        with open(ifilename, 'w') as f:
+        with open(ifilename, 'w', encoding='utf-8') as f:
             f.write(json.dumps(mfobj))
         # Copy file from, to, and with mode unchanged
         d.data.append(InstallDataBase(ifilename, ofilename, None, ''))

mesonbuild/backend/ninjabackend.py

@@ -20,6 +20,7 @@
 from collections import OrderedDict
 from enum import Enum, unique
 import itertools
+from textwrap import dedent
 from pathlib import PurePath, Path
 from functools import lru_cache
 
@@ -462,10 +463,11 @@ def detect_vs_dep_prefix(self, tempfilename):
             return open(tempfilename, 'a', encoding='utf-8')
         filename = os.path.join(self.environment.get_scratch_dir(),
                                 'incdetect.c')
-        with open(filename, 'w') as f:
-            f.write('''#include<stdio.h>
-int dummy;
-''')
+        with open(filename, 'w', encoding='utf-8') as f:
+            f.write(dedent('''\
+                #include<stdio.h>
+                int dummy;
+            '''))
 
         # The output of cl dependency information is language
         # and locale dependent. Any attempt at converting it to
@@ -1215,7 +1217,7 @@ def generate_jar_target(self, target):
         manifest_path = os.path.join(self.get_target_private_dir(target), 'META-INF', 'MANIFEST.MF')
         manifest_fullpath = os.path.join(self.environment.get_build_dir(), manifest_path)
         os.makedirs(os.path.dirname(manifest_fullpath), exist_ok=True)
-        with open(manifest_fullpath, 'w') as manifest:
+        with open(manifest_fullpath, 'w', encoding='utf-8') as manifest:
             if any(target.link_targets):
                 manifest.write('Class-Path: ')
                 cp_paths = [os.path.join(self.get_target_dir(l), l.get_filename()) for l in target.link_targets]

mesonbuild/backend/vs2010backend.py

@@ -206,7 +206,7 @@ def get_regen_stampfile(build_dir: str) -> None:
 
     @staticmethod
     def touch_regen_timestamp(build_dir: str) -> None:
-        with open(Vs2010Backend.get_regen_stampfile(build_dir), 'w'):
+        with open(Vs2010Backend.get_regen_stampfile(build_dir), 'w', encoding='utf-8'):
             pass
 
     def get_vcvars_command(self):

mesonbuild/cmake/fileapi.py

@@ -51,7 +51,7 @@ def setup_request(self) -> None:
         }
 
         query_file = self.request_dir / 'query.json'
-        query_file.write_text(json.dumps(query, indent=2))
+        query_file.write_text(json.dumps(query, indent=2), encoding='utf-8')
 
     def load_reply(self) -> None:
         if not self.reply_dir.is_dir():
@@ -75,7 +75,7 @@ def load_reply(self) -> None:
         # Debug output
         debug_json = self.build_dir / '..' / 'fileAPI.json'
         debug_json = debug_json.resolve()
-        debug_json.write_text(json.dumps(index, indent=2))
+        debug_json.write_text(json.dumps(index, indent=2), encoding='utf-8')
         mlog.cmd_ci_include(debug_json.as_posix())
 
         # parse the JSON
@@ -313,7 +313,7 @@ def _reply_file_content(self, filename: Path) -> T.Dict[str, T.Any]:
         if not real_path.exists():
             raise CMakeException(f'File "{real_path}" does not exist')
 
-        data = json.loads(real_path.read_text())
+        data = json.loads(real_path.read_text(encoding='utf-8'))
         assert isinstance(data, dict)
         for i in data.keys():
             assert isinstance(i, str)

mesonbuild/cmake/toolchain.py

@@ -66,8 +66,8 @@ def __init__(self, cmakebin: 'CMakeExecutor', env: 'Environment', for_machine: M
     def write(self) -> Path:
         if not self.toolchain_file.parent.exists():
             self.toolchain_file.parent.mkdir(parents=True)
-        self.toolchain_file.write_text(self.generate())
-        self.cmcache_file.write_text(self.generate_cache())
+        self.toolchain_file.write_text(self.generate(), encoding='utf-8')
+        self.cmcache_file.write_text(self.generate_cache(), encoding='utf-8')
         mlog.cmd_ci_include(self.toolchain_file.as_posix())
         return self.toolchain_file
 
@@ -215,11 +215,11 @@ def update_cmake_compiler_state(self) -> None:
         build_dir = Path(self.env.scratch_dir) / '__CMake_compiler_info__'
         build_dir.mkdir(parents=True, exist_ok=True)
         cmake_file = build_dir / 'CMakeLists.txt'
-        cmake_file.write_text(cmake_content)
+        cmake_file.write_text(cmake_content, encoding='utf-8')
 
         # Generate the temporary toolchain file
         temp_toolchain_file = build_dir / 'CMakeMesonTempToolchainFile.cmake'
-        temp_toolchain_file.write_text(CMakeToolchain._print_vars(self.variables))
+        temp_toolchain_file.write_text(CMakeToolchain._print_vars(self.variables), encoding='utf-8')
 
         # Configure
         trace = CMakeTraceParser(self.cmakebin.version(), build_dir)

mesonbuild/cmake/traceparser.py

@@ -158,7 +158,7 @@ def parse(self, trace: T.Optional[str] = None) -> None:
         if not self.requires_stderr():
             if not self.trace_file_path.exists and not self.trace_file_path.is_file():
                 raise CMakeException('CMake: Trace file "{}" not found'.format(str(self.trace_file_path)))
-            trace = self.trace_file_path.read_text(errors='ignore')
+            trace = self.trace_file_path.read_text(errors='ignore', encoding='utf-8')
         if not trace:
             raise CMakeException('CMake: The CMake trace was not provided or is empty')
 

mesonbuild/compilers/compilers.py

@@ -765,14 +765,14 @@ def compile(self, code: 'mesonlib.FileOrString',
             if isinstance(code, str):
                 srcname = os.path.join(tmpdirname,
                                     'testfile.' + self.default_suffix)
-                with open(srcname, 'w') as ofile:
+                with open(srcname, 'w', encoding='utf-8') as ofile:
                     ofile.write(code)
                 # ccache would result in a cache miss
                 no_ccache = True
                 contents = code
             elif isinstance(code, mesonlib.File):
                 srcname = code.fname
-                with open(code.fname) as f:
+                with open(code.fname, encoding='utf-8') as f:
                     contents = f.read()
 
             # Construct the compiler command-line

mesonbuild/compilers/cs.py

@@ -87,7 +87,7 @@ def sanity_check(self, work_dir: str, environment: 'Environment') -> None:
         src = 'sanity.cs'
         obj = 'sanity.exe'
         source_name = os.path.join(work_dir, src)
-        with open(source_name, 'w') as ofile:
+        with open(source_name, 'w', encoding='utf-8') as ofile:
             ofile.write(textwrap.dedent('''
                 public class Sanity {
                     static public void Main () {

mesonbuild/compilers/cuda.py

@@ -504,7 +504,7 @@ def sanity_check(self, work_dir: str, env: 'Environment') -> None:
         binname += '_cross' if self.is_cross else ''
         source_name = os.path.join(work_dir, sname)
         binary_name = os.path.join(work_dir, binname + '.exe')
-        with open(source_name, 'w') as ofile:
+        with open(source_name, 'w', encoding='utf-8') as ofile:
             ofile.write(code)
 
         # The Sanity Test for CUDA language will serve as both a sanity test

mesonbuild/compilers/d.py

@@ -535,7 +535,7 @@ def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoic
     def sanity_check(self, work_dir: str, environment: 'Environment') -> None:
         source_name = os.path.join(work_dir, 'sanity.d')
         output_name = os.path.join(work_dir, 'dtest')
-        with open(source_name, 'w') as ofile:
+        with open(source_name, 'w', encoding='utf-8') as ofile:
             ofile.write('''void main() { }''')
         pc = subprocess.Popen(self.exelist + self.get_output_args(output_name) + self._get_target_arch_args() + [source_name], cwd=work_dir)
         pc.wait()

mesonbuild/compilers/fortran.py

@@ -71,7 +71,7 @@ def sanity_check(self, work_dir_: str, environment: 'Environment') -> None:
         if binary_name.is_file():
             binary_name.unlink()
 
-        source_name.write_text('print *, "Fortran compilation is working."; end')
+        source_name.write_text('print *, "Fortran compilation is working."; end', encoding='utf-8')
 
         extra_flags: T.List[str] = []
         extra_flags += environment.coredata.get_external_args(self.for_machine, self.language)

mesonbuild/compilers/java.py

@@ -70,7 +70,7 @@ def sanity_check(self, work_dir: str, environment: 'Environment') -> None:
         src = 'SanityCheck.java'
         obj = 'SanityCheck'
         source_name = os.path.join(work_dir, src)
-        with open(source_name, 'w') as ofile:
+        with open(source_name, 'w', encoding='utf-8') as ofile:
             ofile.write(textwrap.dedent(
                 '''class SanityCheck {
                   public static void main(String[] args) {

mesonbuild/compilers/mixins/clike.py

@@ -306,7 +306,7 @@ def _sanity_check_impl(self, work_dir: str, environment: 'Environment',
         binname += '.exe'
         # Write binary check source
         binary_name = os.path.join(work_dir, binname)
-        with open(source_name, 'w') as ofile:
+        with open(source_name, 'w', encoding='utf-8') as ofile:
             ofile.write(code)
         # Compile sanity check
         # NOTE: extra_flags must be added at the end. On MSVC, it might contain a '/link' argument

mesonbuild/compilers/rust.py

@@ -65,7 +65,7 @@ def needs_static_linker(self) -> bool:
     def sanity_check(self, work_dir: str, environment: 'Environment') -> None:
         source_name = os.path.join(work_dir, 'sanity.rs')
         output_name = os.path.join(work_dir, 'rusttest')
-        with open(source_name, 'w') as ofile:
+        with open(source_name, 'w', encoding='utf-8') as ofile:
             ofile.write(textwrap.dedent(
                 '''fn main() {
                 }

mesonbuild/compilers/swift.py

@@ -107,7 +107,7 @@ def sanity_check(self, work_dir: str, environment: 'Environment') -> None:
             extra_flags += self.get_compile_only_args()
         else:
             extra_flags += environment.coredata.get_external_link_args(self.for_machine, self.language)
-        with open(source_name, 'w') as ofile:
+        with open(source_name, 'w', encoding='utf-8') as ofile:
             ofile.write('''print("Swift compilation is working.")
 ''')
         pc = subprocess.Popen(self.exelist + extra_flags + ['-emit-executable', '-o', output_name, src], cwd=work_dir)

mesonbuild/coredata.py

@@ -481,8 +481,8 @@ def __load_config_files(options: argparse.Namespace, scratch_dir: str, ftype: st
                     # the contents of that file into the meson private (scratch)
                     # directory so that it can be re-read when wiping/reconfiguring
                     copy = os.path.join(scratch_dir, f'{uuid.uuid4()}.{ftype}.ini')
-                    with open(f) as rf:
-                        with open(copy, 'w') as wf:
+                    with open(f, encoding='utf-8') as rf:
+                        with open(copy, 'w', encoding='utf-8') as wf:
                             wf.write(rf.read())
                     real.append(copy)
 
@@ -973,15 +973,15 @@ def write_cmd_line_file(build_dir: str, options: argparse.Namespace) -> None:
 
     config['options'] = {str(k): str(v) for k, v in options.cmd_line_options.items()}
     config['properties'] = properties
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         config.write(f)
 
 def update_cmd_line_file(build_dir: str, options: argparse.Namespace):
     filename = get_cmd_line_file(build_dir)
     config = CmdLineFileParser()
     config.read(filename)
     config['options'].update({str(k): str(v) for k, v in options.cmd_line_options.items()})
-    with open(filename, 'w') as f:
+    with open(filename, 'w', encoding='utf-8') as f:
         config.write(f)
 
 def get_cmd_line_options(build_dir: str, options: argparse.Namespace) -> str:

mesonbuild/dependencies/boost.py

@@ -717,7 +717,7 @@ def _include_dir_from_version_header(self, hfile: Path) -> BoostIncludeDir:
         # also work, however, this is slower (since it the compiler has to be
         # invoked) and overkill since the layout of the header is always the same.
         assert hfile.exists()
-        raw = hfile.read_text()
+        raw = hfile.read_text(encoding='utf-8')
         m = re.search(r'#define\s+BOOST_VERSION\s+([0-9]+)', raw)
         if not m:
             mlog.debug(f'Failed to extract version information from {hfile}')

mesonbuild/dependencies/cmake.py

@@ -620,7 +620,7 @@ def _setup_cmake_dir(self, cmake_file: str) -> Path:
         """).format(' '.join(cmake_language)) + cmake_txt
 
         cm_file = build_dir / 'CMakeLists.txt'
-        cm_file.write_text(cmake_txt)
+        cm_file.write_text(cmake_txt, encoding='utf-8')
         mlog.cmd_ci_include(cm_file.absolute().as_posix())
 
         return build_dir

mesonbuild/dependencies/cuda.py

@@ -184,7 +184,7 @@ def _cuda_toolkit_version(self, path: str) -> str:
     def _read_cuda_runtime_api_version(self, path_str: str) -> T.Optional[str]:
         path = Path(path_str)
         for i in path.rglob('cuda_runtime_api.h'):
-            raw = i.read_text()
+            raw = i.read_text(encoding='utf-8')
             m = self.cudart_version_regex.search(raw)
             if not m:
                 continue
@@ -202,7 +202,7 @@ def _read_toolkit_version_txt(self, path: str) -> T.Optional[str]:
         # Read 'version.txt' at the root of the CUDA Toolkit directory to determine the tookit version
         version_file_path = os.path.join(path, 'version.txt')
         try:
-            with open(version_file_path) as version_file:
+            with open(version_file_path, encoding='utf-8') as version_file:
                 version_str = version_file.readline() # e.g. 'CUDA Version 10.1.168'
                 m = self.toolkit_version_regex.match(version_str)
                 if m:

mesonbuild/dependencies/pkgconfig.py

@@ -423,7 +423,7 @@ def check_pkgconfig(self, pkgbin: ExternalProgram) -> T.Optional[str]:
         return out.strip()
 
     def extract_field(self, la_file: str, fieldname: str) -> T.Optional[str]:
-        with open(la_file) as f:
+        with open(la_file, encoding='utf-8') as f:
             for line in f:
                 arr = line.strip().split('=')
                 if arr[0] == fieldname:

mesonbuild/interpreter/interpreter.py

@@ -973,7 +973,7 @@ def _do_subproject_cmake(self, subp_name, subdir, subdir_abs, default_options, k
                 ast.accept(printer)
                 printer.post_process()
                 meson_filename = os.path.join(self.build.environment.get_build_dir(), subdir, 'meson.build')
-                with open(meson_filename, "w") as f:
+                with open(meson_filename, "w", encoding='utf-8') as f:
                     f.write(printer.result)
 
                 mlog.log('Build file:', meson_filename)
@@ -1964,7 +1964,7 @@ def func_subdir(self, node, args, kwargs):
         if not os.path.isfile(absname):
             self.subdir = prev_subdir
             raise InterpreterException(f"Non-existent build file '{buildfilename!s}'")
-        with open(absname, encoding='utf8') as f:
+        with open(absname, encoding='utf-8') as f:
             code = f.read()
         assert(isinstance(code, str))
         try:
@@ -2198,7 +2198,7 @@ def func_configure_file(self, node, args, kwargs):
                 mesonlib.replace_if_different(ofile_abs, dst_tmp)
             if depfile:
                 mlog.log('Reading depfile:', mlog.bold(depfile))
-                with open(depfile) as f:
+                with open(depfile, encoding='utf-8') as f:
                     df = DepFile(f.readlines())
                     deps = df.get_all_dependencies(ofile_fname)
                     for dep in deps:

mesonbuild/interpreterbase/interpreterbase.py

@@ -95,7 +95,7 @@ def load_root_meson_file(self) -> None:
         mesonfile = os.path.join(self.source_root, self.subdir, environment.build_filename)
         if not os.path.isfile(mesonfile):
             raise InvalidArguments('Missing Meson file in %s' % mesonfile)
-        with open(mesonfile, encoding='utf8') as mf:
+        with open(mesonfile, encoding='utf-8') as mf:
             code = mf.read()
         if code.isspace():
             raise InvalidCode('Builder file is empty.')