The Windows filesystem and process surface: PEP 529 paths, OSError.winerror, the noop os calls, and subprocess

lib-python/3/subprocess.py

@@ -1358,6 +1358,12 @@ def _get_handles(self, stdin, stdout, stderr):
             c2pread, c2pwrite = -1, -1
             errread, errwrite = -1, -1
 
+            # A handle this method made a pipe of is closed as soon as its
+            # inheritable duplicate replaces it.  Leaving that to the `Handle`
+            # wrapper's `__del__` waits on the collector, and until it runs the
+            # parent still holds the write end of the pipe it is reading to
+            # end-of-file — which never arrives.
+            ispread = False
             with self._on_error_fd_closer() as err_close_fds:
                 if stdin is None:
                     p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
@@ -1366,50 +1372,60 @@ def _get_handles(self, stdin, stdout, stderr):
                         p2cread = Handle(p2cread)
                         err_close_fds.append(p2cread)
                         _winapi.CloseHandle(_)
+                        ispread = True
                 elif stdin == PIPE:
                     p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
                     p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
                     err_close_fds.extend((p2cread, p2cwrite))
+                    ispread = True
                 elif stdin == DEVNULL:
                     p2cread = msvcrt.get_osfhandle(self._get_devnull())
                 elif isinstance(stdin, int):
                     p2cread = msvcrt.get_osfhandle(stdin)
                 else:
                     # Assuming file-like object
                     p2cread = msvcrt.get_osfhandle(stdin.fileno())
-                p2cread = self._make_inheritable(p2cread)
+                p2cread = self._make_inheritable(p2cread, ispread)
 
+                ispwrite = False
                 if stdout is None:
                     c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
                     if c2pwrite is None:
                         _, c2pwrite = _winapi.CreatePipe(None, 0)
                         c2pwrite = Handle(c2pwrite)
                         err_close_fds.append(c2pwrite)
                         _winapi.CloseHandle(_)
+                        ispwrite = True
                 elif stdout == PIPE:
                     c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
                     c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
                     err_close_fds.extend((c2pread, c2pwrite))
+                    ispwrite = True
                 elif stdout == DEVNULL:
                     c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
                 elif isinstance(stdout, int):
                     c2pwrite = msvcrt.get_osfhandle(stdout)
                 else:
                     # Assuming file-like object
                     c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
-                c2pwrite = self._make_inheritable(c2pwrite)
+                c2pwrite = self._make_inheritable(c2pwrite, ispwrite)
 
+                # `stderr == STDOUT` names the duplicate `c2pwrite` already
+                # holds, which stays open for as long as that one does.
+                ispwrite = False
                 if stderr is None:
                     errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
                     if errwrite is None:
                         _, errwrite = _winapi.CreatePipe(None, 0)
                         errwrite = Handle(errwrite)
                         err_close_fds.append(errwrite)
                         _winapi.CloseHandle(_)
+                        ispwrite = True
                 elif stderr == PIPE:
                     errread, errwrite = _winapi.CreatePipe(None, 0)
                     errread, errwrite = Handle(errread), Handle(errwrite)
                     err_close_fds.extend((errread, errwrite))
+                    ispwrite = True
                 elif stderr == STDOUT:
                     errwrite = c2pwrite
                 elif stderr == DEVNULL:
@@ -1419,19 +1435,23 @@ def _get_handles(self, stdin, stdout, stderr):
                 else:
                     # Assuming file-like object
                     errwrite = msvcrt.get_osfhandle(stderr.fileno())
-                errwrite = self._make_inheritable(errwrite)
+                errwrite = self._make_inheritable(errwrite, ispwrite)
 
             return (p2cread, p2cwrite,
                     c2pread, c2pwrite,
                     errread, errwrite)
 
 
-        def _make_inheritable(self, handle):
+        def _make_inheritable(self, handle, close=False):
             """Return a duplicate of handle, which is inheritable"""
             h = _winapi.DuplicateHandle(
                 _winapi.GetCurrentProcess(), handle,
                 _winapi.GetCurrentProcess(), 0, 1,
                 _winapi.DUPLICATE_SAME_ACCESS)
+            # The handle the duplicate replaces is closed here when it was
+            # `_get_handles`' own — an end of a pipe it just made.
+            if close:
+                handle.Close()
             return Handle(h)
 
 

pyre/check.py

@@ -24,14 +24,69 @@
 
 EXE = ".exe" if sys.platform == "win32" else ""
 # pyre's compat target is CPython 3.14; its native modules (`_sre.MAGIC`) and the
-# vendored `lib-python/3` are coupled to that version. Prefer a version-matched
-# oracle so a stale `python3` on PATH (an older system CPython) does not diverge
-# from pypy on version-sensitive error text and trip a spurious cpython-vs-pypy
-# baseline mismatch.
-PYTHON3 = os.environ.get("PYRE_CHECK_PYTHON3") or next(
-    (cand for cand in ("python3.14", "python3", "python") if shutil.which(cand)),
-    "python3",
-)
+# vendored `lib-python/3` are coupled to that version. Take a version-matched
+# oracle or none at all: a stale `python3` on PATH (an older system CPython)
+# diverges from pypy on version-sensitive error text and trips a spurious
+# cpython-vs-pypy baseline mismatch, and its timings gate the ratios.
+CPYTHON_TARGET = (3, 14)
+
+
+def _probe_interpreter(command):
+    """What the interpreter reports as its version and its own path.
+
+    `None` when the command did not run at all, which on Windows is what a
+    `python3.14` naming an extensionless shim rather than an executable does —
+    `shutil.which` finds it and `CreateProcess` cannot start it.
+    """
+    probe = "import sys; print(sys.version_info[0], sys.version_info[1]); print(sys.executable)"
+    try:
+        proc = subprocess.run(
+            [command, "-c", probe], capture_output=True, text=True, timeout=30,
+        )
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if proc.returncode != 0:
+        return None
+    lines = (proc.stdout or "").splitlines()
+    if len(lines) < 2:
+        return None
+    try:
+        major, minor = lines[0].split()
+    except ValueError:
+        return None
+    return (int(major), int(minor)), lines[1].strip() or command
+
+
+def _resolve_python3():
+    """The oracle interpreter, as the absolute path it reports for itself.
+
+    A bare name would be resolved against the PATH of whichever environment
+    spawns it, and the timed runs hand the child a curated one.
+    """
+    named = os.environ.get("PYRE_CHECK_PYTHON3")
+    candidates = [named] if named else ["python3.14", "python3", "python"]
+    rejected = []
+    for candidate in candidates:
+        if named is None and shutil.which(candidate) is None:
+            continue
+        probed = _probe_interpreter(candidate)
+        if probed is None:
+            rejected.append(f"  {candidate}: did not run")
+            continue
+        version, executable = probed
+        if version == CPYTHON_TARGET:
+            return executable
+        rejected.append("  %s: %d.%d" % (candidate, *version))
+    wanted = "%d.%d" % CPYTHON_TARGET
+    raise SystemExit(
+        f"no CPython {wanted} to measure against — pyre targets it, and an "
+        f"older one disagrees on version-sensitive behaviour and timings.\n"
+        + ("\n".join(rejected) or "  (no candidate on PATH)")
+        + "\nName one with PYRE_CHECK_PYTHON3."
+    )
+
+
+PYTHON3 = _resolve_python3()
 PYPY3 = os.environ.get("PYRE_CHECK_PYPY3") or (
     "pypy3" if shutil.which("pypy3") else "pypy"
 )

pyre/extra_tests/parity_tests/compile_filename_boundary.py

@@ -16,8 +16,22 @@
 and this suite has to pass under CPython too.
 """
 
+import sys
+
 SOURCE = "x = 1"
 
+# The filename the filesystem encoding carries but plain UTF-8 text cannot, and
+# the bytes that spell it.  Windows encodes with `surrogatepass` (PEP 529), so
+# the lone surrogate is spelled by its own three UTF-8 bytes and a byte that
+# begins no sequence has no spelling at all; every other platform uses
+# `surrogateescape`, where that same byte is what the surrogate stands for.
+if sys.platform == "win32":
+    NAME_BYTES, NAME_TEXT = b"\xed\xb3\xbf.py", "\udcff.py"
+    UNSPELLABLE = b"\xff.py"
+else:
+    NAME_BYTES, NAME_TEXT = b"\xff.py", "\udcff.py"
+    UNSPELLABLE = None
+
 
 class Path:
     def __init__(self, name):
@@ -33,14 +47,23 @@ def __fspath__(self):
 # bytes decode with the filesystem encoding.  The ASCII case matters on its own:
 # it needs no surrogate to expose a filename that never reaches the code object.
 assert compile(SOURCE, b"ascii.py", "exec").co_filename == "ascii.py"
-assert compile(SOURCE, b"\xff.py", "exec").co_filename == "\udcff.py"
+assert compile(SOURCE, NAME_BYTES, "exec").co_filename == NAME_TEXT
+
+# A str the encoding can only spell as a surrogate encodes back to the same
+# bytes instead of raising UnicodeEncodeError.
+assert compile(SOURCE, NAME_TEXT, "exec").co_filename == NAME_TEXT
 
-# A str already carrying a surrogate escape encodes back to that same byte
-# instead of raising UnicodeEncodeError.
-assert compile(SOURCE, "\udcff.py", "exec").co_filename == "\udcff.py"
+# Bytes the encoding cannot spell are reported, not renamed.
+if UNSPELLABLE is not None:
+    try:
+        compile(SOURCE, UNSPELLABLE, "exec")
+    except UnicodeDecodeError:
+        pass
+    else:
+        raise AssertionError("compile() invented a spelling for %r" % (UNSPELLABLE,))
 
 # `__fspath__` is honoured, and may answer with either str or bytes.
-assert compile(SOURCE, Path(b"\xff.py"), "exec").co_filename == "\udcff.py"
+assert compile(SOURCE, Path(NAME_BYTES), "exec").co_filename == NAME_TEXT
 assert compile(SOURCE, Path("spelled.py"), "exec").co_filename == "spelled.py"
 
 # An object that is neither a path nor a string is a TypeError, not a filename
@@ -75,9 +98,9 @@ def __fspath__(self):
 # The filename a SyntaxError reports is the same one the successful compile
 # would have recorded.
 try:
-    compile("(", b"\xff.py", "exec")
+    compile("(", NAME_BYTES, "exec")
 except SyntaxError as exc:
-    assert exc.filename == "\udcff.py", ascii(exc.filename)
+    assert exc.filename == NAME_TEXT, ascii(exc.filename)
 else:
     raise AssertionError("compile() accepted an unterminated '('")
 
@@ -87,26 +110,29 @@ def __fspath__(self):
 code = compile(SOURCE, "before.py", "exec")
 assert 'file "before.py"' in repr(code), repr(code)
 
-replaced = code.replace(co_filename="\udcff.py")
-assert replaced.co_filename == "\udcff.py"
-assert 'file "\udcff.py"' in repr(replaced), ascii(repr(replaced))
+replaced = code.replace(co_filename=NAME_TEXT)
+assert replaced.co_filename == NAME_TEXT
+assert 'file "%s"' % NAME_TEXT in repr(replaced), ascii(repr(replaced))
 
 # `pycode.py:570-572` reports the zero sentinel as line -1.
 assert 'line -1>' in repr(code.replace(co_firstlineno=0)), ascii(
     repr(code.replace(co_firstlineno=0))
 )
 
-# and a filename that never had a UTF-8 spelling reaches repr() the same way.
-from_bytes = compile(SOURCE, b"\xff.py", "exec")
-assert 'file "\udcff.py"' in repr(from_bytes), ascii(repr(from_bytes))
+# and a filename that has no plain-text spelling reaches repr() the same way.
+from_bytes = compile(SOURCE, NAME_BYTES, "exec")
+assert 'file "%s"' % NAME_TEXT in repr(from_bytes), ascii(repr(from_bytes))
 
 
 # Every reader of a code object agrees on the filename, so a frame built from
 # one points at the real file rather than at "<string>".
 namespace = {}
-exec(compile("def f():\n    import sys\n    return sys._getframe()\n", b"\xff.py", "exec"), namespace)
+exec(
+    compile("def f():\n    import sys\n    return sys._getframe()\n", NAME_BYTES, "exec"),
+    namespace,
+)
 frame = namespace["f"]()
-assert frame.f_code.co_filename == "\udcff.py", ascii(frame.f_code.co_filename)
+assert frame.f_code.co_filename == NAME_TEXT, ascii(frame.f_code.co_filename)
 assert "�" not in repr(frame), ascii(repr(frame))
 
 print("OK")

pyre/extra_tests/parity_tests/dir_entry_uninstantiable_python314.py

@@ -7,7 +7,8 @@
 `acceptable_as_base_class = False` (:487); `:172-180` does the same for
 `W_ScandirIterator.typedef`.  `:463-465 descr_reduce_ex` refuses to pickle an
 entry, spelling the type with `%T` (`error.py:592-593` -> the qualified typedef
-name `'posix.DirEntry'`, `:469`).
+name `'posix.DirEntry'`, `:469`).  The module holding these types is the one
+`os` is implemented by, which Windows spells `nt`.
 
 Pickle protocols 0 and 1 are deliberately NOT asserted: they never reach
 `reduce_newobj`, they land in `copyreg._reduce_ex`, and that leg already
@@ -17,46 +18,50 @@
 import copy
 import os
 import pickle
+import sys
 
 HERE = os.path.dirname(os.path.abspath(__file__))
+MODULE = "nt" if sys.platform == "win32" else "posix"
 
-assert os.DirEntry.__module__ == "posix", os.DirEntry.__module__
+assert os.DirEntry.__module__ == MODULE, os.DirEntry.__module__
 assert os.DirEntry.__name__ == "DirEntry", os.DirEntry.__name__
 assert os.DirEntry.__qualname__ == "DirEntry", os.DirEntry.__qualname__
-assert repr(os.DirEntry) == "<class 'posix.DirEntry'>", repr(os.DirEntry)
+assert repr(os.DirEntry) == "<class '%s.DirEntry'>" % MODULE, repr(os.DirEntry)
 
 try:
     os.DirEntry()
 except TypeError as exc:
-    assert str(exc) == "cannot create 'posix.DirEntry' instances", str(exc)
+    assert str(exc) == "cannot create '%s.DirEntry' instances" % MODULE, str(exc)
 else:
     raise AssertionError("os.DirEntry() must not construct an entry")
 
 try:
     class _Sub(os.DirEntry):
         pass
 except TypeError as exc:
-    assert str(exc) == "type 'posix.DirEntry' is not an acceptable base type", str(exc)
+    expected = "type '%s.DirEntry' is not an acceptable base type" % MODULE
+    assert str(exc) == expected, str(exc)
 else:
     raise AssertionError("os.DirEntry must not be an acceptable base type")
 
 it = os.scandir(HERE)
 scandir_iterator = type(it)
-assert scandir_iterator.__module__ == "posix", scandir_iterator.__module__
+assert scandir_iterator.__module__ == MODULE, scandir_iterator.__module__
 assert scandir_iterator.__name__ == "ScandirIterator", scandir_iterator.__name__
 
 try:
     scandir_iterator()
 except TypeError as exc:
-    assert str(exc) == "cannot create 'posix.ScandirIterator' instances", str(exc)
+    assert str(exc) == "cannot create '%s.ScandirIterator' instances" % MODULE, str(exc)
 else:
     raise AssertionError("ScandirIterator() must not construct an iterator")
 
 try:
     class _SubIter(scandir_iterator):
         pass
 except TypeError as exc:
-    assert str(exc) == "type 'posix.ScandirIterator' is not an acceptable base type", str(exc)
+    expected = "type '%s.ScandirIterator' is not an acceptable base type" % MODULE
+    assert str(exc) == expected, str(exc)
 else:
     raise AssertionError("ScandirIterator must not be an acceptable base type")
 
@@ -72,15 +77,15 @@ class _SubIter(scandir_iterator):
     try:
         pickle.dumps(entry, protocol)
     except TypeError as exc:
-        assert str(exc) == "cannot pickle 'posix.DirEntry' object", (protocol, str(exc))
+        assert str(exc) == "cannot pickle '%s.DirEntry' object" % MODULE, (protocol, str(exc))
     else:
         raise AssertionError("a DirEntry must refuse to be pickled")
 
 for clone in (copy.copy, copy.deepcopy):
     try:
         clone(entry)
     except TypeError as exc:
-        assert str(exc) == "cannot pickle 'posix.DirEntry' object", (clone, str(exc))
+        assert str(exc) == "cannot pickle '%s.DirEntry' object" % MODULE, (clone, str(exc))
     else:
         raise AssertionError("a DirEntry must refuse to be copied")