"""Subprocess specification and related utilities."""
import contextlib
import inspect
import io
import os
import pathlib
import re
import shlex
import signal
import stat
import subprocess
import sys
import xonsh.lib.lazyasd as xl
import xonsh.lib.lazyimps as xli
import xonsh.platform as xp
import xonsh.procs.jobs as xj
import xonsh.tools as xt
from xonsh.built_ins import XSH
from xonsh.procs.executables import locate_executable
from xonsh.procs.pipelines import (
STDOUT_CAPTURE_KINDS,
CommandPipeline,
HiddenCommandPipeline,
resume_process,
)
from xonsh.procs.posix import PopenThread
from xonsh.procs.proxies import ProcProxy, ProcProxyThread
from xonsh.procs.readers import ConsoleParallelReader
@xl.lazyobject
def RE_SHEBANG():
return re.compile(r"#![ \t]*(.+?)$")
[docs]
def is_app_execution_alias(fname):
"""App execution aliases behave strangly on Windows and Python.
Here we try to detect if a file is an app execution alias.
"""
fname = pathlib.Path(fname)
try:
return fname.stat().st_reparse_tag == stat.IO_REPARSE_TAG_APPEXECLINK
# os.stat().st_reparse_tag is python 3.8+, and os.stat(app_exec_alias) throws OSError for <= 3.7
# so use old method as fallback
except (AttributeError, OSError):
return not os.path.exists(fname) and fname.name in os.listdir(fname.parent)
def _is_binary(fname, limit=80):
try:
with open(fname, "rb") as f:
for _ in range(limit):
char = f.read(1)
if char == b"\0":
return True
if char == b"\n":
return False
if char == b"":
return
except OSError as e:
if xp.ON_WINDOWS and is_app_execution_alias(fname):
return True
raise e
return False
def _un_shebang(x):
if x == "/usr/bin/env":
return []
elif any(x.startswith(i) for i in ["/usr/bin", "/usr/local/bin", "/bin"]):
x = os.path.basename(x)
elif x.endswith("python") or x.endswith("python.exe"):
x = "python"
if x == "xonsh":
return ["python", "-m", "xonsh.main"]
return [x]
[docs]
def get_script_subproc_command(fname, args):
"""Given the name of a script outside the path, returns a list representing
an appropriate subprocess command to execute the script or None if
the argument is not readable or not a script. Raises PermissionError
if the script is not executable.
"""
# make sure file is executable
if not os.access(fname, os.X_OK):
if not xp.ON_CYGWIN:
raise PermissionError
# explicitly look at all PATH entries for cmd
w_path = os.getenv("PATH").split(":")
w_fpath = list(map(lambda p: p + os.sep + fname, w_path))
if not any(list(map(lambda c: os.access(c, os.X_OK), w_fpath))):
raise PermissionError
if xp.ON_POSIX and not os.access(fname, os.R_OK):
# on some systems, some important programs (e.g. sudo) will have
# execute permissions but not read/write permissions. This enables
# things with the SUID set to be run. Needs to come before _is_binary()
# is called, because that function tries to read the file.
return None
elif _is_binary(fname):
# if the file is a binary, we should call it directly
return None
if xp.ON_WINDOWS:
# Windows can execute various filetypes directly
# as given in PATHEXT
_, ext = os.path.splitext(fname)
if ext.upper() in XSH.env.get("PATHEXT"):
return [fname] + args
# find interpreter
with open(fname, "rb") as f:
first_line = f.readline().decode().strip()
m = RE_SHEBANG.match(first_line)
# xonsh is the default interpreter
if m is None:
interp = ["xonsh"]
else:
interp = m.group(1).strip()
if len(interp) > 0:
interp = shlex.split(interp)
else:
interp = ["xonsh"]
if xp.ON_WINDOWS:
o = []
for i in interp:
o.extend(_un_shebang(i))
interp = o
return interp + [fname] + args
@xl.lazyobject
def _REDIR_REGEX():
name = r"(o(?:ut)?|e(?:rr)?|a(?:ll)?|&?\d?)"
return re.compile(f"{name}(>?>|<){name}$")
@xl.lazyobject
def _MODES():
return {">>": "a", ">": "w", "<": "r"}
@xl.lazyobject
def _WRITE_MODES():
return frozenset({"w", "a"})
@xl.lazyobject
def _REDIR_ALL():
return frozenset({"&", "a", "all"})
@xl.lazyobject
def _REDIR_ERR():
return frozenset({"2", "e", "err"})
@xl.lazyobject
def _REDIR_OUT():
return frozenset({"", "1", "o", "out"})
@xl.lazyobject
def _E2O_MAP():
return frozenset({f"{e}>{o}" for e in _REDIR_ERR for o in _REDIR_OUT if o != ""})
@xl.lazyobject
def _O2E_MAP():
return frozenset({f"{o}>{e}" for e in _REDIR_ERR for o in _REDIR_OUT if o != ""})
[docs]
def safe_open(fname, mode, buffering=-1):
"""Safely attempts to open a file in for xonsh subprocs."""
# file descriptors
try:
return open(fname, mode, buffering=buffering)
except PermissionError as ex:
raise xt.XonshError(f"xonsh: {fname}: permission denied") from ex
except FileNotFoundError as ex:
raise xt.XonshError(f"xonsh: {fname}: no such file or directory") from ex
except Exception as ex:
raise xt.XonshError(f"xonsh: {fname}: unable to open file") from ex
[docs]
def safe_close(x):
"""Safely attempts to close an object."""
if not isinstance(x, io.IOBase):
return
if x.closed:
return
try:
x.close()
except Exception:
pass
def _parse_redirects(r, loc=None):
"""returns origin, mode, destination tuple"""
orig, mode, dest = _REDIR_REGEX.match(r).groups()
# redirect to fd
if dest.startswith("&"):
try:
dest = int(dest[1:])
if loc is None:
loc, dest = dest, "" # NOQA
else:
e = f"Unrecognized redirection command: {r}"
raise xt.XonshError(e)
except (ValueError, xt.XonshError):
raise
except Exception:
pass
mode = _MODES.get(mode, None)
if mode == "r" and (len(orig) > 0 or len(dest) > 0):
raise xt.XonshError(f"Unrecognized redirection command: {r}")
elif mode in _WRITE_MODES and len(dest) > 0:
raise xt.XonshError(f"Unrecognized redirection command: {r}")
return orig, mode, dest
def _redirect_streams(r, loc=None):
"""Returns stdin, stdout, stderr tuple of redirections."""
if isinstance(loc, list):
raise Exception(f"Unsupported redirect: {r!r} {loc!r}")
stdin = stdout = stderr = None
no_ampersand = r.replace("&", "")
# special case of redirecting stderr to stdout
if no_ampersand in _E2O_MAP:
stderr = subprocess.STDOUT
return stdin, stdout, stderr
elif no_ampersand in _O2E_MAP:
stdout = 2 # using 2 as a flag, rather than using a file object
return stdin, stdout, stderr
# get streams
orig, mode, dest = _parse_redirects(r)
if mode == "r":
stdin = safe_open(loc, mode)
elif mode in _WRITE_MODES:
if orig in _REDIR_ALL:
stdout = stderr = safe_open(loc, mode)
elif orig in _REDIR_OUT:
stdout = safe_open(loc, mode)
elif orig in _REDIR_ERR:
stderr = safe_open(loc, mode)
else:
raise xt.XonshError(f"Unrecognized redirection command: {r}")
else:
raise xt.XonshError(f"Unrecognized redirection command: {r}")
return stdin, stdout, stderr
def _flatten_cmd_redirects(cmd):
"""Transforms a command like ['ls', ('>', '/dev/null')] into ['ls', '>', '/dev/null']."""
new_cmd = []
for c in cmd:
if isinstance(c, tuple):
new_cmd.extend(c)
else:
new_cmd.append(c)
return new_cmd
[docs]
def default_signal_pauser(n, f):
"""Pauses a signal, as needed."""
signal.pause()
[docs]
def no_pg_xonsh_preexec_fn():
"""Default subprocess preexec function for when there is no existing
pipeline group.
"""
os.setpgrp()
signal.signal(signal.SIGTSTP, default_signal_pauser)
[docs]
class DecoratorAlias:
"""Decorator alias base class."""
descr = "DecoratorAlias base."
def __call__(
self,
args,
stdin=None,
stdout=None,
stderr=None,
spec=None,
stack=None,
**kwargs,
):
print(self.descr, file=stdout)
[docs]
def decorate_spec(self, spec):
"""Modify spec immediately after modifier added."""
pass
[docs]
def decorate_spec_pre_run(self, pipeline, spec, spec_num):
"""Modify spec before run."""
pass
[docs]
class SpecAttrDecoratorAlias(DecoratorAlias):
"""Decorator Alias for spec attributes."""
def __init__(self, set_attributes: dict, descr=""):
self.set_attributes = set_attributes
self.descr = descr
super().__init__()
[docs]
def decorate_spec(self, spec):
for a, v in self.set_attributes.items():
setattr(spec, a, v)
[docs]
class SubprocSpec:
"""A container for specifying how a subprocess command should be
executed.
"""
kwnames = ("stdin", "stdout", "stderr", "universal_newlines", "close_fds")
def __init__(
self,
cmd,
cls=subprocess.Popen,
stdin=None,
stdout=None,
stderr=None,
universal_newlines=False,
close_fds=False,
captured=False,
env=None,
):
"""
Parameters
----------
cmd : list of str
Command to be run.
cls : Popen-like
Class to run the subprocess with.
stdin : file-like
Popen file descriptor or flag for stdin.
stdout : file-like
Popen file descriptor or flag for stdout.
stderr : file-like
Popen file descriptor or flag for stderr.
universal_newlines : bool
Whether or not to use universal newlines.
close_fds : bool
Whether or not to close the file descriptors when the
process exits.
captured : bool or str, optional
The flag for if the subprocess is captured, may be one of:
False for $[], 'stdout' for $(), 'hiddenobject' for ![], or
'object' for !().
env : dict
Replacement environment to run the subporcess in.
Attributes
----------
args : list of str
Arguments as originally supplied.
alias : list of str, callable, or None
The alias that was resolved for this command, if any.
binary_loc : str or None
Path to binary to execute.
is_proxy : bool
Whether or not the subprocess is or should be run as a proxy.
background : bool
Whether or not the subprocess should be started in the background.
threadable : bool
Whether or not the subprocess is able to be run in a background
thread, rather than the main thread.
pipeline_index : int or None
The index number of this sepc into the pipeline that is being setup.
last_in_pipeline : bool
Whether the subprocess is the last in the execution pipeline.
captured_stdout : file-like
Handle to captured stdin
captured_stderr : file-like
Handle to captured stderr
stack : list of FrameInfo namedtuples or None
The stack of the call-site of alias, if the alias requires it.
None otherwise.
"""
self._stdin = self._stdout = self._stderr = None
# args
self.cmd = list(cmd)
self.cls = cls
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.universal_newlines = universal_newlines
self.close_fds = close_fds
self.captured = captured
if env is not None:
self.env = {
k: v if not (isinstance(v, list)) or len(v) > 1 else v[0]
for (k, v) in env.items()
}
else:
self.env = None
# pure attrs
self.args = _flatten_cmd_redirects(cmd)
self.alias = None
self.alias_name = None
self.alias_stack = XSH.env.get("__ALIAS_STACK", "").split(":")
self.binary_loc = None
self.is_proxy = False
self.background = False
self.threadable = True
self.force_threadable = None # Set this value to ignore threadable prediction.
self.pipeline_index = None
self.last_in_pipeline = False
self.captured_stdout = None
self.captured_stderr = None
self.stack = None
self.decorators = [] # List of DecoratorAlias objects that applied to spec.
self.output_format = XSH.env.get("XONSH_SUBPROC_OUTPUT_FORMAT", "stream_lines")
self.raise_subproc_error = None # Spec-based $RAISE_SUBPROC_ERROR.
def __str__(self):
s = self.__class__.__name__ + "(" + str(self.cmd) + ", "
s += self.cls.__name__ + ", "
kws = [n + "=" + str(getattr(self, n)) for n in self.kwnames]
s += ", ".join(kws) + ")"
return s
def __repr__(self):
s = self.__class__.__name__ + "(" + repr(self.cmd) + ", "
s += self.cls.__name__ + ", "
kws = [n + "=" + repr(getattr(self, n)) for n in self.kwnames]
s += ", ".join(kws) + ")"
return s
#
# Properties
#
@property
def stdin(self):
return self._stdin
@stdin.setter
def stdin(self, value):
if self._stdin is None:
self._stdin = value
elif value is None:
pass
else:
safe_close(value)
msg = "Multiple inputs for stdin for {0!r}"
msg = msg.format(self.get_command_str())
raise xt.XonshError(msg)
@property
def stdout(self):
return self._stdout
@stdout.setter
def stdout(self, value):
if self._stdout is None:
self._stdout = value
elif value is None:
pass
else:
safe_close(value)
msg = "Multiple redirections for stdout for {0!r}"
msg = msg.format(self.get_command_str())
raise xt.XonshError(msg)
@property
def stderr(self):
return self._stderr
@stderr.setter
def stderr(self, value):
if self._stderr is None:
self._stderr = value
elif value is None:
pass
else:
safe_close(value)
msg = "Multiple redirections for stderr for {0!r}"
msg = msg.format(self.get_command_str())
raise xt.XonshError(msg)
[docs]
def get_command_str(self):
return " ".join(arg for arg in self.args)
#
# Execution methods
#
[docs]
def run(self, *, pipeline_group=None):
"""Launches the subprocess and returns the object."""
event_name = self._cmd_event_name()
self._pre_run_event_fire(event_name)
kwargs = {n: getattr(self, n) for n in self.kwnames}
if callable(self.alias):
kwargs["env"] = self.env or {}
kwargs["env"]["__ALIAS_NAME"] = self.alias_name or ""
p = self.cls(self.alias, self.cmd, **kwargs)
else:
self.prep_env_subproc(kwargs)
self.prep_preexec_fn(kwargs, pipeline_group=pipeline_group)
self._fix_null_cmd_bytes()
p = self._run_binary(kwargs)
p.spec = self
p.last_in_pipeline = self.last_in_pipeline
p.captured_stdout = self.captured_stdout
p.captured_stderr = self.captured_stderr
self._post_run_event_fire(event_name, p)
return p
def _run_binary(self, kwargs):
if not self.cmd[0]:
raise xt.XonshError("xonsh: subprocess mode: command is empty")
bufsize = 1
try:
if xp.ON_WINDOWS and self.binary_loc is not None:
# launch process using full paths (https://bugs.python.org/issue8557)
cmd = [self.binary_loc] + self.cmd[1:]
else:
cmd = self.cmd
p = self.cls(cmd, bufsize=bufsize, **kwargs)
except PermissionError as ex:
e = "xonsh: subprocess mode: permission denied: {0}"
raise xt.XonshError(e.format(self.cmd[0])) from ex
except FileNotFoundError as ex:
cmd0 = self.cmd[0]
if len(self.cmd) == 1 and cmd0.endswith("?"):
cmdq = cmd0.rstrip("?")
if cmdq in XSH.aliases:
alias = XSH.aliases[cmdq]
descr = (
repr(alias) + (":\n" + doc)
if (doc := getattr(alias, "__doc__", ""))
else ""
)
return self.cls(["echo", descr], bufsize=bufsize, **kwargs)
else:
with contextlib.suppress(OSError):
return self.cls(["man", cmdq], bufsize=bufsize, **kwargs)
e = f"xonsh: subprocess mode: command not found: {repr(cmd0)}"
env = XSH.env
sug = xt.suggest_commands(cmd0, env)
if len(sug.strip()) > 0:
e += "\n" + xt.suggest_commands(cmd0, env)
if XSH.env.get("XONSH_INTERACTIVE"):
events = XSH.builtins.events
events.on_command_not_found.fire(cmd=self.cmd)
raise xt.XonshError(e) from ex
return p
[docs]
def prep_env_subproc(self, kwargs):
"""Prepares the environment to use in the subprocess."""
with XSH.env.swap(self.env) as env:
denv = env.detype()
if xp.ON_WINDOWS:
# Over write prompt variable as xonsh's $PROMPT does
# not make much sense for other subprocs
denv["PROMPT"] = "$P$G"
kwargs["env"] = denv
[docs]
def prep_preexec_fn(self, kwargs, pipeline_group=None):
"""Prepares the 'preexec_fn' keyword argument"""
if not xp.ON_POSIX:
return
if not XSH.env.get("XONSH_INTERACTIVE"):
return
if pipeline_group is None or xp.ON_WSL1:
# If there is no pipeline group
# or the platform is windows subsystem for linux (WSL)
xonsh_preexec_fn = no_pg_xonsh_preexec_fn
else:
def xonsh_preexec_fn():
"""Preexec function bound to a pipeline group."""
os.setpgid(0, pipeline_group