diff options
Diffstat (limited to 'import-layers/yocto-poky/bitbake/lib/bb/pysh/interp.py')
| -rw-r--r-- | import-layers/yocto-poky/bitbake/lib/bb/pysh/interp.py | 1367 |
1 files changed, 1367 insertions, 0 deletions
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/pysh/interp.py b/import-layers/yocto-poky/bitbake/lib/bb/pysh/interp.py new file mode 100644 index 000000000..25d8c92ec --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/pysh/interp.py @@ -0,0 +1,1367 @@ +# interp.py - shell interpreter for pysh. +# +# Copyright 2007 Patrick Mezard +# +# This software may be used and distributed according to the terms +# of the GNU General Public License, incorporated herein by reference. + +"""Implement the shell interpreter. + +Most references are made to "The Open Group Base Specifications Issue 6". +<http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html> +""" +# TODO: document the fact input streams must implement fileno() so Popen will work correctly. +# it requires non-stdin stream to be implemented as files. Still to be tested... +# DOC: pathsep is used in PATH instead of ':'. Clearly, there are path syntax issues here. +# TODO: stop command execution upon error. +# TODO: sort out the filename/io_number mess. It should be possible to use filenames only. +# TODO: review subshell implementation +# TODO: test environment cloning for non-special builtins +# TODO: set -x should not rebuild commands from tokens, assignments/redirections are lost +# TODO: unit test for variable assignment +# TODO: test error management wrt error type/utility type +# TODO: test for binary output everywhere +# BUG: debug-parsing does not pass log file to PLY. Maybe a PLY upgrade is necessary. +import base64 +import cPickle as pickle +import errno +import glob +import os +import re +import subprocess +import sys +import tempfile + +try: + s = set() + del s +except NameError: + from Set import Set as set + +import builtin +from sherrors import * +import pyshlex +import pyshyacc + +def mappend(func, *args, **kargs): + """Like map but assume func returns a list. Returned lists are merged into + a single one. + """ + return reduce(lambda a,b: a+b, map(func, *args, **kargs), []) + +class FileWrapper: + """File object wrapper to ease debugging. + + Allow mode checking and implement file duplication through a simple + reference counting scheme. Not sure the latter is really useful since + only real file descriptors can be used. + """ + def __init__(self, mode, file, close=True): + if mode not in ('r', 'w', 'a'): + raise IOError('invalid mode: %s' % mode) + self._mode = mode + self._close = close + if isinstance(file, FileWrapper): + if file._refcount[0] <= 0: + raise IOError(0, 'Error') + self._refcount = file._refcount + self._refcount[0] += 1 + self._file = file._file + else: + self._refcount = [1] + self._file = file + + def dup(self): + return FileWrapper(self._mode, self, self._close) + + def fileno(self): + """fileno() should be only necessary for input streams.""" + return self._file.fileno() + + def read(self, size=-1): + if self._mode!='r': + raise IOError(0, 'Error') + return self._file.read(size) + + def readlines(self, *args, **kwargs): + return self._file.readlines(*args, **kwargs) + + def write(self, s): + if self._mode not in ('w', 'a'): + raise IOError(0, 'Error') + return self._file.write(s) + + def flush(self): + self._file.flush() + + def close(self): + if not self._refcount: + return + assert self._refcount[0] > 0 + + self._refcount[0] -= 1 + if self._refcount[0] == 0: + self._mode = 'c' + if self._close: + self._file.close() + self._refcount = None + + def mode(self): + return self._mode + + def __getattr__(self, name): + if name == 'name': + self.name = getattr(self._file, name) + return self.name + else: + raise AttributeError(name) + + def __del__(self): + self.close() + + +def win32_open_devnull(mode): + return open('NUL', mode) + + +class Redirections: + """Stores open files and their mapping to pseudo-sh file descriptor. + """ + # BUG: redirections are not handled correctly: 1>&3 2>&3 3>&4 does + # not make 1 to redirect to 4 + def __init__(self, stdin=None, stdout=None, stderr=None): + self._descriptors = {} + if stdin is not None: + self._add_descriptor(0, stdin) + if stdout is not None: + self._add_descriptor(1, stdout) + if stderr is not None: + self._add_descriptor(2, stderr) + + def add_here_document(self, interp, name, content, io_number=None): + if io_number is None: + io_number = 0 + + if name==pyshlex.unquote_wordtree(name): + content = interp.expand_here_document(('TOKEN', content)) + + # Write document content in a temporary file + tmp = tempfile.TemporaryFile() + try: + tmp.write(content) + tmp.flush() + tmp.seek(0) + self._add_descriptor(io_number, FileWrapper('r', tmp)) + except: + tmp.close() + raise + + def add(self, interp, op, filename, io_number=None): + if op not in ('<', '>', '>|', '>>', '>&'): + # TODO: add descriptor duplication and here_documents + raise RedirectionError('Unsupported redirection operator "%s"' % op) + + if io_number is not None: + io_number = int(io_number) + + if (op == '>&' and filename.isdigit()) or filename=='-': + # No expansion for file descriptors, quote them if you want a filename + fullname = filename + else: + if filename.startswith('/'): + # TODO: win32 kludge + if filename=='/dev/null': + fullname = 'NUL' + else: + # TODO: handle absolute pathnames, they are unlikely to exist on the + # current platform (win32 for instance). + raise NotImplementedError() + else: + fullname = interp.expand_redirection(('TOKEN', filename)) + if not fullname: + raise RedirectionError('%s: ambiguous redirect' % filename) + # Build absolute path based on PWD + fullname = os.path.join(interp.get_env()['PWD'], fullname) + + if op=='<': + return self._add_input_redirection(interp, fullname, io_number) + elif op in ('>', '>|'): + clobber = ('>|'==op) + return self._add_output_redirection(interp, fullname, io_number, clobber) + elif op=='>>': + return self._add_output_appending(interp, fullname, io_number) + elif op=='>&': + return self._dup_output_descriptor(fullname, io_number) + + def close(self): + if self._descriptors is not None: + for desc in self._descriptors.itervalues(): + desc.flush() + desc.close() + self._descriptors = None + + def stdin(self): + return self._descriptors[0] + + def stdout(self): + return self._descriptors[1] + + def stderr(self): + return self._descriptors[2] + + def clone(self): + clone = Redirections() + for desc, fileobj in self._descriptors.iteritems(): + clone._descriptors[desc] = fileobj.dup() + return clone + + def _add_output_redirection(self, interp, filename, io_number, clobber): + if io_number is None: + # io_number default to standard output + io_number = 1 + + if not clobber and interp.get_env().has_opt('-C') and os.path.isfile(filename): + # File already exist in no-clobber mode, bail out + raise RedirectionError('File "%s" already exists' % filename) + + # Open and register + self._add_file_descriptor(io_number, filename, 'w') + + def _add_output_appending(self, interp, filename, io_number): + if io_number is None: + io_number = 1 + self._add_file_descriptor(io_number, filename, 'a') + + def _add_input_redirection(self, interp, filename, io_number): + if io_number is None: + io_number = 0 + self._add_file_descriptor(io_number, filename, 'r') + + def _add_file_descriptor(self, io_number, filename, mode): + try: + if filename.startswith('/'): + if filename=='/dev/null': + f = win32_open_devnull(mode+'b') + else: + # TODO: handle absolute pathnames, they are unlikely to exist on the + # current platform (win32 for instance). + raise NotImplementedError('cannot open absolute path %s' % repr(filename)) + else: + f = file(filename, mode+'b') + except IOError as e: + raise RedirectionError(str(e)) + + wrapper = None + try: + wrapper = FileWrapper(mode, f) + f = None + self._add_descriptor(io_number, wrapper) + except: + if f: f.close() + if wrapper: wrapper.close() + raise + + def _dup_output_descriptor(self, source_fd, dest_fd): + if source_fd is None: + source_fd = 1 + self._dup_file_descriptor(source_fd, dest_fd, 'w') + + def _dup_file_descriptor(self, source_fd, dest_fd, mode): + source_fd = int(source_fd) + if source_fd not in self._descriptors: + raise RedirectionError('"%s" is not a valid file descriptor' % str(source_fd)) + source = self._descriptors[source_fd] + + if source.mode()!=mode: + raise RedirectionError('Descriptor %s cannot be duplicated in mode "%s"' % (str(source), mode)) + + if dest_fd=='-': + # Close the source descriptor + del self._descriptors[source_fd] + source.close() + else: + dest_fd = int(dest_fd) + if dest_fd not in self._descriptors: + raise RedirectionError('Cannot replace file descriptor %s' % str(dest_fd)) + + dest = self._descriptors[dest_fd] + if dest.mode()!=mode: + raise RedirectionError('Descriptor %s cannot be cannot be redirected in mode "%s"' % (str(dest), mode)) + + self._descriptors[dest_fd] = source.dup() + dest.close() + + def _add_descriptor(self, io_number, file): + io_number = int(io_number) + + if io_number in self._descriptors: + # Close the current descriptor + d = self._descriptors[io_number] + del self._descriptors[io_number] + d.close() + + self._descriptors[io_number] = file + + def __str__(self): + names = [('%d=%r' % (k, getattr(v, 'name', None))) for k,v + in self._descriptors.iteritems()] + names = ','.join(names) + return 'Redirections(%s)' % names + + def __del__(self): + self.close() + +def cygwin_to_windows_path(path): + """Turn /cygdrive/c/foo into c:/foo, or return path if it + is not a cygwin path. + """ + if not path.startswith('/cygdrive/'): + return path + path = path[len('/cygdrive/'):] + path = path[:1] + ':' + path[1:] + return path + +def win32_to_unix_path(path): + if path is not None: + path = path.replace('\\', '/') + return path + +_RE_SHEBANG = re.compile(r'^\#!\s?([^\s]+)(?:\s([^\s]+))?') +_SHEBANG_CMDS = { + '/usr/bin/env': 'env', + '/bin/sh': 'pysh', + 'python': 'python', +} + +def resolve_shebang(path, ignoreshell=False): + """Return a list of arguments as shebang interpreter call or an empty list + if path does not refer to an executable script. + See <http://www.opengroup.org/austin/docs/austin_51r2.txt>. + + ignoreshell - set to True to ignore sh shebangs. Return an empty list instead. + """ + try: + f = file(path) + try: + # At most 80 characters in the first line + header = f.read(80).splitlines()[0] + finally: + f.close() + + m = _RE_SHEBANG.search(header) + if not m: + return [] + cmd, arg = m.group(1,2) + if os.path.isfile(cmd): + # Keep this one, the hg script for instance contains a weird windows + # shebang referencing the current python install. + cmdfile = os.path.basename(cmd).lower() + if cmdfile == 'python.exe': + cmd = 'python' + pass + elif cmd not in _SHEBANG_CMDS: + raise CommandNotFound('Unknown interpreter "%s" referenced in '\ + 'shebang' % header) + cmd = _SHEBANG_CMDS.get(cmd) + if cmd is None or (ignoreshell and cmd == 'pysh'): + return [] + if arg is None: + return [cmd, win32_to_unix_path(path)] + return [cmd, arg, win32_to_unix_path(path)] + except IOError as e: + if e.errno!=errno.ENOENT and \ + (e.errno!=errno.EPERM and not os.path.isdir(path)): # Opening a directory raises EPERM + raise + return [] + +def win32_find_in_path(name, path): + if isinstance(path, str): + path = path.split(os.pathsep) + + exts = os.environ.get('PATHEXT', '').lower().split(os.pathsep) + for p in path: + p_name = os.path.join(p, name) + + prefix = resolve_shebang(p_name) + if prefix: + return prefix + + for ext in exts: + p_name_ext = p_name + ext + if os.path.exists(p_name_ext): + return [win32_to_unix_path(p_name_ext)] + return [] + +class Traps(dict): + def __setitem__(self, key, value): + if key not in ('EXIT',): + raise NotImplementedError() + super(Traps, self).__setitem__(key, value) + +# IFS white spaces character class +_IFS_WHITESPACES = (' ', '\t', '\n') + +class Environment: + """Environment holds environment variables, export table, function + definitions and whatever is defined in 2.12 "Shell Execution Environment", + redirection excepted. + """ + def __init__(self, pwd): + self._opt = set() #Shell options + + self._functions = {} + self._env = {'?': '0', '#': '0'} + self._exported = set([ + 'HOME', 'IFS', 'PATH' + ]) + + # Set environment vars with side-effects + self._ifs_ws = None # Set of IFS whitespace characters + self._ifs_re = None # Regular expression used to split between words using IFS classes + self['IFS'] = ''.join(_IFS_WHITESPACES) #Default environment values + self['PWD'] = pwd + self.traps = Traps() + + def clone(self, subshell=False): + env = Environment(self['PWD']) + env._opt = set(self._opt) + for k,v in self.get_variables().iteritems(): + if k in self._exported: + env.export(k,v) + elif subshell: + env[k] = v + + if subshell: + env._functions = dict(self._functions) + + return env + + def __getitem__(self, key): + if key in ('@', '*', '-', '$'): + raise NotImplementedError('%s is not implemented' % repr(key)) + return self._env[key] + + def get(self, key, defval=None): + try: + return self[key] + except KeyError: + return defval + + def __setitem__(self, key, value): + if key=='IFS': + # Update the whitespace/non-whitespace classes + self._update_ifs(value) + elif key=='PWD': + pwd = os.path.abspath(value) + if not os.path.isdir(pwd): + raise VarAssignmentError('Invalid directory %s' % value) + value = pwd + elif key in ('?', '!'): + value = str(int(value)) + self._env[key] = value + + def __delitem__(self, key): + if key in ('IFS', 'PWD', '?'): + raise VarAssignmentError('%s cannot be unset' % key) + del self._env[key] + + def __contains__(self, item): + return item in self._env + + def set_positional_args(self, args): + """Set the content of 'args' as positional argument from 1 to len(args). + Return previous argument as a list of strings. + """ + # Save and remove previous arguments + prevargs = [] + for i in xrange(int(self._env['#'])): + i = str(i+1) + prevargs.append(self._env[i]) + del self._env[i] + self._env['#'] = '0' + + #Set new ones + for i,arg in enumerate(args): + self._env[str(i+1)] = str(arg) + self._env['#'] = str(len(args)) + + return prevargs + + def get_positional_args(self): + return [self._env[str(i+1)] for i in xrange(int(self._env['#']))] + + def get_variables(self): + return dict(self._env) + + def export(self, key, value=None): + if value is not None: + self[key] = value + self._exported.add(key) + + def get_exported(self): + return [(k,self._env.get(k)) for k in self._exported] + + def split_fields(self, word): + if not self._ifs_ws or not word: + return [word] + return re.split(self._ifs_re, word) + + def _update_ifs(self, value): + """Update the split_fields related variables when IFS character set is + changed. + """ + # TODO: handle NULL IFS + + # Separate characters in whitespace and non-whitespace + chars = set(value) + ws = [c for c in chars if c in _IFS_WHITESPACES] + nws = [c for c in chars if c not in _IFS_WHITESPACES] + + # Keep whitespaces in a string for left and right stripping + self._ifs_ws = ''.join(ws) + + # Build a regexp to split fields + trailing = '[' + ''.join([re.escape(c) for c in ws]) + ']' + if nws: + # First, the single non-whitespace occurence. + nws = '[' + ''.join([re.escape(c) for c in nws]) + ']' + nws = '(?:' + trailing + '*' + nws + trailing + '*' + '|' + trailing + '+)' + else: + # Then mix all parts with quantifiers + nws = trailing + '+' + self._ifs_re = re.compile(nws) + + def has_opt(self, opt, val=None): + return (opt, val) in self._opt + + def set_opt(self, opt, val=None): + self._opt.add((opt, val)) + + def find_in_path(self, name, pwd=False): + path = self._env.get('PATH', '').split(os.pathsep) + if pwd: + path[:0] = [self['PWD']] + if os.name == 'nt': + return win32_find_in_path(name, self._env.get('PATH', '')) + else: + raise NotImplementedError() + + def define_function(self, name, body): + if not is_name(name): + raise ShellSyntaxError('%s is not a valid function name' % repr(name)) + self._functions[name] = body + + def remove_function(self, name): + del self._functions[name] + + def is_function(self, name): + return name in self._functions + + def get_function(self, name): + return self._functions.get(name) + + +name_charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_' +name_charset = dict(zip(name_charset,name_charset)) + +def match_name(s): + """Return the length in characters of the longest prefix made of name + allowed characters in s. + """ + for i,c in enumerate(s): + if c not in name_charset: + return s[:i] + return s + +def is_name(s): + return len([c for c in s if c not in name_charset])<=0 + +def is_special_param(c): + return len(c)==1 and c in ('@','*','#','?','-','$','!','0') + +def utility_not_implemented(name, *args, **kwargs): + raise NotImplementedError('%s utility is not implemented' % name) + + +class Utility: + """Define utilities properties: + func -- utility callable. See builtin module for utility samples. + is_special -- see XCU 2.8. + """ + def __init__(self, func, is_special=0): + self.func = func + self.is_special = bool(is_special) + + +def encodeargs(args): + def encodearg(s): + lines = base64.encodestring(s) + lines = [l.splitlines()[0] for l in lines] + return ''.join(lines) + + s = pickle.dumps(args) + return encodearg(s) + +def decodeargs(s): + s = base64.decodestring(s) + return pickle.loads(s) + + +class GlobError(Exception): + pass + +class Options: + def __init__(self): + # True if Mercurial operates with binary streams + self.hgbinary = True + +class Interpreter: + # Implementation is very basic: the execute() method just makes a DFS on the + # AST and execute nodes one by one. Nodes are tuple (name,obj) where name + # is a string identifier and obj the AST element returned by the parser. + # + # Handler are named after the node identifiers. + # TODO: check node names and remove the switch in execute with some + # dynamic getattr() call to find node handlers. + """Shell interpreter. + + The following debugging flags can be passed: + debug-parsing - enable PLY debugging. + debug-tree - print the generated AST. + debug-cmd - trace command execution before word expansion, plus exit status. + debug-utility - trace utility execution. + """ + + # List supported commands. + COMMANDS = { + 'cat': Utility(builtin.utility_cat,), + 'cd': Utility(builtin.utility_cd,), + ':': Utility(builtin.utility_colon,), + 'echo': Utility(builtin.utility_echo), + 'env': Utility(builtin.utility_env), + 'exit': Utility(builtin.utility_exit), + 'export': Utility(builtin.builtin_export, is_special=1), + 'egrep': Utility(builtin.utility_egrep), + 'fgrep': Utility(builtin.utility_fgrep), + 'gunzip': Utility(builtin.utility_gunzip), + 'kill': Utility(builtin.utility_kill), + 'mkdir': Utility(builtin.utility_mkdir), + 'netstat': Utility(builtin.utility_netstat), + 'printf': Utility(builtin.utility_printf), + 'pwd': Utility(builtin.utility_pwd), + 'return': Utility(builtin.builtin_return, is_special=1), + 'sed': Utility(builtin.utility_sed,), + 'set': Utility(builtin.builtin_set,), + 'shift': Utility(builtin.builtin_shift,), + 'sleep': Utility(builtin.utility_sleep,), + 'sort': Utility(builtin.utility_sort,), + 'trap': Utility(builtin.builtin_trap, is_special=1), + 'true': Utility(builtin.utility_true), + 'unset': Utility(builtin.builtin_unset, is_special=1), + 'wait': Utility(builtin.builtin_wait, is_special=1), + } + + def __init__(self, pwd, debugflags = [], env=None, redirs=None, stdin=None, + stdout=None, stderr=None, opts=Options()): + self._env = env + if self._env is None: + self._env = Environment(pwd) + self._children = {} + + self._redirs = redirs + self._close_redirs = False + + if self._redirs is None: + if stdin is None: + stdin = sys.stdin + if stdout is None: + stdout = sys.stdout + if stderr is None: + stderr = sys.stderr + stdin = FileWrapper('r', stdin, False) + stdout = FileWrapper('w', stdout, False) + stderr = FileWrapper('w', stderr, False) + self._redirs = Redirections(stdin, stdout, stderr) + self._close_redirs = True + + self._debugflags = list(debugflags) + self._logfile = sys.stderr + self._options = opts + + def close(self): + """Must be called when the interpreter is no longer used.""" + script = self._env.traps.get('EXIT') + if script: + try: + self.execute_script(script=script) + except: + pass + + if self._redirs is not None and self._close_redirs: + self._redirs.close() + self._redirs = None + + def log(self, s): + self._logfile.write(s) + self._logfile.flush() + + def __getitem__(self, key): + return self._env[key] + + def __setitem__(self, key, value): + self._env[key] = value + + def options(self): + return self._options + + def redirect(self, redirs, ios): + def add_redir(io): + if isinstance(io, pyshyacc.IORedirect): + redirs.add(self, io.op, io.filename, io.io_number) + else: + redirs.add_here_document(self, io.name, io.content, io.io_number) + + map(add_redir, ios) + return redirs + + def execute_script(self, script=None, ast=None, sourced=False, + scriptpath=None): + """If script is not None, parse the input. Otherwise takes the supplied + AST. Then execute the AST. + Return the script exit status. + """ + try: + if scriptpath is not None: + self._env['0'] = os.path.abspath(scriptpath) + + if script is not None: + debug_parsing = ('debug-parsing' in self._debugflags) + cmds, script = pyshyacc.parse(script, True, debug_parsing) + if 'debug-tree' in self._debugflags: + pyshyacc.print_commands(cmds, self._logfile) + self._logfile.flush() + else: + cmds, script = ast, '' + + status = 0 + for cmd in cmds: + try: + status = self.execute(cmd) + except ExitSignal as e: + if sourced: + raise + status = int(e.args[0]) + return status + except ShellError: + self._env['?'] = 1 + raise + if 'debug-utility' in self._debugflags or 'debug-cmd' in self._debugflags: + self.log('returncode ' + str(status)+ '\n') + return status + except CommandNotFound as e: + print >>self._redirs.stderr, str(e) + self._redirs.stderr.flush() + # Command not found by non-interactive shell + # return 127 + raise + except RedirectionError as e: + # TODO: should be handled depending on the utility status + print >>self._redirs.stderr, str(e) + self._redirs.stderr.flush() + # Command not found by non-interactive shell + # return 127 + raise + + def dotcommand(self, env, args): + if len(args) < 1: + raise ShellError('. expects at least one argument') + path = args[0] + if '/' not in path: + found = env.find_in_path(args[0], True) + if found: + path = found[0] + script = file(path).read() + return self.execute_script(script=script, sourced=True) + + def execute(self, token, redirs=None): + """Execute and AST subtree with supplied redirections overriding default + interpreter ones. + Return the exit status. + """ + if not token: + return 0 + + if redirs is None: + redirs = self._redirs + + if isinstance(token, list): + # Commands sequence + res = 0 + for t in token: + res = self.execute(t, redirs) + return res + + type, value = token + status = 0 + if type=='simple_command': + redirs_copy = redirs.clone() + try: + # TODO: define and handle command return values + # TODO: implement set -e + status = self._execute_simple_command(value, redirs_copy) + finally: + redirs_copy.close() + elif type=='pipeline': + status = self._execute_pipeline(value, redirs) + elif type=='and_or': + status = self._execute_and_or(value, redirs) + elif type=='for_clause': + status = self._execute_for_clause(value, redirs) + elif type=='while_clause': + status = self._execute_while_clause(value, redirs) + elif type=='function_definition': + status = self._execute_function_definition(value, redirs) + elif type=='brace_group': + status = self._execute_brace_group(value, redirs) + elif type=='if_clause': + status = self._execute_if_clause(value, redirs) + elif type=='subshell': + status = self.subshell(ast=value.cmds, redirs=redirs) + elif type=='async': + status = self._asynclist(value) + elif type=='redirect_list': + redirs_copy = self.redirect(redirs.clone(), value.redirs) + try: + status = self.execute(value.cmd, redirs_copy) + finally: + redirs_copy.close() + else: + raise NotImplementedError('Unsupported token type ' + type) + + if status < 0: + status = 255 + return status + + def _execute_if_clause(self, if_clause, redirs): + cond_status = self.execute(if_clause.cond, redirs) + if cond_status==0: + return self.execute(if_clause.if_cmds, redirs) + else: + return self.execute(if_clause.else_cmds, redirs) + + def _execute_brace_group(self, group, redirs): + status = 0 + for cmd in group.cmds: + status = self.execute(cmd, redirs) + return status + + def _execute_function_definition(self, fundef, redirs): + self._env.define_function(fundef.name, fundef.body) + return 0 + + def _execute_while_clause(self, while_clause, redirs): + status = 0 + while 1: + cond_status = 0 + for cond in while_clause.condition: + cond_status = self.execute(cond, redirs) + + if cond_status: + break + + for cmd in while_clause.cmds: + status = self.execute(cmd, redirs) + + return status + + def _execute_for_clause(self, for_clause, redirs): + if not is_name(for_clause.name): + raise ShellSyntaxError('%s is not a valid name' % repr(for_clause.name)) + items = mappend(self.expand_token, for_clause.items) + + status = 0 + for item in items: + self._env[for_clause.name] = item + for cmd in for_clause.cmds: + status = self.execute(cmd, redirs) + return status + + def _execute_and_or(self, or_and, redirs): + res = self.execute(or_and.left, redirs) + if (or_and.op=='&&' and res==0) or (or_and.op!='&&' and res!=0): + res = self.execute(or_and.right, redirs) + return res + + def _execute_pipeline(self, pipeline, redirs): + if len(pipeline.commands)==1: + status = self.execute(pipeline.commands[0], redirs) + else: + # Execute all commands one after the other + status = 0 + inpath, outpath = None, None + try: + # Commands inputs and outputs cannot really be plugged as done + # by a real shell. Run commands sequentially and chain their + # input/output throught temporary files. + tmpfd, inpath = tempfile.mkstemp() + os.close(tmpfd) + tmpfd, outpath = tempfile.mkstemp() + os.close(tmpfd) + + inpath = win32_to_unix_path(inpath) + outpath = win32_to_unix_path(outpath) + + for i, cmd in enumerate(pipeline.commands): + call_redirs = redirs.clone() + try: + if i!=0: + call_redirs.add(self, '<', inpath) + if i!=len(pipeline.commands)-1: + call_redirs.add(self, '>', outpath) + + status = self.execute(cmd, call_redirs) + + # Chain inputs/outputs + inpath, outpath = outpath, inpath + finally: + call_redirs.close() + finally: + if inpath: os.remove(inpath) + if outpath: os.remove(outpath) + + if pipeline.reverse_status: + status = int(not status) + self._env['?'] = status + return status + + def _execute_function(self, name, args, interp, env, stdin, stdout, stderr, *others): + assert interp is self + + func = env.get_function(name) + #Set positional parameters + prevargs = None + try: + prevargs = env.set_positional_args(args) + try: + redirs = Redirections(stdin.dup(), stdout.dup(), stderr.dup()) + try: + status = self.execute(func, redirs) + finally: + redirs.close() + except ReturnSignal as e: + status = int(e.args[0]) + env['?'] = status + return status + finally: + #Reset positional parameters + if prevargs is not None: + env.set_positional_args(prevargs) + + def _execute_simple_command(self, token, redirs): + """Can raise ReturnSignal when return builtin is called, ExitSignal when + exit is called, and other shell exceptions upon builtin failures. + """ + debug_command = 'debug-cmd' in self._debugflags + if debug_command: + self.log('word' + repr(token.words) + '\n') + self.log('assigns' + repr(token.assigns) + '\n') + self.log('redirs' + repr(token.redirs) + '\n') + + is_special = None + env = self._env + + try: + # Word expansion + args = [] + for word in token.words: + args += self.expand_token(word) + if is_special is None and args: + is_special = env.is_function(args[0]) or \ + (args[0] in self.COMMANDS and self.COMMANDS[args[0]].is_special) + + if debug_command: + self.log('_execute_simple_command' + str(args) + '\n') + + if not args: + # Redirections happen is a subshell + redirs = redirs.clone() + elif not is_special: + env = self._env.clone() + + # Redirections + self.redirect(redirs, token.redirs) + + # Variables assignments + res = 0 + for type,(k,v) in token.assigns: + status, expanded = self.expand_variable((k,v)) + if status is not None: + res = status + if args: + env.export(k, expanded) + else: + env[k] = expanded + + if args and args[0] in ('.', 'source'): + res = self.dotcommand(env, args[1:]) + elif args: + if args[0] in self.COMMANDS: + command = self.COMMANDS[args[0]] + elif env.is_function(args[0]): + command = Utility(self._execute_function, is_special=True) + else: + if not '/' in args[0].replace('\\', '/'): + cmd = env.find_in_path(args[0]) + if not cmd: + # TODO: test error code on unknown command => 127 + raise CommandNotFound('Unknown command: "%s"' % args[0]) + else: + # Handle commands like '/cygdrive/c/foo.bat' + cmd = cygwin_to_windows_path(args[0]) + if not os.path.exists(cmd): + raise CommandNotFound('%s: No such file or directory' % args[0]) + shebang = resolve_shebang(cmd) + if shebang: + cmd = shebang + else: + cmd = [cmd] + args[0:1] = cmd + command = Utility(builtin.run_command) + + # Command execution + if 'debug-cmd' in self._debugflags: + self.log('redirections ' + str(redirs) + '\n') + + res = command.func(args[0], args[1:], self, env, + redirs.stdin(), redirs.stdout(), + redirs.stderr(), self._debugflags) + + if self._env.has_opt('-x'): + # Trace command execution in shell environment + # BUG: would be hard to reproduce a real shell behaviour since + # the AST is not annotated with source lines/tokens. + self._redirs.stdout().write(' '.join(args)) + + except ReturnSignal: + raise + except ShellError as e: + if is_special or isinstance(e, (ExitSignal, + ShellSyntaxError, ExpansionError)): + raise e + self._redirs.stderr().write(str(e)+'\n') + return 1 + + return res + + def expand_token(self, word): + """Expand a word as specified in [2.6 Word Expansions]. Return the list + of expanded words. + """ + status, wtrees = self._expand_word(word) + return map(pyshlex.wordtree_as_string, wtrees) + + def expand_variable(self, word): + """Return a status code (or None if no command expansion occurred) + and a single word. + """ + status, wtrees = self._expand_word(word, pathname=False, split=False) + words = map(pyshlex.wordtree_as_string, wtrees) + assert len(words)==1 + return status, words[0] + + def expand_here_document(self, word): + """Return the expanded document as a single word. The here document is + assumed to be unquoted. + """ + status, wtrees = self._expand_word(word, pathname=False, + split=False, here_document=True) + words = map(pyshlex.wordtree_as_string, wtrees) + assert len(words)==1 + return words[0] + + def expand_redirection(self, word): + """Return a single word.""" + return self.expand_variable(word)[1] + + def get_env(self): + return self._env + + def _expand_word(self, token, pathname=True, split=True, here_document=False): + wtree = pyshlex.make_wordtree(token[1], here_document=here_document) + + # TODO: implement tilde expansion + def expand(wtree): + """Return a pseudo wordtree: the tree or its subelements can be empty + lists when no value result from the expansion. + """ + status = None + for part in wtree: + if not isinstance(part, list): + continue + if part[0]in ("'", '\\'): + continue + elif part[0] in ('`', '$('): + status, result = self._expand_command(part) + part[:] = result + elif part[0] in ('$', '${'): + part[:] = self._expand_parameter(part, wtree[0]=='"', split) + elif part[0] in ('', '"'): + status, result = expand(part) + part[:] = result + else: + raise NotImplementedError('%s expansion is not implemented' + % part[0]) + # [] is returned when an expansion result in no-field, + # like an empty $@ + wtree = [p for p in wtree if p != []] + if len(wtree) < 3: + return status, [] + return status, wtree + + status, wtree = expand(wtree) + if len(wtree) == 0: + return status, wtree + wtree = pyshlex.normalize_wordtree(wtree) + + if split: + wtrees = self._split_fields(wtree) + else: + wtrees = [wtree] + + if pathname: + wtrees = mappend(self._expand_pathname, wtrees) + + wtrees = map(self._remove_quotes, wtrees) + return status, wtrees + + def _expand_command(self, wtree): + # BUG: there is something to do with backslashes and quoted + # characters here + command = pyshlex.wordtree_as_string(wtree[1:-1]) + status, output = self.subshell_output(command) + return status, ['', output, ''] + + def _expand_parameter(self, wtree, quoted=False, split=False): + """Return a valid wtree or an empty list when no parameter results.""" + # Get the parameter name + # TODO: implement weird expansion rules with ':' + name = pyshlex.wordtree_as_string(wtree[1:-1]) + if not is_name(name) and not is_special_param(name): + raise ExpansionError('Bad substitution "%s"' % name) + # TODO: implement special parameters + if name in ('@', '*'): + args = self._env.get_positional_args() + if len(args) == 0: + return [] + if len(args)<2: + return ['', ''.join(args), ''] + + sep = self._env.get('IFS', '')[:1] + if split and quoted and name=='@': + # Introduce a new token to tell the caller that these parameters + # cause a split as specified in 2.5.2 + return ['@'] + args + [''] + else: + return ['', sep.join(args), ''] + + return ['', self._env.get(name, ''), ''] + + def _split_fields(self, wtree): + def is_empty(split): + return split==['', '', ''] + + def split_positional(quoted): + # Return a list of wtree split according positional parameters rules. + # All remaining '@' groups are removed. + assert quoted[0]=='"' + + splits = [[]] + for part in quoted: + if not isinstance(part, list) or part[0]!='@': + splits[-1].append(part) + else: + # Empty or single argument list were dealt with already + assert len(part)>3 + # First argument must join with the beginning part of the original word + splits[-1].append(part[1]) + # Create double-quotes expressions for every argument after the first + for arg in part[2:-1]: + splits[-1].append('"') + splits.append(['"', arg]) + return splits + + # At this point, all expansions but pathnames have occured. Only quoted + # and positional sequences remain. Thus, all candidates for field splitting + # are in the tree root, or are positional splits ('@') and lie in root + # children. + if not wtree or wtree[0] not in ('', '"'): + # The whole token is quoted or empty, nothing to split + return [wtree] + + if wtree[0]=='"': + wtree = ['', wtree, ''] + + result = [['', '']] + for part in wtree[1:-1]: + if isinstance(part, list): + if part[0]=='"': + splits = split_positional(part) + if len(splits)<=1: + result[-1] += [part, ''] + else: + # Terminate the current split + result[-1] += [splits[0], ''] + result += splits[1:-1] + # Create a new split + result += [['', splits[-1], '']] + else: + result[-1] += [part, ''] + else: + splits = self._env.split_fields(part) + if len(splits)<=1: + # No split + result[-1][-1] += part + else: + # Terminate the current resulting part and create a new one + result[-1][-1] += splits[0] + result[-1].append('') + result += [['', r, ''] for r in splits[1:-1]] + result += [['', splits[-1]]] + result[-1].append('') + + # Leading and trailing empty groups come from leading/trailing blanks + if result and is_empty(result[-1]): + result[-1:] = [] + if result and is_empty(result[0]): + result[:1] = [] + return result + + def _expand_pathname(self, wtree): + """See [2.6.6 Pathname Expansion].""" + if self._env.has_opt('-f'): + return [wtree] + + # All expansions have been performed, only quoted sequences should remain + # in the tree. Generate the pattern by folding the tree, escaping special + # characters when appear quoted + special_chars = '*?[]' + + def make_pattern(wtree): + subpattern = [] + for part in wtree[1:-1]: + if isinstance(part, list): + part = make_pattern(part) + elif wtree[0]!='': + for c in part: + # Meta-characters cannot be quoted + if c in special_chars: + raise GlobError() + subpattern.append(part) + return ''.join(subpattern) + + def pwd_glob(pattern): + cwd = os.getcwd() + os.chdir(self._env['PWD']) + try: + return glob.glob(pattern) + finally: + os.chdir(cwd) + + #TODO: check working directory issues here wrt relative patterns + try: + pattern = make_pattern(wtree) + paths = pwd_glob(pattern) + except GlobError: + # BUG: Meta-characters were found in quoted sequences. The should + # have been used literally but this is unsupported in current glob module. + # Instead we consider the whole tree must be used literally and + # therefore there is no point in globbing. This is wrong when meta + # characters are mixed with quoted meta in the same pattern like: + # < foo*"py*" > + paths = [] + + if not paths: + return [wtree] + return [['', path, ''] for path in paths] + + def _remove_quotes(self, wtree): + """See [2.6.7 Quote Removal].""" + + def unquote(wtree): + unquoted = [] + for part in wtree[1:-1]: + if isinstance(part, list): + part = unquote(part) + unquoted.append(part) + return ''.join(unquoted) + + return ['', unquote(wtree), ''] + + def subshell(self, script=None, ast=None, redirs=None): + """Execute the script or AST in a subshell, with inherited redirections + if redirs is not None. + """ + if redirs: + sub_redirs = redirs + else: + sub_redirs = redirs.clone() + + subshell = None + try: + subshell = Interpreter(None, self._debugflags, self._env.clone(True), + sub_redirs, opts=self._options) + return subshell.execute_script(script, ast) + finally: + if not redirs: sub_redirs.close() + if subshell: subshell.close() + + def subshell_output(self, script): + """Execute the script in a subshell and return the captured output.""" + # Create temporary file to capture subshell output + tmpfd, tmppath = tempfile.mkstemp() + try: + tmpfile = os.fdopen(tmpfd, 'wb') + stdout = FileWrapper('w', tmpfile) + + redirs = Redirections(self._redirs.stdin().dup(), + stdout, + self._redirs.stderr().dup()) + try: + status = self.subshell(script=script, redirs=redirs) + finally: + redirs.close() + redirs = None + + # Extract subshell standard output + tmpfile = open(tmppath, 'rb') + try: + output = tmpfile.read() + return status, output.rstrip('\n') + finally: + tmpfile.close() + finally: + os.remove(tmppath) + + def _asynclist(self, cmd): + args = (self._env.get_variables(), cmd) + arg = encodeargs(args) + assert len(args) < 30*1024 + cmd = ['pysh.bat', '--ast', '-c', arg] + p = subprocess.Popen(cmd, cwd=self._env['PWD']) + self._children[p.pid] = p + self._env['!'] = p.pid + return 0 + + def wait(self, pids=None): + if not pids: + pids = self._children.keys() + + status = 127 + for pid in pids: + if pid not in self._children: + continue + p = self._children.pop(pid) + status = p.wait() + + return status + |
