diff --git a/.gitignore b/.gitignore
index 48488751f453b7dc24fdc1bec38a83868b54d996..4bc4d7fc6fb656527f598779e544316ef09e44e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,5 @@ dist/
*.log
.venv*/
.tox/
+/coverage/
+/.eggs/
diff --git a/.gitreview b/.gitreview
deleted file mode 100644
index c882297be20922874d66a28df3437eefd06d2875..0000000000000000000000000000000000000000
--- a/.gitreview
+++ /dev/null
@@ -1,8 +0,0 @@
-[gerrit]
-host=gerrit.coast-project.org
-port=29418
-project=sconsider.git
-defaultbranch=master
-defaultremote=origin
-defaultrebase=1
-
diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc
index c3d5982324e598545f0a7b1ce3f990b93fd16554..d521aaafd01a1d10eb437a7c0eb0e2662363d3fc 100644
--- a/CHANGELOG.adoc
+++ b/CHANGELOG.adoc
@@ -1,7 +1,94 @@
= Changelog
+== tag: 0.3.17
+* Bumped version to 0.3.17 (91de657)
+
+
+* Worked heavily on how to run and control backend processes (bfac068)
+
+
+* expand search directories for source files (90512af)
+
+
+* make it clear that timeout is a float value (2377aff)
+
+
+* fixed runTimeout setting from command line (6d0faf4)
+
+
+* fixed spurious temp file close errors (ae5461d)
+
+
+* handover signals to started process using exec (9876a1b)
+
+
+* extended RunBuilder to use a timeout for the started subprocess (fdef451)
+
+
+* collecting stderr along with stdout messages in RunBuilder (4019dbb)
+
+
+* terminating subprocess before setting internal termination flag (1fd70ab)
+
+
+* using explicit seconds_to_wait for ProcessRunner (958d436)
+
+
+* replaced all PopenHelper classes with PreocessRunner (6a14556)
+
+
+* exchanged PopenHelper with ProcessRunner (7c0a463)
+
+
+* logging test duration (fbfd0b6)
+
+
+* separated stderr from stdout in ProcessRunner (17e9721)
+
+
+* tests added to test basic functionality of ProcessRunner (eba0229)
+
+
+* renamed property to allow easier PopenHelper replacement (9d92db1)
+
+
+* remove writer to close from witers list prior to closing (1276655)
+
+
+* replaced PopenHelper in RunBuilder with ProcessRunner (9284dc2)
+
+
+* new ProcessRunner to replace PopenHelper class (8567d46)
+
+
+* removed unused code sections (b3c6148)
+
+
+* moved Tee to PopenHelper (3a2e58f)
+
+
+* reworked Tee class to be more flexible (2085fdc)
+
+
+* fixed incorrect return code (fb3e09e)
+
+
+* extended Popen logging to show return code (a9640c7)
+
+
+* added xml file header (1f1e4fe)
+
+
+* removed gitreview file (e225f08)
+
+
+* Merge tag '0.3.16' into develop (38805d0)
+
== tag: 0.3.16
-* updated changelog
+* fixed DoxygenBuilder by using correct array by reference modification (44b2d9f)
+
+
+* updated changelog (cec2fc5)
* Bumped version to 0.3.16 (718be28)
diff --git a/SConsider/LibFinder.py b/SConsider/LibFinder.py
index e0177f7fe9a9554c7911e46c00f4f1ce833499b8..1e22d85405ed966f3f3758b9db04166819d38dda 100644
--- a/SConsider/LibFinder.py
+++ b/SConsider/LibFinder.py
@@ -19,7 +19,7 @@ import functools
import itertools
import operator
from SConsider.SomeUtils import getFlatENV
-from SConsider.PopenHelper import PopenHelper, PIPE
+from SConsider.PopenHelper import PopenHelper, ProcessRunner
def uniquelist(iterable):
@@ -71,15 +71,17 @@ class UnixFinder(LibFinder):
def getLibs(self, env, source, libnames=None, libdirs=None):
if libdirs:
env.AppendENVPath('LD_LIBRARY_PATH', [self.absolutify(j) for j in libdirs])
- ldd = PopenHelper(['ldd', os.path.basename(source[0].get_abspath())],
- stdout=PIPE,
- cwd=os.path.dirname(source[0].get_abspath()),
- env=getFlatENV(env))
- out, _ = ldd.communicate()
- libs = [
- j for j in re.findall(r'^.*=>\s*(not found|[^\s^\(]+)', out, re.MULTILINE)
- if functools.partial(operator.ne, 'not found')(j)
- ]
+ libs = []
+ cmd = ['ldd', os.path.basename(source[0].get_abspath())]
+ with ProcessRunner(cmd,
+ timeout=30,
+ seconds_to_wait=0.1,
+ cwd=os.path.dirname(source[0].get_abspath()),
+ env=getFlatENV(env)) as executor:
+ for out, _ in executor:
+ for j in re.findall(r'^.*=>\s*(not found|[^\s^\(]+)', out, re.MULTILINE):
+ if functools.partial(operator.ne, 'not found')(j):
+ libs.append(j)
if libnames:
libs = [j for j in libs if functools.partial(self.__filterLibs, env, libnames=libnames)(j)]
return libs
@@ -89,16 +91,16 @@ class UnixFinder(LibFinder):
linkercmd = env.subst('$LINK')
if not linkercmd:
return libdirs
- cmdargs = [linkercmd, '-print-search-dirs'] + env.subst('$LINKFLAGS').split(' ')
- linker = PopenHelper(cmdargs, stdout=PIPE, env=getFlatENV(env))
- out, _ = linker.communicate()
- match = re.search('^libraries.*=(.*)$', out, re.MULTILINE)
- if match:
- libdirs.extend(
- unique([
- os.path.abspath(j) for j in match.group(1).split(os.pathsep)
- if os.path.exists(os.path.abspath(j))
- ]))
+ cmd = [linkercmd, '-print-search-dirs'] + env.subst('$LINKFLAGS').split(' ')
+ with ProcessRunner(cmd, timeout=30, env=getFlatENV(env)) as executor:
+ for out, _ in executor:
+ match = re.search('^libraries.*=(.*)$', out, re.MULTILINE)
+ if match:
+ libdirs.extend(
+ unique([
+ os.path.abspath(j) for j in match.group(1).split(os.pathsep)
+ if os.path.exists(os.path.abspath(j))
+ ]))
return libdirs
@@ -120,15 +122,19 @@ class MacFinder(LibFinder):
def getLibs(self, env, source, libnames=None, libdirs=None):
if libdirs:
env.AppendENVPath('DYLD_LIBRARY_PATH', [self.absolutify(j) for j in libdirs])
- ldd = PopenHelper(['otool', '-L', os.path.basename(source[0].get_abspath())],
- stdout=PIPE,
- cwd=os.path.dirname(source[0].get_abspath()),
- env=getFlatENV(env))
- out, _ = ldd.communicate()
- libs = [
- j for j in re.findall(r'^.*=>\s*(not found|[^\s^\(]+)', out, re.MULTILINE)
- if functools.partial(operator.ne, 'not found')(j)
- ]
+
+ libs = []
+ cmd = ['otool', '-L', os.path.basename(source[0].get_abspath())]
+ with ProcessRunner(cmd,
+ timeout=30,
+ seconds_to_wait=0.1,
+ cwd=os.path.dirname(source[0].get_abspath()),
+ env=getFlatENV(env)) as executor:
+ for out, _ in executor:
+ for j in re.findall(r'^.*=>\s*(not found|[^\s^\(]+)', out, re.MULTILINE):
+ if functools.partial(operator.ne, 'not found')(j):
+ libs.append(j)
+
if libnames:
libs = [j for j in libs if functools.partial(self.__filterLibs, env, libnames=libnames)(j)]
return libs
@@ -136,16 +142,17 @@ class MacFinder(LibFinder):
def getSystemLibDirs(self, env):
libdirs = []
linkercmd = env.subst('$LINK')
- cmdargs = [linkercmd, '-print-search-dirs'] + env.subst('$LINKFLAGS').split(' ')
- linker = PopenHelper(cmdargs, stdout=PIPE, env=getFlatENV(env))
- out, _ = linker.communicate()
- match = re.search('^libraries.*=(.*)$', out, re.MULTILINE)
- if match:
- libdirs.extend(
- unique([
- os.path.abspath(j) for j in match.group(1).split(os.pathsep)
- if os.path.exists(os.path.abspath(j))
- ]))
+ cmd = [linkercmd, '-print-search-dirs'] + env.subst('$LINKFLAGS').split(' ')
+
+ with ProcessRunner(cmd, timeout=30, seconds_to_wait=0.1, env=getFlatENV(env)) as executor:
+ for out, _ in executor:
+ match = re.search('^libraries.*=(.*)$', out, re.MULTILINE)
+ if match:
+ libdirs.extend(
+ unique([
+ os.path.abspath(j) for j in match.group(1).split(os.pathsep)
+ if os.path.exists(os.path.abspath(j))
+ ]))
return libdirs
@@ -165,12 +172,15 @@ class Win32Finder(LibFinder):
return None
def getLibs(self, env, source, libnames=None, libdirs=None):
- ldd = PopenHelper(['objdump', '-p', os.path.basename(source[0].get_abspath())],
- stdout=PIPE,
- cwd=os.path.dirname(source[0].get_abspath()),
- env=getFlatENV(env))
- out, _ = ldd.communicate()
- deplibs = re.findall(r'DLL Name:\s*(\S*)', out, re.MULTILINE)
+ deplibs = []
+ cmd = ['objdump', '-p', os.path.basename(source[0].get_abspath())]
+ with ProcessRunner(cmd,
+ timeout=30,
+ seconds_to_wait=0.1,
+ cwd=os.path.dirname(source[0].get_abspath()),
+ env=getFlatENV(env)) as executor:
+ for out, _ in executor:
+ deplibs.extend(re.findall(r'DLL Name:\s*(\S*)', out, re.MULTILINE))
if not libdirs:
libdirs = self.getSystemLibDirs(env)
if libnames:
diff --git a/SConsider/PopenHelper.py b/SConsider/PopenHelper.py
index 9babf780234b13ad625a8c0207ac78298595d9a6..1f75413d1c9f7aba774051ee69d50c0396467cb1 100644
--- a/SConsider/PopenHelper.py
+++ b/SConsider/PopenHelper.py
@@ -9,28 +9,57 @@
# -------------------------------------------------------------------------
import shlex
+import sys
+import os
+from locale import getpreferredencoding
+import time
+from threading import Timer, Thread, Event
+from tempfile import NamedTemporaryFile
from logging import getLogger
logger = getLogger(__name__)
has_timeout_param = True
try:
- from subprocess32 import Popen, PIPE, STDOUT, TimeoutExpired
+ from subprocess32 import Popen, PIPE, STDOUT, TimeoutExpired, CalledProcessError
except:
try:
- from subprocess import Popen, PIPE, STDOUT, TimeoutExpired
+ from subprocess import Popen, PIPE, STDOUT, TimeoutExpired, CalledProcessError
except:
from subprocess import Popen, PIPE, STDOUT
+ # Exception classes copied over from subprocess32
+ class CalledProcessError(Exception):
+ """This exception is raised when a process run by check_call() or
+ check_output() returns a non-zero exit status.
+
+ The exit status will be stored in the returncode attribute;
+ check_output() will also store the output in the output
+ attribute.
+ """
+ def __init__(self, returncode, cmd, output=None):
+ self.returncode = returncode
+ self.cmd = cmd
+ self.output = output
+
+ def __str__(self):
+ return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
+
class TimeoutExpired(Exception):
- def __init__(self):
- pass
+ """This exception is raised when the timeout expires while waiting
+ for a child process."""
+ def __init__(self, cmd, timeout, output=None):
+ self.cmd = cmd
+ self.timeout = timeout
+ self.output = output
+
+ def __str__(self):
+ return ("Command '%s' timed out after %s seconds" % (self.cmd, self.timeout))
has_timeout_param = False
class PopenHelper(object):
def __init__(self, command, **kw):
- self.os_except = None
self.returncode = None
self.has_timeout = has_timeout_param
_exec_using_shell = kw.get('shell', False)
@@ -40,11 +69,12 @@ class PopenHelper(object):
logger.debug("Executing command: %s with kw-args: %s", _command_list, kw)
self.po = Popen(_command_list, **kw)
- def communicate(self, stdincontent=None, timeout=10, raise_except=False):
+ def communicate(self, stdincontent=None, timeout=10):
_kwargs = {'input': stdincontent}
if self.has_timeout:
_kwargs['timeout'] = timeout
try:
+ logger.debug("communicating with _kwargs: %s", _kwargs)
_stdout, _stderr = self.po.communicate(**_kwargs)
except TimeoutExpired:
_kwargs = {'input': None}
@@ -53,12 +83,201 @@ class PopenHelper(object):
self.po.kill()
_stdout, _stderr = self.po.communicate(**_kwargs)
except OSError as ex:
- self.os_except = ex
+ # raised if the executed program cannot be found
+ logger.debug(ex)
finally:
self.returncode = self.po.poll()
- if raise_except:
- raise
+ logger.debug("Popen returncode: %d, poll() returncode: %d", self.po.returncode, self.returncode)
return (_stdout, _stderr)
def __getattr__(self, name):
return getattr(self.po, name)
+
+
+class Tee(object):
+ def __init__(self):
+ self.writers = []
+ # Wrap sys.stdout into a StreamWriter to allow writing unicode.
+ # https://stackoverflow.com/a/4546129/542082
+ # if not sys.stdout.isatty():
+ # sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
+ self._preferred_encoding = getpreferredencoding()
+
+ def __del__(self):
+ self.close()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
+ def attach_file(self, writer):
+ def flush(stream):
+ stream.flush()
+ os.fsync(stream.fileno())
+
+ _decoder = (lambda msg: msg.decode(writer.encoding)) if writer.encoding else (lambda msg: msg)
+ self.writers.append((writer, _decoder, flush, lambda stream: stream.close()))
+
+ def attach_std(self, writer=sys.stdout):
+ def flush(stream):
+ stream.flush()
+
+ _decoder = (lambda msg: msg.decode(writer.encoding)) if writer.encoding else (lambda msg: msg)
+ self.writers.append((writer, _decoder, flush, lambda _: ()))
+
+ def write(self, message):
+ for writer, decoder, _, _ in self.writers:
+ if not writer.closed:
+ writer.write(decoder(message))
+
+ def flush(self):
+ for stream, _, flusher, _ in self.writers:
+ flusher(stream)
+
+ def close(self):
+ while self.writers:
+ stream, _, _, closer = self.writers.pop()
+ closer(stream)
+
+
+#https://stackoverflow.com/a/54868710/542082
+class ProcessRunner(object):
+ def __init__(self, args, timeout=None, bufsize=-1, seconds_to_wait=0.25, **kwargs):
+ """Constructor facade to subprocess.Popen that receives parameters
+ which are more specifically required for the.
+
+ Process Runner. This is a class that should be used as a context manager - and that provides an iterator
+ for reading captured output from subprocess.communicate in near realtime.
+
+ Example usage:
+
+ exitcode = -9
+ with Tee() as tee:
+ tee.attach_std()
+ process_runner = None
+ try:
+ with ProcessRunner(('ls', '-lAR', '.'), seconds_to_wait=0.25) as process_runner:
+ for out, err in process_runner:
+ tee.write(out)
+ exitcode = process_runner.returncode
+ except CalledProcessError as e:
+ logger.debug("non-zero exitcode: %s", e)
+ except TimeoutExpired as e:
+ logger.debug(e)
+ except OSError as e:
+ logger.debug("executable error: %s", e)
+ # follow shell exit code
+ exitcode = 127
+ except Exception as e:
+ logger.debug("process creation failure: %s", e)
+ finally:
+ if process_runner:
+ exitcode = process_runner.returncode
+
+ :param args: same as subprocess.Popen
+ :param timeout: same as subprocess.communicate
+ :param bufsize: same as subprocess.Popen
+ :param seconds_to_wait: time to wait between each readline from the temporary file
+ :param kwargs: same as subprocess.Popen
+ """
+ self._seconds_to_wait = seconds_to_wait
+ self._process_has_timed_out = False
+ self._timeout = timeout
+ self._has_timeout = has_timeout_param
+ self._process_done = False
+ self._stdout_file_handle = NamedTemporaryFile()
+ self._stderr_file_handle = kwargs.pop('stderr', None)
+ # DEVNULL and STDOUT will be passed through
+ # redirect to err file otherwise
+ if self._stderr_file_handle in (None, PIPE):
+ self._stderr_file_handle = NamedTemporaryFile()
+ self._stdin_handle = kwargs.pop('stdin_handle', None)
+ if not self._stdin_handle is None:
+ kwargs.setdefault('stdin', PIPE)
+
+ _exec_using_shell = kwargs.get('shell', False)
+ self._command_list = args
+ if not _exec_using_shell and not isinstance(args, list) and not isinstance(args, tuple):
+ self._command_list = shlex.split(args)
+ logger.debug("Executing command: %s with kwargs: %s", self._command_list, kwargs)
+
+ self._process = Popen(self._command_list,
+ bufsize=bufsize,
+ stdout=self._stdout_file_handle,
+ stderr=self._stderr_file_handle,
+ **kwargs)
+ self._thread = Thread(target=self._run_process)
+ self._thread.daemon = True
+
+ def __enter__(self):
+ self._thread.start()
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self._thread.join()
+ self._stdout_file_handle.close()
+ # for newer versions, DEVNULL should also be masked
+ if not self._stderr_file_handle == STDOUT:
+ self._stderr_file_handle.close()
+
+ def __iter__(self):
+ # read all output from stdout file that subprocess.communicate fills
+ _firstiter = True
+ _stderr = None
+ _err = ''
+ try:
+ if not self._stderr_file_handle == STDOUT:
+ _stderr = open(self._stderr_file_handle.name, 'r')
+ with open(self._stdout_file_handle.name, 'r') as _stdout:
+ # while process is alive, keep reading data
+ while not self._process_done:
+ _out = _stdout.readline()
+ if _stderr:
+ _err = _stderr.readline()
+ if _out or _err:
+ yield (_out, _err)
+ else:
+ # if there is nothing to read, then please wait a tiny little bit
+ if _firstiter:
+ time.sleep(0.001)
+ _firstiter = False
+ continue
+ logger.debug("Command %s (pid: %s) has nothing to read from, sleeping for %ss",
+ self._command_list, self._process.pid, self._seconds_to_wait)
+ time.sleep(self._seconds_to_wait)
+
+ # catch writes to buffer after process has finished
+ _out = _stdout.read()
+ if _stderr:
+ _err = _stderr.read()
+ if _out or _err:
+ yield (_out, _err)
+ finally:
+ if _stderr:
+ _stderr.close()
+
+ if self._process_has_timed_out:
+ raise TimeoutExpired(self._command_list, self._timeout)
+
+ if self._process.returncode != 0:
+ raise CalledProcessError(self.returncode, self._command_list)
+
+ def _run_process(self):
+ try:
+ # Start gathering information (stdout and stderr) from the opened process
+ self._process.communicate(input=self._stdin_handle, timeout=self._timeout)
+ # Graceful termination of the opened process
+ self._process.terminate()
+ except TimeoutExpired:
+ # Force termination of the opened process
+ self._process.terminate()
+ self._process.communicate(timeout=5.0)
+ self._process_has_timed_out = True
+
+ self._process_done = True
+
+ @property
+ def returncode(self):
+ return self._process.returncode
diff --git a/SConsider/SomeUtils.py b/SConsider/SomeUtils.py
index dd93d45bde964062ddb500fda4c814c4264f07cc..b421fbdab71dfb548035f338f29682f5b6c0e5da 100644
--- a/SConsider/SomeUtils.py
+++ b/SConsider/SomeUtils.py
@@ -15,7 +15,9 @@ Collection of helper functions
import os
import re
-from SConsider.PopenHelper import PopenHelper, PIPE
+from logging import getLogger
+from SConsider.PopenHelper import ProcessRunner, CalledProcessError, TimeoutExpired
+logger = getLogger(__name__)
def FileNodeComparer(left, right):
@@ -325,8 +327,7 @@ def getfqdn():
return (hostname, domain, fqdn)
-def runCommand(args, logpath='', filename=None, stdincontent=None, timeout=120, **kw):
- res = 1
+def runCommand(args, logpath='', filename=None, stdincontent=None, timeout=120.0, **kw):
if filename:
with open(filename) as f:
stdincontent = f.read()
@@ -341,8 +342,6 @@ def runCommand(args, logpath='', filename=None, stdincontent=None, timeout=120,
if callable(filter_func):
stdincontent = filter_func(stdincontent)
- popenObject = PopenHelper(args, stdin=PIPE, stderr=PIPE, stdout=PIPE, **kw)
-
if not os.path.isdir(logpath):
os.makedirs(logpath)
logfilebasename = os.path.basename(args[0])
@@ -350,21 +349,33 @@ def runCommand(args, logpath='', filename=None, stdincontent=None, timeout=120,
logfilebasename = logfilebasename + '.' + os.path.basename(filename)
errfilename = os.path.join(logpath, logfilebasename + '.stderr')
outfilename = os.path.join(logpath, logfilebasename + '.stdout')
+
+ executor = None
try:
- popen_out, popen_err = popenObject.communicate(stdincontent, timeout=timeout)
- if popen_err:
- with open(errfilename, 'w') as errfile:
- errfile.write(popen_err)
- if popen_out:
- with open(outfilename, 'w') as outfile:
- outfile.write(popen_out)
- res = popenObject.returncode
- except OSError as ex:
- with open(errfilename, 'w') as errfile:
- print >> errfile, ex
- for line in popenObject.stderr:
- print >> errfile, line
- return res
+ errfile = open(errfilename, 'w')
+ outfile = open(outfilename, 'w')
+ kw.setdefault('seconds_to_wait', 0.1)
+ with ProcessRunner(args, timeout=timeout, stdin_handle=stdincontent, **kw) as executor:
+ for out, err in executor:
+ outfile.write(out)
+ errfile.write(err)
+ except CalledProcessError as e:
+ logger.debug("non-zero exitcode: %s", e)
+ except TimeoutExpired as e:
+ logger.debug(e)
+ except OSError as e:
+ logger.debug("executable error: %s", e)
+ # follow shell exit code
+ exitcode = 127
+ except Exception as e:
+ logger.debug("process creation failure: %s", e)
+ print >> errfile, e
+ finally:
+ if executor:
+ exitcode = executor.returncode
+ errfile.close()
+ outfile.close()
+ return exitcode
def getLibCVersion(bits='32'):
diff --git a/SConsider/site_tools/DoxygenBuilder.py b/SConsider/site_tools/DoxygenBuilder.py
index 9e93308fc907341766f2856278d436ecdfb02eb4..8ea122b1e91eca40713430562587e9a2293fd387 100644
--- a/SConsider/site_tools/DoxygenBuilder.py
+++ b/SConsider/site_tools/DoxygenBuilder.py
@@ -18,7 +18,7 @@ from __future__ import with_statement
import os
import re
from logging import getLogger
-from SConsider.PopenHelper import PopenHelper, PIPE
+from SConsider.PopenHelper import ProcessRunner
logger = getLogger(__name__)
@@ -78,12 +78,13 @@ def getPackageInputDirs(registry, packagename, relativeTo=None):
else:
return abspath
+ import SCons
for _, settings in buildSettings.items():
- import SCons
# directories of own cpp files
for sourcefile in settings.get('sourceFiles', []):
- if isinstance(sourcefile, SCons.Node.FS.File):
- sourceDirs.add(resolvePath(sourcefile.srcnode().dir.get_abspath(), relativeTo))
+ if not isinstance(sourcefile, SCons.Node.FS.File):
+ sourcefile = includeBasedir.File(sourcefile)
+ sourceDirs.add(resolvePath(sourcefile.srcnode().dir.get_abspath(), relativeTo))
# include directory of own private headers
includeSubdirPrivate = settings.get('includeSubdir', '')
@@ -95,8 +96,9 @@ def getPackageInputDirs(registry, packagename, relativeTo=None):
# directories of own public headers which are going to be copied
for sourcefile in settings.get('public', {}).get('includes', []):
- if isinstance(sourcefile, SCons.Node.FS.File):
- sourceDirs.add(resolvePath(sourcefile.srcnode().dir.get_abspath(), relativeTo))
+ if not isinstance(sourcefile, SCons.Node.FS.File):
+ sourcefile = includeBasedir.File(sourcefile)
+ sourceDirs.add(resolvePath(sourcefile.srcnode().dir.get_abspath(), relativeTo))
return sourceDirs
@@ -143,9 +145,12 @@ def setDoxyfileData(doxyfile, doxyDict):
def getDoxyfileTemplate():
- proc = PopenHelper(['doxygen', '-s', '-g', '-'], stdout=PIPE, stderr=PIPE)
- stdout, _ = proc.communicate()
- return parseDoxyfileContent(stdout, {})
+ _cmd = ['doxygen', '-s', '-g', '-']
+ _out = ''
+ with ProcessRunner(_cmd, timeout=30) as executor:
+ for out, _ in executor:
+ _out += out
+ return parseDoxyfileContent(_out, {})
def parseDoxyfile(file_node, env):
@@ -326,10 +331,13 @@ def buildDoxyfile(target, env, **kw):
doxyfile.write('%s \\\n' % define)
doxyfile.write('\n')
- log_out, log_err = openLogFiles(env)
try:
- proc = PopenHelper(['doxygen', '-s', '-u', target[0].get_abspath()], stdout=log_out, stderr=log_err)
- proc.wait()
+ log_out, log_err = openLogFiles(env)
+ _cmd = ['doxygen', '-s', '-u', target[0].get_abspath()]
+ with ProcessRunner(_cmd, timeout=60) as executor:
+ for out, err in executor:
+ log_out.write(out)
+ log_err.write(err)
finally:
closeLogFiles(log_out, log_err)
@@ -373,10 +381,12 @@ def callDoxygen(source, env, **kw):
cmd = 'cd %s && %s %s' % (source[0].get_dir().get_abspath(), 'doxygen', os.path.basename(str(source[0])))
- log_out, log_err = openLogFiles(env)
try:
- proc = PopenHelper(cmd, shell=True, stdout=log_out, stderr=log_err)
- proc.wait()
+ log_out, log_err = openLogFiles(env)
+ with ProcessRunner(cmd, timeout=300, shell=True) as executor:
+ for out, err in executor:
+ log_out.write(out)
+ log_err.write(err)
finally:
closeLogFiles(log_out, log_err)
@@ -654,13 +664,6 @@ class DoxygenToolException(Exception):
def determineCompilerDefines(env):
- cmd = 'PATH=' + env.get('ENV', []).get('PATH', '')
- cmd += ' ' + os.path.join(os.path.dirname(__file__), 'defines.sh')
- cmd += ' ' + env['CXX']
-
- proc = PopenHelper(cmd, shell=True, stdout=PIPE, stderr=PIPE)
- stdoutdata, _ = proc.communicate()
-
defines = {}
ignores = [
'cc',
@@ -672,10 +675,17 @@ def determineCompilerDefines(env):
'__TIMESTAMP__',
]
pattern = re.compile(r'^([^\s]+)\s*=\s*(.*)\s*')
- for line in stdoutdata.splitlines():
- match = re.match(pattern, line)
- if match and match.group(1) not in ignores:
- defines[match.group(1)] = match.group(2)
+
+ cmd = 'PATH=' + env.get('ENV', []).get('PATH', '')
+ cmd += ' ' + os.path.join(os.path.dirname(__file__), 'defines.sh')
+ cmd += ' ' + env['CXX']
+
+ with ProcessRunner(cmd, timeout=60, shell=True) as executor:
+ for out, _ in executor:
+ for line in out.splitlines():
+ match = re.match(pattern, line)
+ if match and match.group(1) not in ignores:
+ defines[match.group(1)] = match.group(2)
return defines
diff --git a/SConsider/site_tools/RunBuilder.py b/SConsider/site_tools/RunBuilder.py
index b835c87d9849155f17dab4e648d57f32bd0029a4..59e1ebc55e0ef91c6cd60717a804228e910c2681 100644
--- a/SConsider/site_tools/RunBuilder.py
+++ b/SConsider/site_tools/RunBuilder.py
@@ -20,9 +20,9 @@ setup/teardown functions executed before and after running the program.
from __future__ import with_statement
import os
import optparse
-import sys
import shlex
from logging import getLogger
+import sys
from SCons.Action import Action
from SCons.Builder import Builder
from SCons.Script import AddOption, GetOption, COMMAND_LINE_TARGETS
@@ -30,10 +30,11 @@ from SCons.Util import is_List
from SConsider.PackageRegistry import PackageRegistry
from SConsider.Callback import Callback
from SConsider.SomeUtils import hasPathPart, isFileNode, isDerivedNode, getNodeDependencies, getFlatENV
-from SConsider.PopenHelper import PopenHelper, PIPE, STDOUT
+from SConsider.PopenHelper import ProcessRunner, Tee, CalledProcessError, TimeoutExpired, STDOUT
logger = getLogger(__name__)
runtargets = {}
+_DEFAULT_TIMEOUT = 120.0
def setTarget(packagename, targetname, target):
@@ -57,52 +58,38 @@ def getTargets(packagename=None, targetname=None):
return targets
-class Tee(object):
- def __init__(self):
- self.writers = []
-
- def add(self, writer, flush=False, close=True):
- self.writers.append((writer, flush, close))
-
- def write(self, output):
- for writer, flush, _ in self.writers:
- writer.write(output)
- if flush:
- writer.flush()
-
- def close(self):
- for writer, _, close in self.writers:
- if close:
- writer.close()
-
-
def run(cmd, logfile=None, **kw):
"""Run a Unix command and return the exit code."""
- tee = Tee()
- tee.add(sys.stdout, flush=True, close=False)
- rcode = 99
- proc = None
- try:
+ exitcode = 99
+ with Tee() as tee:
+ tee.attach_std()
if logfile:
if not os.path.isdir(logfile.dir.get_abspath()):
os.makedirs(logfile.dir.get_abspath())
- tee.add(open(logfile.get_abspath(), 'w'))
- proc = PopenHelper(cmd, stdin=None, stdout=PIPE, stderr=STDOUT, **kw)
- while True:
- out = proc.stdout.read(1)
- if out == '' and proc.poll() is not None:
- break
- tee.write(out)
- rcode = proc.returncode
- finally:
- while True and proc:
- out = proc.stdout.readline()
- if out == '' and proc.poll() is not None:
- break
- tee.write(out)
- tee.close()
-
- return rcode
+ tee.attach_file(open(logfile.get_abspath(), 'w'))
+ process_runner = None
+ try:
+ #FIXME: add timeout parameter
+ with ProcessRunner(cmd, stderr=STDOUT, seconds_to_wait=0.2, **kw) as process_runner:
+ for out, _ in process_runner:
+ tee.write(out)
+ exitcode = process_runner.returncode
+ except CalledProcessError as e:
+ logger.debug("non-zero exitcode: %s", e)
+ except TimeoutExpired as e:
+ logger.debug(e)
+ except OSError as e:
+ logger.debug("executable error: %s", e)
+ # follow shell exit code
+ exitcode = 127
+ except Exception as e:
+ logger.debug("process creation failure: %s", e)
+ finally:
+ if process_runner:
+ exitcode = process_runner.returncode
+
+ logger.debug("returncode: %d", exitcode)
+ return exitcode
def emitPassedFile(target, source, env):
@@ -120,7 +107,10 @@ def execute(command, env):
if 'mingw' in env["TOOLS"]:
args.insert(0, "sh.exe")
- return run(args, env=getFlatENV(env), logfile=env.get('logfile', None))
+ return run(args,
+ env=getFlatENV(env),
+ logfile=env.get('logfile', None),
+ timeout=env.get('timeout', _DEFAULT_TIMEOUT))
def doTest(target, source, env):
@@ -144,15 +134,28 @@ def doRun(target, source, env):
return res
-def getRunParams(buildSettings, defaultRunParams):
+def getRunParams(buildSettings, default):
+ runConfig = buildSettings.get('runConfig', {})
+ params = GetOption('runParams')
+ if not params:
+ if not runConfig:
+ runConfig = dict()
+ params = runConfig.get('runParams', default)
+ if isinstance(params, list):
+ params = ' '.join(params)
+ return params
+
+
+def getRunTimeout(buildSettings, default):
runConfig = buildSettings.get('runConfig', {})
- if GetOption('runParams'):
- runParams = " ".join(GetOption('runParams'))
- else:
+ param = GetOption('runTimeout')
+ if param < 0.0:
if not runConfig:
runConfig = dict()
- runParams = runConfig.get('runParams', defaultRunParams)
- return runParams
+ param = runConfig.get('runTimeout', default)
+ if param <= 0.0:
+ param = None
+ return param
class SkipTest(Exception):
@@ -204,7 +207,11 @@ def createTestTarget(env, source, packagename, targetname, settings, defaultRunP
return (source, fullTargetName)
logfile = env.getLogInstallDir().File(targetname + '.test.log')
- runner = env.TestBuilder([], source, runParams=getRunParams(settings, defaultRunParams), logfile=logfile)
+ runner = env.TestBuilder([],
+ source,
+ runParams=getRunParams(settings, defaultRunParams),
+ logfile=logfile,
+ timeout=getRunTimeout(settings, default=_DEFAULT_TIMEOUT))
if GetOption('run-force'):
env.AlwaysBuild(runner)
@@ -251,7 +258,8 @@ def createRunTarget(env, source, packagename, targetname, settings, defaultRunPa
runner = env.RunBuilder(['dummyRunner_' + fullTargetName],
source,
runParams=getRunParams(settings, defaultRunParams),
- logfile=logfile)
+ logfile=logfile,
+ timeout=getRunTimeout(settings, default=_DEFAULT_TIMEOUT))
addRunConfigHooks(env, source, runner, settings)
@@ -281,18 +289,27 @@ def composeRunTargets(env, source, packagename, targetname, settings, defaultRun
def generate(env):
try:
- AddOption('--run', dest='run', action='store_true', default=False, help='Should we run the target')
+ AddOption('--run',
+ dest='run',
+ action='store_true',
+ default=False,
+ help='Run the target if not done yet')
AddOption('--run-force',
dest='run-force',
action='store_true',
default=False,
- help='Should we run the target and ignore .passed files')
+ help='Run the target regardless of the last state (.passed file)')
AddOption('--runparams',
dest='runParams',
action='append',
type='string',
default=[],
help='The parameters to hand over')
+ AddOption('--run-timeout',
+ dest='runTimeout',
+ action='store',
+ type='float',
+ help='Time in seconds after which the running process gets killed')
except optparse.OptionConflictError:
pass
diff --git a/SConsider/site_tools/TestfwTransformer.py b/SConsider/site_tools/TestfwTransformer.py
index 1385c292b179415c3a0f8c0b669830d7762f5064..9a153a7efd139b38b55c6512c1cc312200defc07 100644
--- a/SConsider/site_tools/TestfwTransformer.py
+++ b/SConsider/site_tools/TestfwTransformer.py
@@ -253,6 +253,7 @@ def callPostTest(target, packagename, targetname, logfile, **kw):
with open(logfile.get_abspath()) as f:
result = parser.parse(f)
with open(logfile.dir.File(targetname + '.test.xml').get_abspath(), 'w') as xmlfile:
+ xmlfile.write('<?xml version="1.0" encoding="UTF-8" ?>')
xmlfile.write(result.toXML(packagename + '.' + targetname))
diff --git a/SConsider/site_tools/g++.py b/SConsider/site_tools/g++.py
index 7abc5a84965c539cf1e1c87d3b71d0af64737ee2..1db68da46e11d68c68b3eb9f9cf9ece9fc328478 100644
--- a/SConsider/site_tools/g++.py
+++ b/SConsider/site_tools/g++.py
@@ -18,7 +18,7 @@ import re
from logging import getLogger
import SCons.Tool
import SCons.Util
-from SConsider.PopenHelper import PopenHelper, PIPE
+from SConsider.PopenHelper import ProcessRunner
logger = getLogger(__name__)
compilers = ['g++']
@@ -57,10 +57,15 @@ def generate(env):
bitwidth = env.getBitwidth()
if compiler_subject:
- _proc = PopenHelper([compiler_subject, '--version'], stdout=PIPE, stderr=PIPE)
- _out, _err = _proc.communicate()
-
- if _proc.returncode != 0:
+ _cmd = [compiler_subject, '--version']
+ _out = ''
+ _err = ''
+ with ProcessRunner(_cmd, timeout=20, seconds_to_wait=0.1) as executor:
+ for out, err in executor:
+ _out += out
+ _err += err
+
+ if executor.returncode != 0:
return
# -dumpversion was added in GCC 3.0. As long as we're supporting
# GCC versions older than that, we should use --version and a
@@ -90,10 +95,13 @@ def generate(env):
logger.error("failed to create compiler input file, check folder permissions and retry",
exc_info=True)
return
- _proc = PopenHelper([compiler_subject, '-v', '-xc++', tFile, '-o', outFile, '-m' + bitwidth],
- stdout=PIPE,
- stderr=PIPE)
- _out, _err = _proc.communicate()
+ _cmd = [compiler_subject, '-v', '-xc++', tFile, '-o', outFile, '-m' + bitwidth]
+ _out = ''
+ _err = ''
+ with ProcessRunner(_cmd, timeout=20, seconds_to_wait=0.1) as executor:
+ for out, err in executor:
+ _out += out
+ _err += err
text_to_join = ['---- stdout ----', _out, '---- stderr ----', _err]
build_output = os.linesep.join(text_to_join)
@@ -109,7 +117,7 @@ def generate(env):
exc_info=True)
raise SCons.Errors.UserError(
'Build aborted, {0} compiler detection failed!'.format(compiler_subject))
- if _proc.returncode != 0:
+ if executor.returncode != 0:
logger.error("compile command failed with return code {0}:".format(proc.returncode) + os.linesep +
build_output)
raise SCons.Errors.UserError(
diff --git a/SConsider/site_tools/gcc.py b/SConsider/site_tools/gcc.py
index 4857c9efcf3bfbdbfb7e0357d49570f1d8ccfbab..b2c06766ff5f2c32cfd91c5350ae626595d5b0b9 100644
--- a/SConsider/site_tools/gcc.py
+++ b/SConsider/site_tools/gcc.py
@@ -18,7 +18,7 @@ import re
from logging import getLogger
import SCons.Tool
import SCons.Util
-from SConsider.PopenHelper import PopenHelper, PIPE
+from SConsider.PopenHelper import ProcessRunner
logger = getLogger(__name__)
compilers = ['gcc', 'cc']
@@ -44,10 +44,15 @@ def generate(env):
bitwidth = env.getBitwidth()
if compiler_subject:
- _proc = PopenHelper([compiler_subject, '--version'], stdout=PIPE, stderr=PIPE)
- _out, _err = _proc.communicate()
-
- if _proc.returncode != 0:
+ _cmd = [compiler_subject, '--version']
+ _out = ''
+ _err = ''
+ with ProcessRunner(_cmd, timeout=20, seconds_to_wait=0.1) as executor:
+ for out, err in executor:
+ _out += out
+ _err += err
+
+ if executor.returncode != 0:
return
# -dumpversion was added in GCC 3.0. As long as we're supporting
# GCC versions older than that, we should use --version and a
@@ -76,10 +81,13 @@ def generate(env):
logger.error("failed to create compiler input file, check folder permissions and retry",
exc_info=True)
return
- _proc = PopenHelper([compiler_subject, '-v', '-xc', tFile, '-o', outFile, '-m' + bitwidth],
- stdout=PIPE,
- stderr=PIPE)
- _out, _err = _proc.communicate()
+ _cmd = [compiler_subject, '-v', '-xc', tFile, '-o', outFile, '-m' + bitwidth]
+ _out = ''
+ _err = ''
+ with ProcessRunner(_cmd, timeout=20, seconds_to_wait=0.1) as executor:
+ for out, err in executor:
+ _out += out
+ _err += err
text_to_join = ['---- stdout ----', _out, '---- stderr ----', _err]
build_output = os.linesep.join(text_to_join)
@@ -95,7 +103,7 @@ def generate(env):
exc_info=True)
raise SCons.Errors.UserError(
'Build aborted, {0} compiler detection failed!'.format(compiler_subject))
- if _proc.returncode != 0:
+ if executor.returncode != 0:
logger.error("compile command failed with return code {0}:".format(proc.returncode) + os.linesep +
build_output)
raise SCons.Errors.UserError(
diff --git a/SConsider/site_tools/generateScript.py b/SConsider/site_tools/generateScript.py
index b535f972e2f46dea96b0ebb9f6414f22b85c7e59..064af53a1fef5e0ead1292a231902fa784f99575 100644
--- a/SConsider/site_tools/generateScript.py
+++ b/SConsider/site_tools/generateScript.py
@@ -266,17 +266,15 @@ if [ ${doDebug:-0} -ge 1 ]; then
test ${doTrace} -eq 1 && cat ${cfg_gdbcommands}
cfg_gdbcommands="--command $cfg_gdbcommands";
test $doDebug -gt 1 && cfg_gdbcommands="--batch $cfg_gdbcommands";
- eval gdb ${cfg_gdbcommands}
+ exec eval gdb ${cfg_gdbcommands}
elif [ ${doDebugServer:-0} -eq 1 -a -x "$(type -fP gdbserver 2>/dev/null)" ]; then
- gdbserver :${GDBSERVERPORT} "${CMD}" "$@"
+ exec gdbserver :${GDBSERVERPORT} "${CMD}" "$@"
elif [ ${doCommandWithArgs:-0} -eq 1 ]; then
test ${doTrace} -eq 1 && echo "executing command [${cmdArr[*]} ${CMD} $@]"
- eval "${cmdArr[*]} ${CMD} $@"
+ exec eval "${cmdArr[*]} ${CMD} $@"
else
- "$CMD" "$@"
+ exec "$CMD" "$@"
fi
-
-exit $?
"""
scriptFile.write(scriptText)
diff --git a/VERSION b/VERSION
index b88c14da673b43c3ea634c60b8f87dd47d87c13b..6e46293aeb25f76497c29f5f8c386c931b5408b8 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.3.16
\ No newline at end of file
+0.3.17
\ No newline at end of file
diff --git a/tests/conftest.py b/tests/conftest.py
index d62a7b08175e82b95497979e695a81ad7c750049..c5fe694928d4c058e47df8ca333bd57423e91654 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -67,3 +67,19 @@ def copy_testdir_to_tmp(tmpdir_factory, invocation_path, current_testdir):
sources_path = invocation_path(current_testdir)
shutil.copytree(str(sources_path), str(fn), ignore=ignorefiles)
return fn
+
+
+# https://stackoverflow.com/a/51495108/542082
+@pytest.fixture(scope='function', autouse=True)
+def testcase_result(request):
+ def fin():
+ print(" (DURATION={})".format(request.node.rep_call.duration))
+
+ request.addfinalizer(fin)
+
+
+@pytest.mark.tryfirst
+def pytest_runtest_makereport(item, call, __multicall__):
+ rep = __multicall__.execute()
+ setattr(item, "rep_" + rep.when, rep)
+ return rep
diff --git a/tests/invocation/debuglog.yaml b/tests/invocation/debuglog.yaml
index 239ed680fd92b8a12203d03f99184c937f52557a..36e97ee2fa58af4fa23322ca5979bf432feafb2c 100644
--- a/tests/invocation/debuglog.yaml
+++ b/tests/invocation/debuglog.yaml
@@ -21,6 +21,10 @@ filters:
'()': SConsider.Logging.RegexFilter
pattern: "^executing \\["
flags: 0
+ filterProcessHelperWaitOnStream:
+ '()': SConsider.Logging.RegexFilter
+ pattern: ".*has nothing to read from, sleeping for.*"
+ flags: 0
handlers:
console:
@@ -54,6 +58,23 @@ loggers:
propagate: no
filters: []
+ g++:
+ level: INFO
+ handlers: [console]
+ propagate: no
+ filters: []
+
+ gcc:
+ level: INFO
+ handlers: [console]
+ propagate: no
+ filters: []
+
+ py.warnings:
+ level: WARNING
+ handlers: [console]
+ propagate: no
+
TargetMaker:
level: DEBUG
handlers: [console]
@@ -72,6 +93,12 @@ loggers:
propagate: no
filters: [filterLookup]
+ SConsider.PopenHelper:
+ level: DEBUG
+ handlers: [console]
+ propagate: no
+ filters: [filterProcessHelperWaitOnStream]
+
setupBuildTools:
level: DEBUG
handlers: [console]
diff --git a/tests/invocation/infolog.yaml b/tests/invocation/infolog.yaml
index 8b82c591053f85d84651e63a734d25f21f9a510c..df2d484e4d75f12fe63c7e09e9aa8dc4f49889dc 100644
--- a/tests/invocation/infolog.yaml
+++ b/tests/invocation/infolog.yaml
@@ -21,6 +21,10 @@ filters:
'()': SConsider.Logging.RegexFilter
pattern: "^executing \\["
flags: 0
+ filterProcessHelperWaitOnStream:
+ '()': SConsider.Logging.RegexFilter
+ pattern: ".*has nothing to read from, sleeping for.*"
+ flags: 0
handlers:
console:
@@ -54,6 +58,23 @@ loggers:
propagate: no
filters: []
+ g++:
+ level: INFO
+ handlers: [console]
+ propagate: no
+ filters: []
+
+ gcc:
+ level: INFO
+ handlers: [console]
+ propagate: no
+ filters: []
+
+ py.warnings:
+ level: WARNING
+ handlers: [console]
+ propagate: no
+
TargetMaker:
level: WARNING
handlers: [console]
@@ -72,6 +93,12 @@ loggers:
propagate: no
filters: [filterLookup, filterExecuting]
+ SConsider.PopenHelper:
+ level: INFO
+ handlers: [console]
+ propagate: no
+ filters: []
+
setupBuildTools:
level: INFO
handlers: [console]
@@ -90,4 +117,5 @@ loggers:
root:
level: INFO
handlers: [console, info_file_handler, error_file_handler]
+ filters: []
...
diff --git a/tests/test_ProcessRunner.py b/tests/test_ProcessRunner.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9cf1362d1a7384d0134192d7a225a92dd04d336
--- /dev/null
+++ b/tests/test_ProcessRunner.py
@@ -0,0 +1,93 @@
+# -------------------------------------------------------------------------
+# Copyright (c) 2016, Peter Sommerlad and IFS Institute for Software
+# at HSR Rapperswil, Switzerland
+# All rights reserved.
+#
+# This library/application is free software; you can redistribute and/or
+# modify it under the terms of the license that is included with this
+# library/application in the file license.txt.
+# -------------------------------------------------------------------------
+
+import pytest
+import sys
+from SConsider.PopenHelper import ProcessRunner, STDOUT, CalledProcessError, TimeoutExpired
+
+
+def test_CommandStringWithStdout(capfd):
+ cmd = 'ls'
+ with ProcessRunner(cmd) as executor:
+ for out, _ in executor:
+ sys.stdout.write(out)
+ assert 0 == executor.returncode
+ captured = capfd.readouterr()
+ assert 'setup.py' in captured.out
+
+
+def test_CommandStringWithStderr(capfd):
+ with pytest.raises(CalledProcessError) as excinfo:
+ cmd = 'ls _NotEx.isting_'
+ with ProcessRunner(cmd) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 2 == excinfo.value.returncode
+ captured = capfd.readouterr()
+ assert 'cannot access' in captured.err
+ assert '' == captured.out
+
+
+def test_CommandStringWithStderrOnStdout(capfd):
+ with pytest.raises(CalledProcessError) as excinfo:
+ cmd = 'ls _NotEx.isting_'
+ with ProcessRunner(cmd, stderr=STDOUT) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 2 == excinfo.value.returncode
+ captured = capfd.readouterr()
+ assert 'cannot access' in captured.out
+ assert '' == captured.err
+
+
+def test_CommandArrayWithOutput(capfd):
+ cmd = ['ls']
+ with ProcessRunner(cmd) as executor:
+ for out, _ in executor:
+ sys.stdout.write(out)
+ assert 0 == executor.returncode
+ captured = capfd.readouterr()
+ assert 'setup.py' in captured.out
+
+
+def test_CommandStringWithOutputFromInput(capfd):
+ cmd = 'cat -'
+ send_on_stdin = 'loopback string'
+ with ProcessRunner(cmd, timeout=3, stdin_handle=send_on_stdin) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ captured = capfd.readouterr()
+ assert 0 == executor.returncode
+ assert send_on_stdin in captured.out
+ assert '' == captured.err
+
+
+def test_CommandStringWithTimeoutResultsInKill():
+ with pytest.raises(TimeoutExpired) as excinfo:
+ cmd = 'sleep 3'
+ with ProcessRunner(cmd, timeout=1) as executor:
+ for out, _ in executor:
+ sys.stdout.write(out)
+ assert 'sleep' in str(excinfo.value.cmd)
+
+
+def test_CommandStringNotSplitWhenUsingShell(capfd):
+ cmd = r"""for n in 1 2 3; do
+ echo $n;
+done"""
+ with ProcessRunner(cmd, timeout=1, shell=True) as executor:
+ for out, _ in executor:
+ sys.stdout.write(out)
+ assert 0 == executor.returncode
+ captured = capfd.readouterr()
+ assert '1\n2\n3\n' in captured.out
diff --git a/tests/test_SConsInvocations.py b/tests/test_SConsInvocations.py
index c08fe6a5e4ce624da107358a8c6a9797cc8d5f02..c6726c9c8a50337b4b40f9cebdc0a5cd2b9ffab5 100644
--- a/tests/test_SConsInvocations.py
+++ b/tests/test_SConsInvocations.py
@@ -11,8 +11,9 @@
import pytest
import os
import re
+import sys
from glob import glob
-from SConsider.PopenHelper import PopenHelper
+from SConsider.PopenHelper import ProcessRunner
@pytest.mark.invocation
@@ -21,11 +22,13 @@ def test_SConsiderGetHelp(copy_testdir_to_tmp, pypath_extended_env, popen_timeou
invocation_path, capfd):
pypath_extended_env.update({'LOG_CFG': str(invocation_path('debuglog.yaml'))})
- sub_p = PopenHelper(r'scons -h' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons -h' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert '--baseoutdir' in captured.out
assert '--archbits' in captured.out
@@ -35,11 +38,13 @@ def test_SConsiderGetHelp(copy_testdir_to_tmp, pypath_extended_env, popen_timeou
@pytest.mark.parametrize('current_testdir', ['gethelp'])
def test_SConsiderSconstructDirSameAsLaunchDir(copy_testdir_to_tmp, pypath_extended_env, popen_timeout,
scons_platform_options, capfd):
- sub_p = PopenHelper(r'scons -h' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons -h' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'get_sconstruct_dir=' + str(copy_testdir_to_tmp) in captured.out
assert 'get_launch_dir=' + str(copy_testdir_to_tmp) in captured.out
@@ -51,9 +56,12 @@ def test_SConsiderSconstructDirSameAsLaunchDir(copy_testdir_to_tmp, pypath_exten
def test_SConsiderSconstructDirBelowLaunchDir(copy_testdir_to_tmp, pypath_extended_env, popen_timeout,
scons_platform_options, capfd):
sconstruct_file = os.path.join(str(copy_testdir_to_tmp), 'SConstruct')
- sub_p = PopenHelper(r'scons -h -f ' + sconstruct_file + scons_platform_options, env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons -h -f ' + sconstruct_file + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'get_sconstruct_dir=' + str(copy_testdir_to_tmp) in captured.out
assert 'get_launch_dir=' + os.getcwd() in captured.out
@@ -64,11 +72,13 @@ def test_SConsiderSconstructDirBelowLaunchDir(copy_testdir_to_tmp, pypath_extend
def test_SConsiderSconstructAboveLaunchDir(copy_testdir_to_tmp, pypath_extended_env, popen_timeout,
scons_platform_options, capfd):
progdir_path = copy_testdir_to_tmp.join('progdir')
- sub_p = PopenHelper(r'scons -h -u ' + scons_platform_options,
- cwd=str(progdir_path),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons -h -u ' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(progdir_path),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'get_sconstruct_dir=' + str(copy_testdir_to_tmp) in captured.out
assert 'get_launch_dir=' + str(progdir_path) in captured.out
@@ -78,11 +88,13 @@ def test_SConsiderSconstructAboveLaunchDir(copy_testdir_to_tmp, pypath_extended_
@pytest.mark.invocation
def test_SConsiderStaticProgBuild(copy_testdir_to_tmp, pypath_extended_env, popen_timeout,
scons_platform_options, capfd):
- sub_p = PopenHelper(r'scons --3rdparty=' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --3rdparty=' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'done building targets.' in captured.out
@@ -90,11 +102,13 @@ def test_SConsiderStaticProgBuild(copy_testdir_to_tmp, pypath_extended_env, pope
@pytest.mark.invocation
def test_SConsiderStaticProgDoxygenOnly(copy_testdir_to_tmp, pypath_extended_env, popen_timeout,
scons_platform_options, capfd):
- sub_p = PopenHelper(r'scons --usetool=DoxygenBuilder --doxygen-only hello' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --usetool=DoxygenBuilder --doxygen-only hello' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'done building targets.' in captured.out
assert re.search('BUILD_TARGETS.*doxygen', captured.out)
@@ -103,11 +117,13 @@ def test_SConsiderStaticProgDoxygenOnly(copy_testdir_to_tmp, pypath_extended_env
@pytest.mark.invocation
def test_SConsiderStaticProgIncludingDoxygen(copy_testdir_to_tmp, pypath_extended_env, popen_timeout,
scons_platform_options, capfd):
- sub_p = PopenHelper(r'scons --usetool=DoxygenBuilder --doxygen hello' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --usetool=DoxygenBuilder --doxygen hello' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'done building targets.' in captured.out
assert re.search('BUILD_TARGETS.*doxygen', captured.out)
@@ -117,11 +133,13 @@ def test_SConsiderStaticProgIncludingDoxygen(copy_testdir_to_tmp, pypath_extende
@pytest.mark.invocation
def test_SConsiderStaticProgRunWithoutCommandLineTarget(copy_testdir_to_tmp, pypath_extended_env,
popen_timeout, scons_platform_options, capfd):
- sub_p = PopenHelper(r'scons --3rdparty= --run' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --3rdparty= --run' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'Hello from SConsider' in captured.out
@@ -129,11 +147,13 @@ def test_SConsiderStaticProgRunWithoutCommandLineTarget(copy_testdir_to_tmp, pyp
@pytest.mark.invocation
def test_SConsiderStaticProgRunWithPackageAsTarget(copy_testdir_to_tmp, pypath_extended_env, popen_timeout,
scons_platform_options, capfd):
- sub_p = PopenHelper(r'scons --3rdparty= --run hello' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --3rdparty= --run hello' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'Hello from SConsider' in captured.out
@@ -141,11 +161,13 @@ def test_SConsiderStaticProgRunWithPackageAsTarget(copy_testdir_to_tmp, pypath_e
@pytest.mark.invocation
def test_SConsiderStaticProgRunWithExplicitPackageTarget(copy_testdir_to_tmp, pypath_extended_env,
popen_timeout, scons_platform_options, capfd):
- sub_p = PopenHelper(r'scons --3rdparty= --run hello.runner' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --3rdparty= --run hello.runner' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'Hello from SConsider' in captured.out
@@ -162,11 +184,13 @@ def assert_outputfiles_exist(baseoutdir, predicate=lambda l: l >= 1):
@pytest.mark.invocation
def test_SConsiderStaticProgBuildOutputFilesBelowStartdir(copy_testdir_to_tmp, pypath_extended_env,
popen_timeout, scons_platform_options):
- sub_p = PopenHelper(r'scons --3rdparty= --run hello.runner' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --3rdparty= --run hello.runner' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
assert copy_testdir_to_tmp.join('apps').isdir()
assert copy_testdir_to_tmp.join('progdir').join('.build').isdir()
assert_outputfiles_exist(str(copy_testdir_to_tmp))
@@ -177,12 +201,13 @@ def test_SConsiderStaticProgBuildOutputFilesInBaseoutdir(copy_testdir_to_tmp, py
popen_timeout, scons_platform_options,
tmpdir_factory):
baseoutdir = str(tmpdir_factory.mktemp('baseoutdir', numbered=True))
- sub_p = PopenHelper(r'scons --3rdparty= --baseoutdir=' + baseoutdir + ' --run hello.runner' +
- scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --3rdparty= --baseoutdir=' + baseoutdir + ' --run hello.runner' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
assert not copy_testdir_to_tmp.join('apps').isdir()
assert not copy_testdir_to_tmp.join('progdir').join('.build').isdir()
assert_outputfiles_exist(str(copy_testdir_to_tmp), lambda l: l == 0)
@@ -198,17 +223,23 @@ def test_SConsiderThirdPartyBuildOnceOnly(copy_testdir_to_tmp, pypath_extended_e
cpp_in_second_build, capfd):
"""Show that using the source directory name as prefix of the 3rdparty
build output always builds the sources over and over again."""
- cmdline = r'scons --3rdparty=my3pscons --with-src-3plib=my3psrc' + scons_platform_options
+ cmd = r'scons --3rdparty=my3pscons --with-src-3plib=my3psrc' + scons_platform_options
if buildprefix:
- cmdline += ' --3rdparty-build-prefix=' + buildprefix
- sub_p = PopenHelper(cmdline, cwd=str(copy_testdir_to_tmp), env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd += ' --3rdparty-build-prefix=' + buildprefix
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert os.sep + '3plib.cpp' in captured.out
- sub_p = PopenHelper(cmdline, cwd=str(copy_testdir_to_tmp), env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert cpp_in_second_build(os.sep + '3plib.cpp' in captured.out)
@@ -217,11 +248,13 @@ def test_SConsiderThirdPartyBuildOnceOnly(copy_testdir_to_tmp, pypath_extended_e
@pytest.mark.parametrize('current_testdir', ['samedirtest'])
def test_SConstructAndSConsiderInSameDirBuild(copy_testdir_to_tmp, pypath_extended_env, popen_timeout,
scons_platform_options, capfd):
- sub_p = PopenHelper(r'scons --3rdparty=' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --3rdparty=' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'done building targets.' in captured.out
@@ -231,10 +264,12 @@ def test_SConstructAndSConsiderInSameDirBuild(copy_testdir_to_tmp, pypath_extend
def test_SConstructAndSConsiderInSameDirRunWithoutCommandLineTarget(copy_testdir_to_tmp, pypath_extended_env,
popen_timeout, scons_platform_options,
capfd):
- sub_p = PopenHelper(r'scons --3rdparty= --run' + scons_platform_options,
- cwd=str(copy_testdir_to_tmp),
- env=pypath_extended_env)
- sub_p.communicate(timeout=popen_timeout)
- assert 0 == sub_p.returncode
+ cmd = r'scons --3rdparty= --run' + scons_platform_options
+ with ProcessRunner(cmd, timeout=popen_timeout, cwd=str(copy_testdir_to_tmp),
+ env=pypath_extended_env) as executor:
+ for out, err in executor:
+ sys.stdout.write(out)
+ sys.stderr.write(err)
+ assert 0 == executor.returncode
captured = capfd.readouterr()
assert 'Hello from SConsider' in captured.out