# Copyright 2008 Google Inc. All Rights Reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Provides facilities for running SCons-built Google Test/Mock tests.""" import optparse import os import re import sets import sys try: # subrocess module is a preferable way to invoke subprocesses but it may # not be available on MacOS X 10.4. # Suppresses the 'Import not at the top of the file' lint complaint. # pylint: disable-msg=C6204 import subprocess except ImportError: subprocess = None HELP_MSG = """Runs the specified tests for %(proj)s. SYNOPSIS run_tests.py [OPTION]... [BUILD_DIR]... [TEST]... DESCRIPTION Runs the specified tests (either binary or Python), and prints a summary of the results. BUILD_DIRS will be used to search for the binaries. If no TESTs are specified, all binary tests found in BUILD_DIRs and all Python tests found in the directory test/ (in the %(proj)s root) are run. TEST is a name of either a binary or a Python test. A binary test is an executable file named *_test or *_unittest (with the .exe extension on Windows) A Python test is a script named *_test.py or *_unittest.py. OPTIONS -h, --help Print this help message. -c CONFIGURATIONS Specify build directories via build configurations. CONFIGURATIONS is either a comma-separated list of build configurations or 'all'. Each configuration is equivalent to adding 'scons/build//%(proj)s/scons' to BUILD_DIRs. Specifying -c=all is equivalent to providing all directories listed in KNOWN BUILD DIRECTORIES section below. -a Equivalent to -c=all -b Equivalent to -c=all with the exception that the script will not fail if some of the KNOWN BUILD DIRECTORIES do not exists; the script will simply not run the tests there. 'b' stands for 'built directories'. RETURN VALUE Returns 0 if all tests are successful; otherwise returns 1. EXAMPLES run_tests.py Runs all tests for the default build configuration. run_tests.py -a Runs all tests with binaries in KNOWN BUILD DIRECTORIES. run_tests.py -b Runs all tests in KNOWN BUILD DIRECTORIES that have been built. run_tests.py foo/ Runs all tests in the foo/ directory and all Python tests in the directory test. The Python tests are instructed to look for binaries in foo/. run_tests.py bar_test.exe test/baz_test.exe foo/ bar/ Runs foo/bar_test.exe, bar/bar_test.exe, foo/baz_test.exe, and bar/baz_test.exe. run_tests.py foo bar test/foo_test.py Runs test/foo_test.py twice instructing it to look for its test binaries in the directories foo and bar, correspondingly. KNOWN BUILD DIRECTORIES run_tests.py knows about directories where the SCons build script deposits its products. These are the directories where run_tests.py will be looking for its binaries. Currently, %(proj)s's SConstruct file defines them as follows (the default build directory is the first one listed in each group): On Windows: <%(proj)s root>/scons/build/win-dbg8/%(proj)s/scons/ <%(proj)s root>/scons/build/win-opt8/%(proj)s/scons/ <%(proj)s root>/scons/build/win-dbg/%(proj)s/scons/ <%(proj)s root>/scons/build/win-opt/%(proj)s/scons/ On Mac: <%(proj)s root>/scons/build/mac-dbg/%(proj)s/scons/ <%(proj)s root>/scons/build/mac-opt/%(proj)s/scons/ On other platforms: <%(proj)s root>/scons/build/dbg/%(proj)s/scons/ <%(proj)s root>/scons/build/opt/%(proj)s/scons/""" IS_WINDOWS = os.name == 'nt' IS_MAC = os.name == 'posix' and os.uname()[0] == 'Darwin' IS_CYGWIN = os.name == 'posix' and 'CYGWIN' in os.uname()[0] # Definition of CONFIGS must match that of the build directory names in the # SConstruct script. The first list item is the default build configuration. if IS_WINDOWS: CONFIGS = ('win-dbg8', 'win-opt8', 'win-dbg', 'win-opt') elif IS_MAC: CONFIGS = ('mac-dbg', 'mac-opt') else: CONFIGS = ('dbg', 'opt') if IS_WINDOWS or IS_CYGWIN: PYTHON_TEST_REGEX = re.compile(r'_(unit)?test\.py$', re.IGNORECASE) BINARY_TEST_REGEX = re.compile(r'_(unit)?test(\.exe)?$', re.IGNORECASE) BINARY_TEST_SEARCH_REGEX = re.compile(r'_(unit)?test\.exe$', re.IGNORECASE) else: PYTHON_TEST_REGEX = re.compile(r'_(unit)?test\.py$') BINARY_TEST_REGEX = re.compile(r'_(unit)?test$') BINARY_TEST_SEARCH_REGEX = BINARY_TEST_REGEX def _GetGtestBuildDir(injected_os, script_dir, config): """Calculates path to the Google Test SCons build directory.""" return injected_os.path.normpath(injected_os.path.join(script_dir, 'scons/build', config, 'gtest/scons')) # All paths in this script are either absolute or relative to the current # working directory, unless otherwise specified. class TestRunner(object): """Provides facilities for running Python and binary tests for Google Test.""" def __init__(self, script_dir, build_dir_var_name='GTEST_BUILD_DIR', injected_os=os, injected_subprocess=subprocess, injected_build_dir_finder=_GetGtestBuildDir): """Initializes a TestRunner instance. Args: script_dir: File path to the calling script. build_dir_var_name: Name of the env variable used to pass the the build directory path to the invoked tests. injected_os: standard os module or a mock/stub for testing. injected_subprocess: standard subprocess module or a mock/stub for testing injected_build_dir_finder: function that determines the path to the build directory. """ self.os = injected_os self.subprocess = injected_subprocess self.build_dir_finder = injected_build_dir_finder self.build_dir_var_name = build_dir_var_name self.script_dir = script_dir def _GetBuildDirForConfig(self, config): """Returns the build directory for a given configuration.""" return self.build_dir_finder(self.os, self.script_dir, config) def _Run(self, args): """Runs the executable with given args (args[0] is the executable name). Args: args: Command line arguments for the process. Returns: Process's exit code if it exits normally, or -signal if the process is killed by a signal. """ if self.subprocess: return self.subprocess.Popen(args).wait() else: return self.os.spawnv(self.os.P_WAIT, args[0], args) def _RunBinaryTest(self, test): """Runs the binary test given its path. Args: test: Path to the test binary. Returns: Process's exit code if it exits normally, or -signal if the process is killed by a signal. """ return self._Run([test]) def _RunPythonTest(self, test, build_dir): """Runs the Python test script with the specified build directory. Args: test: Path to the test's Python script. build_dir: Path to the directory where the test binary is to be found. Returns: Process's exit code if it exits normally, or -signal if the process is killed by a signal. """ old_build_dir = self.os.environ.get(self.build_dir_var_name) try: self.os.environ[self.build_dir_var_name] = build_dir # If this script is run on a Windows machine that has no association # between the .py extension and a python interpreter, simply passing # the script name into subprocess.Popen/os.spawn will not work. print 'Running %s . . .' % (test,) return self._Run([sys.executable, test]) finally: if old_build_dir is None: del self.os.environ[self.build_dir_var_name] else: self.os.environ[self.build_dir_var_name] = old_build_dir def _FindFilesByRegex(self, directory, regex): """Returns files in a directory whose names match a regular expression. Args: directory: Path to the directory to search for files. regex: Regular expression to filter file names. Returns: The list of the paths to the files in the directory. """ return [self.os.path.join(directory, file_name) for file_name in self.os.listdir(directory) if re.search(regex, file_name)] # TODO(vladl@google.com): Implement parsing of scons/SConscript to run all # tests defined there when no tests are specified. # TODO(vladl@google.com): Update the docstring after the code is changed to # try to test all builds defined in scons/SConscript. def GetTestsToRun(self, args, named_configurations, built_configurations, available_configurations=CONFIGS): """Determines what tests should be run. Args: args: The list of non-option arguments from the command line. named_configurations: The list of configurations specified via -c or -a. built_configurations: True if -b has been specified. available_configurations: a list of configurations available on the current platform, injectable for testing. Returns: A tuple with 2 elements: the list of Python tests to run and the list of binary tests to run. """ if named_configurations == 'all': named_configurations = ','.join(available_configurations) normalized_args = [self.os.path.normpath(arg) for arg in args] # A final list of build directories which will be searched for the test # binaries. First, add directories specified directly on the command # line. build_dirs = filter(self.os.path.isdir, normalized_args) # Adds build directories specified via their build configurations using # the -c or -a options. if named_configurations: build_dirs += [self._GetBuildDirForConfig(config) for config in named_configurations.split(',')] # Adds KNOWN BUILD DIRECTORIES if -b is specified. if built_configurations: build_dirs += [self._GetBuildDirForConfig(config) for config in available_configurations if self.os.path.isdir(self._GetBuildDirForConfig(config))] # If no directories were specified either via -a, -b, -c, or directly, use # the default configuration. elif not build_dirs: build_dirs = [self._GetBuildDirForConfig(available_configurations[0])] # Makes sure there are no duplications. build_dirs = sets.Set(build_dirs) errors_found = False listed_python_tests = [] # All Python tests listed on the command line. listed_binary_tests = [] # All binary tests listed on the command line. test_dir = self.os.path.normpath(self.os.path.join(self.script_dir, 'test')) # Sifts through non-directory arguments fishing for any Python or binary # tests and detecting errors. for argument in sets.Set(normalized_args) - build_dirs: if re.search(PYTHON_TEST_REGEX, argument): python_path = self.os.path.join(test_dir, self.os.path.basename(argument)) if self.os.path.isfile(python_path): listed_python_tests.append(python_path) else: sys.stderr.write('Unable to find Python test %s' % argument) errors_found = True elif re.search(BINARY_TEST_REGEX, argument): # This script also accepts binary test names prefixed with test/ for # the convenience of typing them (can use path completions in the # shell). Strips test/ prefix from the binary test names. listed_binary_tests.append(self.os.path.basename(argument)) else: sys.stderr.write('%s is neither test nor build directory' % argument) errors_found = True if errors_found: return None user_has_listed_tests = listed_python_tests or listed_binary_tests if user_has_listed_tests: selected_python_tests = listed_python_tests else: selected_python_tests = self._FindFilesByRegex(test_dir, PYTHON_TEST_REGEX) # TODO(vladl@google.com): skip unbuilt Python tests when -b is specified. python_test_pairs = [] for directory in build_dirs: for test in selected_python_tests: python_test_pairs.append((directory, test)) binary_test_pairs = [] for directory in build_dirs: if user_has_listed_tests: binary_test_pairs.extend( [(directory, self.os.path.join(directory, test)) for test in listed_binary_tests]) else: tests = self._FindFilesByRegex(directory, BINARY_TEST_SEARCH_REGEX) binary_test_pairs.extend([(directory, test) for test in tests]) return (python_test_pairs, binary_test_pairs) def RunTests(self, python_tests, binary_tests): """Runs Python and binary tests and reports results to the standard output. Args: python_tests: List of Python tests to run in the form of tuples (build directory, Python test script). binary_tests: List of binary tests to run in the form of tuples (build directory, binary file). Returns: The exit code the program should pass into sys.exit(). """ if python_tests or binary_tests: results = [] for directory, test in python_tests: results.append((directory, test, self._RunPythonTest(test, directory) == 0)) for directory, test in binary_tests: results.append((directory, self.os.path.basename(test), self._RunBinaryTest(test) == 0)) failed = [(directory, test) for (directory, test, success) in results if not success] print print '%d tests run.' % len(results) if failed: print 'The following %d tests failed:' % len(failed) for (directory, test) in failed: print '%s in %s' % (test, directory) return 1 else: print 'All tests passed!' else: # No tests defined print 'Nothing to test - no tests specified!' return 0 def ParseArgs(project_name, argv=None, help_callback=None): """Parses the options run_tests.py uses.""" # Suppresses lint warning on unused arguments. These arguments are # required by optparse, even though they are unused. # pylint: disable-msg=W0613 def PrintHelp(option, opt, value, parser): print HELP_MSG % {'proj': project_name} sys.exit(1) parser = optparse.OptionParser() parser.add_option('-c', action='store', dest='configurations', default=None) parser.add_option('-a', action='store_const', dest='configurations', default=None, const='all') parser.add_option('-b', action='store_const', dest='built_configurations', default=False, const=True) # Replaces the built-in help with ours. parser.remove_option('-h') parser.add_option('-h', '--help', action='callback', callback=help_callback or PrintHelp) return parser.parse_args(argv)