com pftt2: test same scenario set and test-pack several times: conf/app/drupal.groovy conf/app/phpbb3.groovy conf/app/phpmyadmin.groovy
conf/app/semantic_mediawiki.groovy conf/app/wordpress.g roovy conf/task/code_coverage.groovy conf/task/code_p rofile.groovy
conf/task/each_times10.groovy conf/ task/each_times20.groovy conf/task/each_times50 .groovy conf/task/gdb.groovy conf/task/list_builtin
_functions.groovy conf/task/repeat10.groovy conf/ task/repea

Commit: 8eeb03426f61a493a0bd8050d601566db7237ab5 Author: Matt Ficken <v-mafick@microsoft.com> Thu, 11 Jul 2013 15:49:23 -0700 Parents: bf01edeb8553031324caa77d8be79803bc0fbd4d Branches: master Link: http://git.php.net/?p=pftt2.git;a=commitdiff;h=8eeb03426f61a493a0bd8050d601566db7237ab5 Log: test same scenario set and test-pack several times Changed paths: M conf/app/drupal.groovy M conf/app/phpbb3.groovy M conf/app/phpmyadmin.groovy M conf/app/semantic_mediawiki.groovy M conf/app/wordpress.groovy M conf/task/code_coverage.groovy M conf/task/code_profile.groovy A conf/task/each_times10.groovy A conf/task/each_times20.groovy A conf/task/each_times50.groovy A conf/task/gdb.groovy M conf/task/list_builtin_functions.groovy D conf/task/repeat10.groovy D conf/task/repeat100.groovy D conf/task/repeat1000.groovy M conf/task/smokey.groovy A conf/task/test_pack10.groovy A conf/task/test_pack20.groovy A conf/task/test_pack5.groovy A conf/task/test_pack50.groovy A conf/task/valgrind.groovy A conf/task/windbg.groovy M conf/xdebug.groovy M lib/php_parser.jar M src/com/github/mattficken/io/ArrayUtil.java M src/com/mostc/pftt/host/LocalHost.java M src/com/mostc/pftt/host/PSCAgentServer.java M src/com/mostc/pftt/host/RemotePhptTestPackRunner.java M src/com/mostc/pftt/main/Config.java M src/com/mostc/pftt/main/PfttAgentMain.java M src/com/mostc/pftt/main/PfttMain.java M src/com/mostc/pftt/model/app/PhpUnitSourceTestPack.java M src/com/mostc/pftt/model/app/PhpUnitTemplate.groovy M src/com/mostc/pftt/model/core/PhpParser.java M src/com/mostc/pftt/results/AbstractPhpUnitRW.java M src/com/mostc/pftt/results/AbstractPhptRW.java M src/com/mostc/pftt/results/CodeCoverageSummary.java M src/com/mostc/pftt/results/ConsoleManager.java A src/com/mostc/pftt/results/ISerializer.java M src/com/mostc/pftt/results/LocalConsoleManager.java M src/com/mostc/pftt/results/PhpResultPackWriter.java M src/com/mostc/pftt/results/PhpUnitResultWriter.java M src/com/mostc/pftt/results/PhpUnitTestResult.java M src/com/mostc/pftt/results/PhptResultWriter.java M src/com/mostc/pftt/results/PhptTestResult.java M src/com/mostc/pftt/results/TestCaseCodeCoverage.java M src/com/mostc/pftt/runner/AbstractLocalTestPackRunner.java M src/com/mostc/pftt/runner/LocalPhpUnitTestPackRunner.java M src/com/mostc/pftt/runner/LocalPhptTestPackRunner.java M src/com/mostc/pftt/scenario/BuiltinWebServerScenario.java M src/com/mostc/pftt/scenario/SAPIScenario.java A src/com/mostc/pftt/scenario/XDebugProfilingScenario.java M src/com/mostc/pftt/scenario/XDebugScenario.java M src/com/mostc/pftt/util/ValgrindMemoryCheckManager.java

diff --git a/conf/app/drupal.groovy b/conf/app/drupal.groovy index d1ec682..be0fc27 100644 --- a/conf/app/drupal.groovy +++ b/conf/app/drupal.groovy @@ -29,7 +29,7 @@ class DrupalPhpUnitTestPack extends PhpUnitSourceTestPack { } @Override - protected boolean isFileNameATest(String file_name) { + public boolean isFileNameATest(String file_name) { // many apps/frameworks name their test files Test*.php. // wordpress does not ... check all .php files for PhpUnit test case classes. return file_name.endsWith(".php"); diff --git a/conf/app/phpbb3.groovy b/conf/app/phpbb3.groovy index 6f1663f..6c582cb 100644 --- a/conf/app/phpbb3.groovy +++ b/conf/app/phpbb3.groovy @@ -32,7 +32,7 @@ class PhpBB3PhpUnitTestPack extends PhpUnitSourceTestPack { } @Override - protected boolean isFileNameATest(String file_name) { + public boolean isFileNameATest(String file_name) { return file_name.endsWith(".php"); } diff --git a/conf/app/phpmyadmin.groovy b/conf/app/phpmyadmin.groovy index 9d82cda..de45c6f 100644 --- a/conf/app/phpmyadmin.groovy +++ b/conf/app/phpmyadmin.groovy @@ -25,7 +25,7 @@ class PhpMyAdminPhpUnitTestPack extends PhpUnitSourceTestPack { } @Override - protected boolean isFileNameATest(String file_name) { + public boolean isFileNameATest(String file_name) { return file_name.endsWith(".php"); } diff --git a/conf/app/semantic_mediawiki.groovy b/conf/app/semantic_mediawiki.groovy index 1761e22..413b3e2 100644 --- a/conf/app/semantic_mediawiki.groovy +++ b/conf/app/semantic_mediawiki.groovy @@ -34,7 +34,7 @@ class SemanticMediaWikiPhpUnitTestPack extends DatabasePhpUnitSourceTestPack { } @Override - protected boolean isFileNameATest(String file_name) { + public boolean isFileNameATest(String file_name) { return file_name.endsWith(".php"); } diff --git a/conf/app/wordpress.groovy b/conf/app/wordpress.groovy index 80b2000..7fda3b9 100644 --- a/conf/app/wordpress.groovy +++ b/conf/app/wordpress.groovy @@ -33,7 +33,7 @@ class WordpressPhpUnitTestPack extends DatabasePhpUnitSourceTestPack { } @Override - protected boolean isFileNameATest(String file_name) { + public boolean isFileNameATest(String file_name) { // many apps/frameworks name their test files Test*.php. // wordpress does not ... check all .php files for PhpUnit test case classes. return file_name.endsWith(".php"); diff --git a/conf/task/code_coverage.groovy b/conf/task/code_coverage.groovy index 53d8ef8..fe538d9 100644 --- a/conf/task/code_coverage.groovy +++ b/conf/task/code_coverage.groovy @@ -4,7 +4,10 @@ def describe() { } def scenarios() { + // @see PhpUnitTemplate#renderTemplate + // -it will now collect the code coverage data + // -and the test case runner will provide it to the test-pack which will store it + // new XDebugScenario() } -// TODO note code coverage data stored in each test result diff --git a/conf/task/code_profile.groovy b/conf/task/code_profile.groovy index 52b0832..d57b076 100644 --- a/conf/task/code_profile.groovy +++ b/conf/task/code_profile.groovy @@ -1,7 +1,8 @@ -// TODO -// see http://xdebug.org/docs/profiler - def describe() { "Profile code to get run-times and find slow code points using Xdebug (Dynamic Run-Time Analysis)" } + +def scenarios() { + new XDebugProfilingScenario() +} diff --git a/conf/task/each_times10.groovy b/conf/task/each_times10.groovy new file mode 100644 index 0000000..33480f6 --- /dev/null +++ b/conf/task/each_times10.groovy @@ -0,0 +1,9 @@ + +def describe() { + "run each test 10 times" +} + +def processConsoleOptions(List options) { + options.add("-run_test_times_all") + options.add("10") +} diff --git a/conf/task/each_times20.groovy b/conf/task/each_times20.groovy new file mode 100644 index 0000000..ffef508 --- /dev/null +++ b/conf/task/each_times20.groovy @@ -0,0 +1,9 @@ + +def describe() { + "run each test 20 times" +} + +def processConsoleOptions(List options) { + options.add("-run_test_times_all") + options.add("20") +} diff --git a/conf/task/each_times50.groovy b/conf/task/each_times50.groovy new file mode 100644 index 0000000..909e003 --- /dev/null +++ b/conf/task/each_times50.groovy @@ -0,0 +1,9 @@ + +def describe() { + "run each test 50 times" +} + +def processConsoleOptions(List options) { + options.add("-run_test_times_all") + options.add("50") +} diff --git a/conf/task/gdb.groovy b/conf/task/gdb.groovy new file mode 100644 index 0000000..3e09597 --- /dev/null +++ b/conf/task/gdb.groovy @@ -0,0 +1,8 @@ + +def describe() { + "Run with GDB" +} + +def processConsoleOptions(List options) { + options.add("-debugger")options.add("gdb") +} diff --git a/conf/task/list_builtin_functions.groovy b/conf/task/list_builtin_functions.groovy index 4f7b0e0..f1a256a 100644 --- a/conf/task/list_builtin_functions.groovy +++ b/conf/task/list_builtin_functions.groovy @@ -5,74 +5,114 @@ // lists all builtin functions used in symfony's PhpUnit tests // parses symfony_demo_app.php to find all builtin functions // generates lists: -// 1. ANY LIST phpunit tests that use any builtin function also used in symfony_demo_app.php -// 2. IN-ORDER LIST phpunit tests that use builtin functions in same order as symfony_demo_app.php +// 1. ALPHABETICAL LIST phpunit tests that use any builtin function also used in symfony_demo_app.php +// 2. CALL-ORDER LIST phpunit tests that use builtin functions in same order as symfony_demo_app.php // -can also generate PHPT tests from this list // make_core_test.groovy task - +import com.mostc.pftt.results.PhptTestResult; def describe() { - "Adds list of the builtin functions test cases use to the result-pack (Static Analysis)" + """Adds list of the builtin functions test cases use to the result-pack (Static Analysis). + +The hard part of code analysis is navigating all the data, this helps by listing only the builtin functions""" } -// TODO for performance comparison, WHAT ABOUT SIZE OF PHP CODE RUN? (maybe that differs on each OS) +def scenarios() { + // need this scenario to get the code coverage data + new XDebugScenario() +} -// TODO store output of these 2 functions for each test, in result-pack +def processConsoleOptions(List options) { + // threads take more CPU time to do the analysis, etc... + // so run fewer threads + options.add("-thread_count"); + options.add(""+( 1 * new LocalHost().getCPUCount())); +} -def storeTestResult() { - +def processUITestResult() { + // TODO implement (Config doesn't support this function also) } -// -// tells: which functions were called OR which functions could be called (if test wasn't run (see `parse` command)) -String getBuiltinFunctionNamesAlphabetical() { - List func_names; - - TestCaseCodeCoverage test_coverage; // TODO - if (test_coverage==null) { - // called using `parse` command - - for ( Function f : parseAST(php_code).getFunctions() ) { - if (!func_names.contains(f.getName())) { - func_names.add(f.getName()); - } - } - } else { - // called using `core_all`, `app_all`, etc... - - - for ( String file_name : test_coverage.getFileNames() ) { - String php_code = test_coverage.getPhpCode(file_name); - - for ( Function f : parseAST(php_code).getFunctions() ) { - if (test_coverage.isExecuted(file_name, f.getLineNum())) { - if (!func_names.contains(f.getName())) { - func_names.add(f.getName()); - } - } - } - } // end for - } // end if - return func_names; +// this called for each phpt result +def processPhptTestResult(ConsoleManager cm, PhptTestResult result) { + if (cm.isNoResultFileForPassSkipXSkip() && !PhptTestResult.shouldStoreAllInfo(result.status)) { + // don't store builtin functions for this result + // + // don't store code coverage for this result either + result.code_coverage = null + return; + } + result.extra = new BuiltinFunctionSerializer(code_coverage: result.code_coverage) } -// tells: which functions were called and how frequently (and the order) -String getBuiltinFunctionNamesInCallOrder() { - // need code coverage data for this - TestCaseCodeCoverage test_coverage; // TODO - - List func_names; +// this called for each phpunit result +def processPhpUnitTestResult(ConsoleManager cm, PhpUnitTestResult result) { + if (cm.isNoResultFileForPassSkipXSkip() && !PhpUnitTestResult.shouldStoreAllInfo(result.status)) { + // don't store builtin functions for this result + // + // don't store code coverage for this result either + result.code_coverage = null + return; + } + result.extra = new BuiltinFunctionSerializer(code_coverage: result.code_coverage) +} + +import com.mostc.pftt.model.core.PhpParser.*; + +// gets the builtin functions from the code coverage, and serializes them +class BuiltinFunctionSerializer implements ISerializer { + TestCaseCodeCoverage code_coverage; - for ( String file_name : test_coverage.getFileNames() ) { - String php_code = test_coverage.getPhpCode(file_name); - - for ( Function f : parseAST(php_code).getFunctions() ) { - if (test_coverage.isExecuted(file_name, f.getLineNum())) { - func_names.add(f.getName()); + public void serial(XmlSerializer serial) { + List functions = [] + List func_names = [] + if (code_coverage==null) { + // TODO called using `parse` command + } else { + // First, read all called functions + for ( String file_name : code_coverage.getFileNames() ) { + String php_code = code_coverage.getPhpCode(file_name); + if (php_code==null) + continue; + // important: pass file_name to #parseScript (or it won't know if code was executed) + for ( def function : PhpParser.parseScript(php_code, file_name).getBuiltinFunctions(code_coverage) ) { + functions.add(function) + } + } + + + // Second, get names alphabetically + for ( def function : functions ) { + func_names.add(function.getFunctionName()) + } + // remove duplicate function names + func_names = ArrayUtil.copyNoDuplicates(func_names) + // alphabetize + Collections.sort(func_names) + + // record + serial.startTag("pftt", "builtinFunctionsAlphabetical") + for ( String func_name : func_names ) { + serial.startTag("pftt", "builtin") + serial.attribute("pftt", "function", func_name) + serial.endTag("pftt", "builtin") } + serial.endTag("pftt", "builtinFunctionsAlphabetical") + + + // Third, store function names in call order + serial.startTag("pftt", "builtinFunctionsInCallOrder") + for ( def function : functions ) { + serial.startTag("pftt", "builtin"); + serial.attribute("pftt", "function", function.getFunctionName()); + // store location of function call + serial.attribute("pftt", "file", function.getFileName()); + serial.attribute("pftt", "line", Integer.toString(function.getLineNumber())); + serial.endTag("pftt", "builtin"); + } + serial.endTag("pftt", "builtinFunctionsInCallOrder") } - } - - return func_names; -} + } // end public void serial + +} // end class BuiltinFunctionSerializer diff --git a/conf/task/repeat10.groovy b/conf/task/repeat10.groovy deleted file mode 100644 index 33480f6..0000000 --- a/conf/task/repeat10.groovy +++ /dev/null @@ -1,9 +0,0 @@ - -def describe() { - "run each test 10 times" -} - -def processConsoleOptions(List options) { - options.add("-run_test_times_all") - options.add("10") -} diff --git a/conf/task/repeat100.groovy b/conf/task/repeat100.groovy deleted file mode 100644 index 0275bae..0000000 --- a/conf/task/repeat100.groovy +++ /dev/null @@ -1,9 +0,0 @@ - -def describe() { - "run each test 100 times" -} - -def processConsoleOptions(List options) { - options.add("-run_test_times_all") - options.add("100") -} diff --git a/conf/task/repeat1000.groovy b/conf/task/repeat1000.groovy deleted file mode 100644 index 9fb81de..0000000 --- a/conf/task/repeat1000.groovy +++ /dev/null @@ -1,9 +0,0 @@ - -def describe() { - "run each test 1000 times" -} - -def processConsoleOptions(List options) { - options.add("-run_test_times_all") - options.add("1000") -} diff --git a/conf/task/smokey.groovy b/conf/task/smokey.groovy index 1d900e6..a2d3cef 100644 --- a/conf/task/smokey.groovy +++ b/conf/task/smokey.groovy @@ -7,6 +7,6 @@ Helps find bad builds that the regular smoke tests won't detect.""" def processConsoleOptions(List options) { options.add("-run_count"); - options.add("1000"); + options.add("500"); options.add("-randomize_order") } diff --git a/conf/task/test_pack10.groovy b/conf/task/test_pack10.groovy new file mode 100644 index 0000000..16dc87b --- /dev/null +++ b/conf/task/test_pack10.groovy @@ -0,0 +1,9 @@ + +def describe() { + "run test-pack 10 times" +} + +def processConsoleOptions(List options) { + options.add("-run_test_pack") + options.add("10") +} diff --git a/conf/task/test_pack20.groovy b/conf/task/test_pack20.groovy new file mode 100644 index 0000000..27bd963 --- /dev/null +++ b/conf/task/test_pack20.groovy @@ -0,0 +1,9 @@ + +def describe() { + "run test-pack 20 times" +} + +def processConsoleOptions(List options) { + options.add("-run_test_pack") + options.add("20") +} diff --git a/conf/task/test_pack5.groovy b/conf/task/test_pack5.groovy new file mode 100644 index 0000000..aeb758e --- /dev/null +++ b/conf/task/test_pack5.groovy @@ -0,0 +1,9 @@ + +def describe() { + "run test-pack 5 times" +} + +def processConsoleOptions(List options) { + options.add("-run_test_pack") + options.add("5") +} diff --git a/conf/task/test_pack50.groovy b/conf/task/test_pack50.groovy new file mode 100644 index 0000000..5308062 --- /dev/null +++ b/conf/task/test_pack50.groovy @@ -0,0 +1,9 @@ + +def describe() { + "run test-pack 50 times" +} + +def processConsoleOptions(List options) { + options.add("-run_test_pack") + options.add("50") +} diff --git a/conf/task/valgrind.groovy b/conf/task/valgrind.groovy new file mode 100644 index 0000000..db567b8 --- /dev/null +++ b/conf/task/valgrind.groovy @@ -0,0 +1,8 @@ + +def describe() { + "Run with Valgrind" +} + +def processConsoleOptions(List options) { + options.add("-debugger")options.add("valgrind") +} diff --git a/conf/task/windbg.groovy b/conf/task/windbg.groovy new file mode 100644 index 0000000..2e0882d --- /dev/null +++ b/conf/task/windbg.groovy @@ -0,0 +1,8 @@ + +def describe() { + "Run with WinDebug" +} + +def processConsoleOptions(List options) { + options.add("-windbg")options.add("valgrind") +} diff --git a/conf/xdebug.groovy b/conf/xdebug.groovy index 504f435..48c2bf1 100644 --- a/conf/xdebug.groovy +++ b/conf/xdebug.groovy @@ -1,7 +1,7 @@ def describe() { """Load XDebug DLL only (test load)... -to actually use it, use a config task like `code_coverage` instead.""" +to actually use it, use a config task like `code_coverage` or `code_profile` instead.""" } def scenarios() { diff --git a/lib/php_parser.jar b/lib/php_parser.jar index 26597df..2b97f41 100644 Binary files a/lib/php_parser.jar and b/lib/php_parser.jar differ diff --git a/src/com/github/mattficken/io/ArrayUtil.java b/src/com/github/mattficken/io/ArrayUtil.java index 2fc61c4..8a4d692 100644 --- a/src/com/github/mattficken/io/ArrayUtil.java +++ b/src/com/github/mattficken/io/ArrayUtil.java @@ -41,27 +41,71 @@ public class ArrayUtil { return (E[]) Array.newInstance(sample.getClass(), size); } + public static final int NOT_FOUND = -1; + public static <E extends Object> int indexOf(E[] in, E a, int off, int len) { + for ( ; off < len ; off++ ) { + if (in[off].equals(a)) + return off; + } + return NOT_FOUND; + } + + public static <E extends Object> boolean contains(E[] in, E a, int off, int len) { + return indexOf(in, a, off, len) != NOT_FOUND; + } + + public static <E extends Object> List<E> copyNoDuplicates(List<E> list) { + ArrayList<E> out = new ArrayList<E>(list.size()); + copyNoDuplicates(list, out); + return out; + } + + public static <E extends Object> void copyNoDuplicates(List<E> in, List<E> out) { + for ( E e : in ) { + if (!out.contains(e)) + out.add(e); + } + } + + public static <E extends Object> E[] copyNoDuplicates(E[] in) { + if (in==null) + return null; + else if (in.length<2) + return in; + E[] out = newArray(in, in.length); + int i=0; + for ( E e : in ) { + if (!contains(out, e, 0, i)) { + out[i++] = e; + } + } + return out; + } + public static <E extends Object> E[] mergeNoDuplicates(Class<E> clazz, E[] ... lists) { - if (lists==null) + if (lists==null) { return null; - else if (true) + } else if (lists.length==0) { return lists[0]; - - ArrayList<E> out = new ArrayList<E>(lists[0]==null?lists[0].length+16:16); - - for (E[] list : lists) { - if (list==null) - continue; + } else if (lists.length==1) { + return copyNoDuplicates(lists[0]); + } else { + ArrayList<E> out = new ArrayList<E>(lists[0]==null?lists[0].length+16:16); - for (E e : list) { - if (e==null) + for (E[] list : lists) { + if (list==null) continue; - else if (!out.contains(e)) - out.add(e); + + for (E e : list) { + if (e==null) + continue; + else if (!out.contains(e)) + out.add(e); + } } + + return clazz == null ? null : (E[]) out.toArray((E[])Array.newInstance(clazz, out.size())); } - - return clazz == null ? null : (E[]) out.toArray((E[])Array.newInstance(clazz, out.size())); } public static <E extends Object> E[] mergeAllowDuplicates(E[] ... lists) { diff --git a/src/com/mostc/pftt/host/LocalHost.java b/src/com/mostc/pftt/host/LocalHost.java index b76b22c..cbd35f5 100644 --- a/src/com/mostc/pftt/host/LocalHost.java +++ b/src/com/mostc/pftt/host/LocalHost.java @@ -765,6 +765,9 @@ public class LocalHost extends AHost { // start the process try { process = guardStart(builder); + if (process==null && isWindows()) + // try again + process = guardStart(builder); } catch ( IOException ex ) { if (isWindows() && ex.getMessage().contains("Not enough storage")) { // diff --git a/src/com/mostc/pftt/host/PSCAgentServer.java b/src/com/mostc/pftt/host/PSCAgentServer.java index 6f3e3ed..5c1e4c6 100644 --- a/src/com/mostc/pftt/host/PSCAgentServer.java +++ b/src/com/mostc/pftt/host/PSCAgentServer.java @@ -30,7 +30,6 @@ import com.mostc.pftt.results.ITestResultReceiver; import com.mostc.pftt.results.PhpUnitTestResult; import com.mostc.pftt.results.PhptTestResult; import com.mostc.pftt.scenario.Scenario; -import com.mostc.pftt.scenario.ScenarioSet; import com.mostc.pftt.scenario.ScenarioSetSetup; import com.mostc.pftt.util.ErrorUtil; @@ -46,7 +45,7 @@ public abstract class PSCAgentServer implements ConsoleManager, ITestResultRecei protected InputStream parser_in; protected OutputStream serial_out; protected boolean no_result_file_for_pass_xskip_skip, randomize_order, thread_safety; - protected int run_test_times_all = 1, run_test_times_list_times = 1, run_group_times_list_times = 1, run_group_times = 1; + protected int run_test_times_all = 1, run_test_pack = 1, run_test_times_list_times = 1, run_group_times_list_times = 1, run_group_times = 1; protected final LinkedList<String> run_test_times_list, run_group_times_list, skip_list; public PSCAgentServer() { @@ -110,6 +109,7 @@ public abstract class PSCAgentServer implements ConsoleManager, ITestResultRecei no_result_file_for_pass_xskip_skip = StringUtil.equalsCS("true", parser.getAttributeValue(null, "no_result_file_for_pass_xskip_skip")); randomize_order = StringUtil.equalsCS("true", parser.getAttributeValue(null, "randomize_order")); run_test_times_all = StringUtil.parseInt(parser.getAttributeValue(null, "run_test_times_all")); + run_test_pack = StringUtil.parseInt(parser.getAttributeValue(null, "run_test_pack")); thread_safety = StringUtil.equalsCS("true", parser.getAttributeValue(null, "thread_safety")); run_test_times_list_times = StringUtil.parseInt(parser.getAttributeValue(null, "run_test_times_list_times")); run_group_times = StringUtil.parseInt(parser.getAttributeValue(null, "run_group_times")); @@ -224,7 +224,7 @@ public abstract class PSCAgentServer implements ConsoleManager, ITestResultRecei synchronized(serial_out) { // don't do #startDocument -> all it does is print the <?xml header //serial.startDocument("utf-8", Boolean.FALSE); - result.serialize(serial);// TODO , test_name); + result.serial(serial);// TODO , test_name); // important: call #endDocument or all results will be buffered until last result sent serial.endDocument(); serial_out.write('\n'); @@ -294,6 +294,11 @@ public abstract class PSCAgentServer implements ConsoleManager, ITestResultRecei } @Override + public int getRunTestPack() { + return run_test_pack; + } + + @Override public boolean isNoRestartAll() { return false; } diff --git a/src/com/mostc/pftt/host/RemotePhptTestPackRunner.java b/src/com/mostc/pftt/host/RemotePhptTestPackRunner.java index c6fd5b1..4796523 100644 --- a/src/com/mostc/pftt/host/RemotePhptTestPackRunner.java +++ b/src/com/mostc/pftt/host/RemotePhptTestPackRunner.java @@ -53,7 +53,7 @@ public class RemotePhptTestPackRunner extends AbstractRemoteTestPackRunner<PhptA public static void main(String[] args) throws Exception { LocalHost host = new LocalHost(); - LocalConsoleManager cm = new LocalConsoleManager(null, null, false, false, false, false, true, false, true, false, false, false, 1, true, 1, 1, 1, null, null, null, null, false, 0, 0, false, false, 0, 0, 0, false); + LocalConsoleManager cm = new LocalConsoleManager(null, null, false, false, false, false, true, false, true, false, false, false, 1, 1, true, 1, 1, 1, null, null, null, null, false, 0, 0, false, false, 0, 0, 0, false); PhpBuild build = new PhpBuild("C:\\php-sdk\\php-5.5-ts-windows-vc11-x64-re3aeb6c"); build.open(cm, host); @@ -61,7 +61,7 @@ public class RemotePhptTestPackRunner extends AbstractRemoteTestPackRunner<PhptA PhptSourceTestPack test_pack = new PhptSourceTestPack("C:\\php-sdk\\php-test-pack-5.5-ts-windows-vc11-x86-r0704e4b"); test_pack.open(cm, host); - PhpResultPackWriter tmgr = new PhpResultPackWriter(host, cm, new File(host.getPhpSdkDir()), build, test_pack); + PhpResultPackWriter tmgr = new PhpResultPackWriter(host, cm, new File(host.getPhpSdkDir()), build, test_pack, null); // TODO ScenarioSet scenario_set = ScenarioSet.getDefaultScenarioSets().get(0); diff --git a/src/com/mostc/pftt/main/Config.java b/src/com/mostc/pftt/main/Config.java index dfa1cdd..245edee 100644 --- a/src/com/mostc/pftt/main/Config.java +++ b/src/com/mostc/pftt/main/Config.java @@ -17,7 +17,9 @@ import org.codehaus.groovy.control.CompilationFailedException; import org.codehaus.groovy.runtime.metaclass.MissingMethodExceptionNoStack; import org.codehaus.groovy.runtime.metaclass.MissingPropertyExceptionNoStack; import org.columba.ristretto.smtp.SMTPProtocol; +import org.xmlpull.v1.XmlSerializer; +import com.github.mattficken.io.ArrayUtil; import com.github.mattficken.io.IOUtil; import com.github.mattficken.io.StringUtil; import com.mostc.pftt.host.AHost; @@ -31,8 +33,11 @@ import com.mostc.pftt.model.core.PhptTestCase; import com.mostc.pftt.model.ui.UITestPack; import com.mostc.pftt.results.ConsoleManager; import com.mostc.pftt.results.EPrintType; +import com.mostc.pftt.results.ISerializer; import com.mostc.pftt.results.ITestResultReceiver; import com.mostc.pftt.results.PhpResultPackWriter; +import com.mostc.pftt.results.PhpUnitTestResult; +import com.mostc.pftt.results.PhptTestResult; import com.mostc.pftt.scenario.EScenarioSetPermutationLayer; import com.mostc.pftt.scenario.Scenario; import com.mostc.pftt.scenario.ScenarioSet; @@ -102,7 +107,8 @@ import groovy.lang.MetaMethod; * } * def processPHPTTestPack * def processPhpUnitTestPack - * + * def processPhptTestResult + * def processPhpUnitTestResult * * @author Matt Ficken * @@ -125,6 +131,8 @@ public final class Config implements IENVINIFilter { public static final String PROCESS_PHPUNIT_METHOD_NAME = "processPhpUnit"; public static final String PROCESS_PHPT_TEST_PACK_METHOD_NAME = "processPHPTTestPack"; public static final String PROCESS_PHPUNIT_TEST_PACK_METHOD_NAME = "processPhpUnitTestPack"; + public static final String PROCESS_PHPT_TEST_RESULT_METHOD_NAME = "processPhptTestResult"; + public static final String PROCESS_PHP_UNIT_TEST_RESULT_METHOD_NAME = "processPhpUnitTestResult"; // protected final LinkedList<AHost> hosts; protected final LinkedList<Scenario> scenarios; @@ -171,6 +179,24 @@ public final class Config implements IENVINIFilter { } } + public void processPhptTestResult(ConsoleManager cm, PhptTestResult result) { + ArrayList<MethodImpl> methods = by_method_name.get(PROCESS_PHPT_TEST_RESULT_METHOD_NAME); + if (methods==null) + return; + for ( MethodImpl m : methods ) { + invokeMethod(null, null, m.go, PROCESS_PHPT_TEST_RESULT_METHOD_NAME, new Object[]{cm, result}, null); + } + } + + public void processPhpUnitTestResult(ConsoleManager cm, PhpUnitTestResult result) { + ArrayList<MethodImpl> methods = by_method_name.get(PROCESS_PHP_UNIT_TEST_RESULT_METHOD_NAME); + if (methods==null) + return; + for ( MethodImpl m : methods ) { + invokeMethod(null, null, m.go, PROCESS_PHP_UNIT_TEST_RESULT_METHOD_NAME, new Object[]{cm, result}, null); + } + } + public void processPHPT(PhptTestCase test_case) { ArrayList<MethodImpl> methods = by_method_name.get(PROCESS_PHPT_METHOD_NAME); if (methods==null) @@ -578,14 +604,20 @@ public final class Config implements IENVINIFilter { sb.append("import ");sb.append(UITestPack.class.getPackage().getName());sb.append(".*;\n"); sb.append("import ");sb.append(PhpUnitSourceTestPack.class.getPackage().getName());sb.append(".*;\n"); sb.append("import ");sb.append(PhptTestCase.class.getPackage().getName());sb.append(".*;\n"); + sb.append("import ");sb.append(PhptTestResult.class.getPackage().getName());sb.append(".*;\n"); + sb.append("import ");sb.append(PhpUnitTestResult.class.getPackage().getName());sb.append(".*;\n"); sb.append("import ");sb.append(JoomlaScenario.class.getPackage().getName());sb.append(".*;\n"); sb.append("import ");sb.append(Scenario.class.getPackage().getName());sb.append(".*;\n"); sb.append("import ");sb.append(AHost.class.getPackage().getName());sb.append(".*;\n"); sb.append("import ");sb.append(SMTPProtocol.class.getName());sb.append(";\n"); sb.append("import ");sb.append(FTPClient.class.getName());sb.append(";\n"); sb.append("import ");sb.append(StringUtil.class.getName());sb.append(";\n"); + sb.append("import ");sb.append(ArrayUtil.class.getName());sb.append(";\n"); + sb.append("import ");sb.append(ISerializer.class.getName());sb.append(";\n"); + sb.append("import ");sb.append(XmlSerializer.class.getName());sb.append(";\n"); sb.append("import ");sb.append(ConsoleManager.class.getPackage().getName());sb.append(".*;\n"); sb.append("import ");sb.append(EPrintType.class.getName());sb.append(";\n"); + sb.append("import groovy.transform.Field;\n"); if (filename!=null) { sb.append("@Field String __FILE__=\""+AHost.toUnixPath(filename)+"\";"); @@ -710,6 +742,8 @@ public final class Config implements IENVINIFilter { registerMethod(cm, config, go, PROCESS_PHPUNIT_METHOD_NAME, file_name); registerMethod(cm, config, go, PROCESS_PHPT_TEST_PACK_METHOD_NAME, file_name); registerMethod(cm, config, go, PROCESS_PHPUNIT_TEST_PACK_METHOD_NAME, file_name); + registerMethod(cm, config, go, PROCESS_PHPT_TEST_RESULT_METHOD_NAME, file_name); + registerMethod(cm, config, go, PROCESS_PHP_UNIT_TEST_RESULT_METHOD_NAME, file_name); } // end protected static void loadObjectToConfig private static void checkImplemented(ConsoleManager cm, Scenario o) { diff --git a/src/com/mostc/pftt/main/PfttAgentMain.java b/src/com/mostc/pftt/main/PfttAgentMain.java index a351d24..57e3b48 100644 --- a/src/com/mostc/pftt/main/PfttAgentMain.java +++ b/src/com/mostc/pftt/main/PfttAgentMain.java @@ -2,7 +2,6 @@ package com.mostc.pftt.main; import java.util.ArrayList; -import com.mostc.pftt.host.LocalHost; import com.mostc.pftt.host.PSCAgentServer; import com.mostc.pftt.model.core.PhpBuild; import com.mostc.pftt.model.core.PhptActiveTestPack; diff --git a/src/com/mostc/pftt/main/PfttMain.java b/src/com/mostc/pftt/main/PfttMain.java index 81333b5..a202c69 100644 --- a/src/com/mostc/pftt/main/PfttMain.java +++ b/src/com/mostc/pftt/main/PfttMain.java @@ -78,6 +78,11 @@ import com.mostc.pftt.util.WindowsSnapshotDownloadUtil.FindBuildTestPackPair; // real unattended (automated) testing that actually works // the php test tool that you'd actually want to use // doesn't resort to brittle shell scripts +/* -rename ostcpftt@ name to `OSTC PFTT Unattended Test System` + -name still clear who owns it + -plus database field would tell + -only `php` is missing from new name, but any 2nd/3rd party who saw the name wouldn't care about that + -just care about owner*/ // TODO new nsis based installer for windows // TODO linux apache support @@ -85,10 +90,11 @@ import com.mostc.pftt.util.WindowsSnapshotDownloadUtil.FindBuildTestPackPair; // TODO iis support // TODO -debug_all -debugger windbg // TODO -debug_all -debugger valgrind +// so php cli and apache on linux can be compared to windows +// for any windows specific memory problems, etc... // TODO -debug_all -debugger gdb -// TODO task/code_profile -// TODO task/list_builtin_functions -// TODO task/code_coverage +// TODO xdebug only for 5.4-ts +// TODO task/code_coverage store in result-pack // TODO get code coverage data of Symfony demo app (UI?) // -get list of builtin functions // -find phpunit tests that use same builtin functions @@ -99,6 +105,10 @@ import com.mostc.pftt.util.WindowsSnapshotDownloadUtil.FindBuildTestPackPair; // -compare both a, b, c // -additional builtin functions and/or additional php code could be // executed on Windows +// TODO http fuzzing? +// focus on PHPTs with REQUEST, GET, POST, COOKIE sections +// that's data that can be fuzzed to hit different code paths +// fuzzing a GET request to just run some PHP code won't do that // TODO UI testing // no Anon-Logout // TODO filesystem tests w/ non-european locales @@ -170,6 +180,14 @@ import com.mostc.pftt.util.WindowsSnapshotDownloadUtil.FindBuildTestPackPair; // test UI of other applications // // Better-PFTT +// +// what about the times the build servers were down for week+ +// -because you didn't use RAID5 or 6? +// -how hard is that? +// -just scripting VC11, nmake, etc... +// -scripts evolved over many years +// -not really that much effort relative to PFTT +// // get actual version of apache, wordpress, symfony, etc... instead of assuming hardcoded value // if WinDebug isn't found, register VS as the `default postmortem debugger` // optimized for long-term time savings @@ -261,17 +279,24 @@ import com.mostc.pftt.util.WindowsSnapshotDownloadUtil.FindBuildTestPackPair; // (after ported to windows) drizzle weakref fpdf gnupg xdebug? suhosin?? public class PfttMain { - protected LocalHost host; + protected final LocalHost host; protected LocalConsoleManager cm; - protected HashMap<PhpBuild,PhpResultPackWriter> writer_map; + protected final HashMap<PhpBuild,PhpResultPackWriter> writer_map; + protected final Config config; - public PfttMain(LocalConsoleManager cm) { + public PfttMain(LocalConsoleManager cm, Config config) { this.cm = cm; + this.config = config; host = new LocalHost(); writer_map = new HashMap<PhpBuild,PhpResultPackWriter>(); } + public void closeWriter(PhpResultPackWriter tmgr) { + tmgr.close(); + writer_map.remove(tmgr); + } + public PhpResultPackWriter getWriter(PhpBuild build) throws Exception { return getWriter(build, null); } @@ -280,7 +305,7 @@ public class PfttMain { PhpResultPackWriter writer = writer_map.get(build); if (writer!=null) return writer; - writer = new PhpResultPackWriter(host, cm, telem_dir(), build, test_pack); + writer = new PhpResultPackWriter(host, cm, telem_dir(), build, test_pack, config); writer_map.put(build, writer); return writer; } @@ -390,6 +415,8 @@ public class PfttMain { System.out.println(" === Crash Debugging ==="); System.out.println(new AlignedTable(2, 85) .addRow("-debug_all", "runs all tests in Debugger") + .addRow("-debugger <gdb|windbg|valgrind>", "specify which Debugger to use (optional, or PFTT will decide)") + // TODO support name fragments in -debug_list (more consistent with core_name, etc...) .addRow("-debug_list <list files>", "runs tests in list in Debugger (exact name)") .addRow("-src_pack <path>", "folder with the source code") .addRow("-debug_pack <path>", "folder with debugger symbols (usually folder with .pdb files)")); @@ -407,6 +434,7 @@ public class PfttMain { System.out.println(); System.out.println(" === Test Times ==="); System.out.println(new AlignedTable(2, 85) + .addRow("-run_test_pack <N>", "runs entire test-pack N times") .addRow("-run_group_times_all <N>", "runs all groups of tests N times (in same order every time, unless -randomize used)") .addRow("-run_group_times_list <N> <list file>", "just like run_group_times_all and run_test_times_list (but for groups of tests)") .addRow("-run_test_times_all <N>", "runs each test N times in a row/consecutively") @@ -494,24 +522,28 @@ public class PfttMain { cm.println(EPrintType.TIP, PfttMain.class, "Add test-pack configuration file to -c console option. For example `aa -c symfony`"); return; } - for ( PhpUnitSourceTestPack test_pack : test_packs ) { - cm.println(EPrintType.CLUE, PfttMain.class, "Test-Pack: "+test_pack); - cm.println(EPrintType.CLUE, PfttMain.class, "Writing Result-Pack: "+tmgr.getResultPackPath()); - - for ( ScenarioSet scenario_set : getScenarioSets(config, EScenarioSetPermutationLayer.WEB_APPLICATION)) { - - List<AHost> hosts = config.getHosts(); - AHost host = hosts.isEmpty()?this.host:hosts.get(0); - LocalPhpUnitTestPackRunner r = new LocalPhpUnitTestPackRunner(cm, tmgr, scenario_set, build, host, host); - cm.showGUI(r, test_pack); - - // TODO implement app_list - //test_pack.read(test_cases, cm, tmgr, build); - //r.runTestList(test_pack, test_cases); + for ( int i=0 ; i < cm.getRunTestPack() ; i++ ) { + for ( PhpUnitSourceTestPack test_pack : test_packs ) { + cm.println(EPrintType.CLUE, PfttMain.class, "Test-Pack: "+test_pack); + cm.println(EPrintType.CLUE, PfttMain.class, "Writing Result-Pack: "+tmgr.getResultPackPath()); - tmgr.notifyPhpUnitFinished(host, r.getScenarioSetSetup(), test_pack); + for ( ScenarioSet scenario_set : getScenarioSets(config, EScenarioSetPermutationLayer.WEB_APPLICATION)) { - } // end for (scenario_set) + List<AHost> hosts = config.getHosts(); + AHost host = hosts.isEmpty()?this.host:hosts.get(0); + LocalPhpUnitTestPackRunner r = new LocalPhpUnitTestPackRunner(cm, tmgr, scenario_set, build, host, host); + cm.showGUI(r, test_pack); + + // TODO implement app_list + //test_pack.read(test_cases, cm, tmgr, build); + //r.runTestList(test_pack, test_cases); + + tmgr.notifyPhpUnitFinished(host, r.getScenarioSetSetup(), test_pack); + + } // end for (scenario_set) + } + if (cm.getRunTestPack()>1) + closeWriter(tmgr); } } // end public void appList @@ -526,19 +558,23 @@ public class PfttMain { cm.println(EPrintType.CLUE, PfttMain.class, "No test-pack provided by configuration file(s)"); return; } - for ( PhpUnitSourceTestPack test_pack : test_packs ) { - cm.println(EPrintType.CLUE, PfttMain.class, "Test-Pack: "+test_pack); - - cm.println(EPrintType.CLUE, PfttMain.class, "Writing Result-Pack: "+tmgr.getResultPackPath()); - - for (ScenarioSet scenario_set : getScenarioSets(config, EScenarioSetPermutationLayer.WEB_APPLICATION)) { - List<AHost> hosts = config.getHosts(); - AHost host = hosts.isEmpty()?this.host:hosts.get(0); - LocalPhpUnitTestPackRunner r = new LocalPhpUnitTestPackRunner(cm, tmgr, scenario_set, build, host, host); - cm.showGUI(r, test_pack); - r.runAllTests(config, test_pack); - tmgr.notifyPhpUnitFinished(host, r.getScenarioSetSetup(), test_pack); + for ( int i=0 ; i < cm.getRunTestPack() ; i++ ) { + for ( PhpUnitSourceTestPack test_pack : test_packs ) { + cm.println(EPrintType.CLUE, PfttMain.class, "Test-Pack: "+test_pack); + + cm.println(EPrintType.CLUE, PfttMain.class, "Writing Result-Pack: "+tmgr.getResultPackPath()); + + for (ScenarioSet scenario_set : getScenarioSets(config, EScenarioSetPermutationLayer.WEB_APPLICATION)) { + List<AHost> hosts = config.getHosts(); + AHost host = hosts.isEmpty()?this.host:hosts.get(0); + LocalPhpUnitTestPackRunner r = new LocalPhpUnitTestPackRunner(cm, tmgr, scenario_set, build, host, host); + cm.showGUI(r, test_pack); + r.runAllTests(config, test_pack); + tmgr.notifyPhpUnitFinished(host, r.getScenarioSetSetup(), test_pack); + } } + if (cm.getRunTestPack()>1) + closeWriter(tmgr); } } // end public void appAll @@ -550,50 +586,55 @@ public class PfttMain { hosts = new ArrayList<AHost>(1); hosts.add(this.host); } - for ( ScenarioSet scenario_set : getScenarioSets(config, EScenarioSetPermutationLayer.PHP_CORE) ) { - if (!cm.isSkipSmokeTests()) { - { - // TODO test running PHPTs on a build that is missing a DLL that is - RequiredExtensionsSmokeTest test = new RequiredExtensionsSmokeTest(); - // - // on Windows, missing .DLLs from a php build will cause a blocking winpop dialog msg to appear - // in such a case, the test will timeout after 1 minute and then fail (stopping at that point is important) - // @see PhpBuild#getExtensionList - if (test.test(build, cm, host, SAPIScenario.getSAPIScenario(scenario_set).getSAPIType())==ESmokeTestStatus.FAIL) { - // if this test fails, RequiredFeaturesSmokeTest will fail for sure - cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); - break; + for ( int i=0 ; i < cm.getRunTestPack() ; i++ ) { + for ( ScenarioSet scenario_set : getScenarioSets(config, EScenarioSetPermutationLayer.PHP_CORE) ) { + if (!cm.isSkipSmokeTests()) { + { + // TODO test running PHPTs on a build that is missing a DLL that is + RequiredExtensionsSmokeTest test = new RequiredExtensionsSmokeTest(); + // + // on Windows, missing .DLLs from a php build will cause a blocking winpop dialog msg to appear + // in such a case, the test will timeout after 1 minute and then fail (stopping at that point is important) + // @see PhpBuild#getExtensionList + if (test.test(build, cm, host, SAPIScenario.getSAPIScenario(scenario_set).getSAPIType())==ESmokeTestStatus.FAIL) { + // if this test fails, RequiredFeaturesSmokeTest will fail for sure + cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); + break; + } + } + { + RequiredFeaturesSmokeTest test = new RequiredFeaturesSmokeTest(); + if (test.test(build, cm, host)==ESmokeTestStatus.FAIL) { + cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); + break; + } } } + + // + for ( AHost storage_host : hosts ) { + LocalPhptTestPackRunner test_pack_runner = new LocalPhptTestPackRunner(tmgr.getConsoleManager(), tmgr, scenario_set, build, storage_host, host, config); + cm.showGUI(test_pack_runner); + + test_pack_runner.runAllTests(config, test_pack); + + tmgr.notifyPhptFinished(host, test_pack_runner.getScenarioSetSetup()); + } + + // { - RequiredFeaturesSmokeTest test = new RequiredFeaturesSmokeTest(); - if (test.test(build, cm, host)==ESmokeTestStatus.FAIL) { + PhptTestCountsMatchSmokeTest test = new PhptTestCountsMatchSmokeTest(); + if (test.test(tmgr)==ESmokeTestStatus.FAIL) { cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); - break; } } - } - - // - for ( AHost storage_host : hosts ) { - LocalPhptTestPackRunner test_pack_runner = new LocalPhptTestPackRunner(tmgr.getConsoleManager(), tmgr, scenario_set, build, storage_host, host, config); - cm.showGUI(test_pack_runner); + // - test_pack_runner.runAllTests(config, test_pack); - - tmgr.notifyPhptFinished(host, test_pack_runner.getScenarioSetSetup()); - } - - // - { - PhptTestCountsMatchSmokeTest test = new PhptTestCountsMatchSmokeTest(); - if (test.test(tmgr)==ESmokeTestStatus.FAIL) { - cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); - } + } // end for + if (cm.getRunTestPack()>1) { + closeWriter(tmgr); } - // - - } // end for + } } // end public void coreAll public void coreList(PhpBuild build, PhptSourceTestPack test_pack, Config config, PhpResultPackWriter tmgr, List<String> names) throws FileNotFoundException, IOException, Exception { @@ -607,53 +648,57 @@ public class PfttMain { hosts = new ArrayList<AHost>(1); hosts.add(this.host); } - for ( ScenarioSet scenario_set : getScenarioSets(config, EScenarioSetPermutationLayer.PHP_CORE) ) { - // - if (!cm.isSkipSmokeTests()) { - { - RequiredExtensionsSmokeTest test = new RequiredExtensionsSmokeTest(); - if (test.test(build, cm, host, SAPIScenario.getSAPIScenario(scenario_set).getSAPIType())==ESmokeTestStatus.FAIL) { - cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); - break; + for ( int i=0 ; i < cm.getRunTestPack() ; i++ ) { + for ( ScenarioSet scenario_set : getScenarioSets(config, EScenarioSetPermutationLayer.PHP_CORE) ) { + // + if (!cm.isSkipSmokeTests()) { + { + RequiredExtensionsSmokeTest test = new RequiredExtensionsSmokeTest(); + if (test.test(build, cm, host, SAPIScenario.getSAPIScenario(scenario_set).getSAPIType())==ESmokeTestStatus.FAIL) { + cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); + break; + } + } + { + RequiredFeaturesSmokeTest test = new RequiredFeaturesSmokeTest(); + if (test.test(build, cm, host)==ESmokeTestStatus.FAIL) { + cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); + break; + } } } + // + + LinkedList<PhptTestCase> test_cases = new LinkedList<PhptTestCase>(); + + cm.println(EPrintType.CLUE, getClass(), "Writing Result-Pack: "+tmgr.getResultPackPath()); + test_pack.cleanup(cm); + cm.println(EPrintType.IN_PROGRESS, "PhptSourceTestPack", "enumerating test cases from test-pack..."); + test_pack.read(config, test_cases, names, tmgr.getConsoleManager(), tmgr, build, true); // TODO true? + cm.println(EPrintType.IN_PROGRESS, "PhptSourceTestPack", "enumerated test cases."); + + for ( AHost storage_host : hosts ) { + LocalPhptTestPackRunner test_pack_runner = new LocalPhptTestPackRunner(tmgr.getConsoleManager(), tmgr, scenario_set, build, storage_host, host, config); + cm.showGUI(test_pack_runner); + + test_pack_runner.runTestList(test_pack, test_cases); + + tmgr.notifyPhptFinished(host, test_pack_runner.getScenarioSetSetup()); + } + + // { - RequiredFeaturesSmokeTest test = new RequiredFeaturesSmokeTest(); - if (test.test(build, cm, host)==ESmokeTestStatus.FAIL) { + PhptTestCountsMatchSmokeTest test = new PhptTestCountsMatchSmokeTest(); + if (test.test(tmgr)==ESmokeTestStatus.FAIL) { cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); - break; } } - } - // - - LinkedList<PhptTestCase> test_cases = new LinkedList<PhptTestCase>(); - - cm.println(EPrintType.CLUE, getClass(), "Writing Result-Pack: "+tmgr.getResultPackPath()); - test_pack.cleanup(cm); - cm.println(EPrintType.IN_PROGRESS, "PhptSourceTestPack", "enumerating test cases from test-pack..."); - test_pack.read(config, test_cases, names, tmgr.getConsoleManager(), tmgr, build, true); // TODO true? - cm.println(EPrintType.IN_PROGRESS, "PhptSourceTestPack", "enumerated test cases."); - - for ( AHost storage_host : hosts ) { - LocalPhptTestPackRunner test_pack_runner = new LocalPhptTestPackRunner(tmgr.getConsoleManager(), tmgr, scenario_set, build, storage_host, host, config); - cm.showGUI(test_pack_runner); + // - test_pack_runner.runTestList(test_pack, test_cases); - - tmgr.notifyPhptFinished(host, test_pack_runner.getScenarioSetSetup()); - } - - // - { - PhptTestCountsMatchSmokeTest test = new PhptTestCountsMatchSmokeTest(); - if (test.test(tmgr)==ESmokeTestStatus.FAIL) { - cm.println(EPrintType.CANT_CONTINUE, "Main", "Failed smoke test: "+test.getName()); - } - } - // - - } // end for (scenario_set) + } // end for (scenario_set) + if (cm.getRunTestPack()>1) + closeWriter(tmgr); + } } // end public void coreList public enum ERevisionGetOption { @@ -1173,13 +1218,12 @@ public class PfttMain { } // - PfttMain p = new PfttMain(null); // int args_i = 0; Config config = null; boolean is_uac = false, debug = false, randomize_order = false, no_result_file_for_pass_xskip_skip = false, pftt_debug = false, show_gui = false, overwrite = false, disable_debug_prompt = false, results_only = false, dont_cleanup_test_pack = false, phpt_not_in_place = false, thread_safety = true, skip_smoke_tests = false, pause = false, restart_each_test_all = false, no_restart_all = false, ignore_unknown_option = false, ini_actual_all = false; - int run_test_times_all = 1, delay_between_ms = 0, run_test_times_list_times = 1, run_group_times_all = 1, run_group_times_list_times = 1, max_test_read_count = 0, thread_count = 0, run_count = 0, suspend_seconds = 0; + int run_test_times_all = 1, run_test_pack = 1, delay_between_ms = 0, run_test_times_list_times = 1, run_group_times_all = 1, run_group_times_list_times = 1, max_test_read_count = 0, thread_count = 0, run_count = 0, suspend_seconds = 0; LinkedList<String> debug_list = new LinkedList<String>(); LinkedList<String> run_test_times_list = new LinkedList<String>(); LinkedList<String> run_group_times_list = new LinkedList<String>(); @@ -1211,7 +1255,7 @@ public class PfttMain { System.exit(-255); System.out.println("PFTT: Config: loaded "+config_files); } else { - File default_config_file = new File(p.host.getPfttDir()+"/conf/default.groovy"); + File default_config_file = new File(new LocalHost().getPfttDir()+"/conf/default.groovy"); config = Config.loadConfigFromFiles(cm, default_config_file); System.out.println("PFTT: Config: no config files loaded... using default only ("+default_config_file+")"); } @@ -1223,12 +1267,14 @@ public class PfttMain { if (config.processConsoleOptions(cm, args_list)) { config_args = true; // config file(s) changed console options. show the console options PFTT will now be run with. - System.out.println("PFTT: Console Options: "+args_list); + System.out.println("PFTT: Console Options(2): "+args_list); } args = ArrayUtil.toArray(args_list); } // + PfttMain p = new PfttMain(null, config); + // process all console args now for ( args_i = 0 ; args_i < args.length ; args_i++ ) { if (args[args_i].equals("-gui")||args[args_i].equals("-g")) { @@ -1273,6 +1319,9 @@ public class PfttMain { no_result_file_for_pass_xskip_skip = true; } else if (args[args_i].equals("-randomize_order")) { randomize_order = true; + } else if (args[args_i].equals("-run_test_pack")) { + args_i++; + run_test_pack = Integer.parseInt(args[args_i]); } else if (args[args_i].equals("-run_test_times_all")) { args_i++; run_test_times_all = Integer.parseInt(args[args_i]); @@ -1302,7 +1351,7 @@ public class PfttMain { } else if (args[args_i].equals("-suspend_seconds")) { args_i++; suspend_seconds = Integer.parseInt(args[args_i]); - } else if (args[args_i].equals("-run_count")) { + } else if (args[args_i].equals("-run_count")||args[args_i].equals("-rc")) { args_i++; run_count = Integer.parseInt(args[args_i]); } else if (args[args_i].equals("-max_test_read_count")) { @@ -1339,6 +1388,12 @@ public class PfttMain { // also intercepted and handled by bin/pftt.cmd batch script debug = true; + } else if (args[args_i].startsWith("-debugger")) { + // also intercepted and handled by bin/pftt.cmd batch script + debug = true; + args_i++; + // TODO debugger = args[args_i]; + } else if (args[args_i].equals("-disable_debug_prompt")||args[args_i].equals("-debug_none")||args[args_i].equals("-d")) { disable_debug_prompt = true; } else if (args[args_i].equals("-results_only")) { @@ -1405,7 +1460,7 @@ public class PfttMain { } - cm = new LocalConsoleManager(source_pack, debug_pack, overwrite, debug, results_only, show_gui, disable_debug_prompt, dont_cleanup_test_pack, phpt_not_in_place, pftt_debug, no_result_file_for_pass_xskip_skip, randomize_order, run_test_times_all, + cm = new LocalConsoleManager(source_pack, debug_pack, overwrite, debug, results_only, show_gui, disable_debug_prompt, dont_cleanup_test_pack, phpt_not_in_place, pftt_debug, no_result_file_for_pass_xskip_skip, randomize_order, run_test_times_all, run_test_pack, thread_safety, run_test_times_list_times, run_group_times_all, run_group_times_list_times, debug_list, run_test_times_list, run_group_times_list, skip_list, skip_smoke_tests, max_test_read_count, thread_count, restart_each_test_all, no_restart_all, delay_between_ms, run_count, suspend_seconds, ini_actual_all); @@ -1864,7 +1919,18 @@ public class PfttMain { if (scenario_set_setup==null) continue; - web = ws_mgr.getWebServerInstance(cm, host, scenario_set, build, RequiredExtensionsSmokeTest.createDefaultIniCopy(cm, host, build), null, "C:/PHP-SDK/APPS", null, cm.isDebugAll(), test_pack.getNameAndVersionInfo()); + web = ws_mgr.getWebServerInstance( + cm, + host, + scenario_set, + build, + RequiredExtensionsSmokeTest.createDefaultIniCopy(cm, host, build), + null, + "C:/PHP-SDK/APPS", // TODO + null, + cm.isDebugAll(), + test_pack.getNameAndVersionInfo() + ); UITestRunner runner = new UITestRunner( cm, diff --git a/src/com/mostc/pftt/model/app/PhpUnitSourceTestPack.java b/src/com/mostc/pftt/model/app/PhpUnitSourceTestPack.java index a4971c1..87c9d0b 100644 --- a/src/com/mostc/pftt/model/app/PhpUnitSourceTestPack.java +++ b/src/com/mostc/pftt/model/app/PhpUnitSourceTestPack.java @@ -1,7 +1,6 @@ package com.mostc.pftt.model.app; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.lang.ref.SoftReference; @@ -13,15 +12,6 @@ import java.util.Map; import javax.annotation.Nullable; -import com.caucho.quercus.QuercusContext; -import com.caucho.quercus.function.AbstractFunction; -import com.caucho.quercus.parser.QuercusParseException; -import com.caucho.quercus.parser.QuercusParser; -import com.caucho.quercus.program.InterpretedClassDef; -import com.caucho.quercus.program.QuercusProgram; -import com.caucho.vfs.FilePath; -import com.caucho.vfs.FileReadStream; -import com.caucho.vfs.ReadStream; import com.github.mattficken.Overridable; import com.mostc.pftt.host.AHost; import com.mostc.pftt.host.Host; @@ -31,6 +21,10 @@ import com.mostc.pftt.model.SourceTestPack; import com.mostc.pftt.model.core.EBuildBranch; import com.mostc.pftt.model.core.PhpBuild; import com.mostc.pftt.model.core.PhpIni; +import com.mostc.pftt.model.core.PhpParser; +import com.mostc.pftt.model.core.PhpParser.ClassDefinition; +import com.mostc.pftt.model.core.PhpParser.FunctionDefinition; +import com.mostc.pftt.model.core.PhpParser.PhpScript; import com.mostc.pftt.results.ConsoleManager; import com.mostc.pftt.results.ITestResultReceiver; import com.mostc.pftt.scenario.ScenarioSet; @@ -67,7 +61,6 @@ public abstract class PhpUnitSourceTestPack implements SourceTestPack<PhpUnitAct protected String test_pack_root; protected final ArrayList<PhpUnitDist> php_unit_dists; protected final ArrayList<String> blacklist_test_names, whitelist_test_names, include_dirs, include_files; - protected final QuercusContext qctx; protected SoftReference<ArrayList<PhpUnitTestCase>> _ref_test_cases; public PhpUnitSourceTestPack() { @@ -77,8 +70,6 @@ public abstract class PhpUnitSourceTestPack implements SourceTestPack<PhpUnitAct include_dirs = new ArrayList<String>(5); include_files = new ArrayList<String>(3); - qctx = new QuercusContext(); - // add default entries to include_path addIncludeDirectory("."); } @@ -238,11 +229,31 @@ public abstract class PhpUnitSourceTestPack implements SourceTestPack<PhpUnitAct // } // end public void read + /** Many test-packs store their phpunit tests only in files that end with `Test.php`, + * but some don't. + * + * This checks for that. you can override that for custom checks. + * + * @param file_name + * @return + */ @Overridable - protected boolean isFileNameATest(String file_name) { + public boolean isFileNameATest(String file_name) { return file_name.endsWith("Test.php"); } + /** PhpUnit tests are supposed to be stored functions with names beginning with `test`. + * + * This allows for overridding that behavior. + * + * @param function_name + * @return + */ + @Overridable + public boolean isFunctionATest(String function_name) { + return function_name.startsWith("test"); + } + /** scans for *Test.php files and reads PhpUnitTestCase(s) from them * * @param config @@ -278,11 +289,7 @@ public abstract class PhpUnitSourceTestPack implements SourceTestPack<PhpUnitAct else if (!whitelist_test_names.isEmpty() && !whitelist_test_names.contains(lc_test_file_name)) continue; - try { - readTestFile(config, max_read_count, rel_file_name, abs_file_name, php_unit_dist, test_cases, file); - } catch ( QuercusParseException ex ) { - ex.printStackTrace(); - } + readTestFile(config, max_read_count, rel_file_name, abs_file_name, php_unit_dist, test_cases, file); if (max_read_count > 0 && test_cases.size() >= max_read_count) return; @@ -300,30 +307,17 @@ public abstract class PhpUnitSourceTestPack implements SourceTestPack<PhpUnitAct * @param php_unit_dist * @param test_cases * @param file - * @throws IOException */ - protected void readTestFile(Config config, final int max_read_count, String rel_test_file_name, String abs_test_file_name, PhpUnitDist php_unit_dist, List<PhpUnitTestCase> test_cases, File file) throws IOException { - FileInputStream fin = new FileInputStream(file); - // - // with all the `non-technical` obstacles around developing PFTT, there isn't time to develop a simple - // PHP parser here... have to use Quercus to get this done - // - QuercusParser p = new QuercusParser(qctx, new FilePath(file.getAbsolutePath()), new ReadStream(new FileReadStream(fin))); - QuercusProgram prog = p.parse(); + protected void readTestFile(Config config, final int max_read_count, String rel_test_file_name, String abs_test_file_name, PhpUnitDist php_unit_dist, List<PhpUnitTestCase> test_cases, File file) { + PhpScript script = PhpParser.parseScript(file); - // Expr that may be useful - // FunIncludeOnceExpr => PHP include_once require_once - // FunIncludeExpr => PHP include require - // ImportExpr => PHP import - - // search file for classes - for (InterpretedClassDef clazz : prog.getClasses() ) { + for ( ClassDefinition clazz : script.getClasses() ) { if (clazz.isAbstract()||clazz.isInterface()) continue; - for (Map.Entry<String, AbstractFunction> e : clazz.functionSet()) { + for ( FunctionDefinition func : clazz.getFunctions() ) { // search class for functions that start with 'test' - if (e.getValue().getName().startsWith("test")) { + if (isFunctionATest(func.getName())) { // this is a test case PhpUnitTestCase test_case = new PhpUnitTestCase( php_unit_dist, @@ -334,8 +328,8 @@ public abstract class PhpUnitSourceTestPack implements SourceTestPack<PhpUnitAct // in such cases, so nothing special needs to be done here for them clazz.getName(), // name of method within the class - e.getValue().getName(), - e.getValue().getArgs().length + func.getName(), + func.getArgumentCount() ); config.processPhpUnit(test_case); test_cases.add(test_case); @@ -345,8 +339,6 @@ public abstract class PhpUnitSourceTestPack implements SourceTestPack<PhpUnitAct } } } - - fin.close(); } // end protected void readTestFile @Override diff --git a/src/com/mostc/pftt/model/app/PhpUnitTemplate.groovy b/src/com/mostc/pftt/model/app/PhpUnitTemplate.groovy index d434618..f562170 100644 --- a/src/com/mostc/pftt/model/app/PhpUnitTemplate.groovy +++ b/src/com/mostc/pftt/model/app/PhpUnitTemplate.groovy @@ -5,6 +5,7 @@ import java.util.Map; import com.github.mattficken.io.StringUtil; import com.mostc.pftt.host.AHost; import com.mostc.pftt.scenario.ScenarioSet; +import com.mostc.pftt.scenario.XDebugScenario; /** Groovy implementation of PHPUnit's Process\TestCaseMethod template * @@ -50,7 +51,7 @@ public class PhpUnitTemplate { my_temp_dir = StringUtil.cslashes(host.fixPath(my_temp_dir)); - def use_xdebug = false; + def use_xdebug = scenario_set.contains(XDebugScenario.class); // PFTT mod: need to set date.timezone=UTC... for some reason, // ini file doesn't work, date_default_timezone_set() and ini_set() must both diff --git a/src/com/mostc/pftt/model/core/PhpParser.java b/src/com/mostc/pftt/model/core/PhpParser.java index 6339bda..606e4d3 100644 --- a/src/com/mostc/pftt/model/core/PhpParser.java +++ b/src/com/mostc/pftt/model/core/PhpParser.java @@ -1,219 +1,1023 @@ package com.mostc.pftt.model.core; import java.io.File; + +import java.io.FileInputStream; import java.io.IOException; -import java.io.PrintStream; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import org.xmlpull.v1.XmlSerializer; +import com.caucho.quercus.Location; +import com.caucho.quercus.QuercusContext; +import com.caucho.quercus.env.Env; +import com.caucho.quercus.env.NumberValue; +import com.caucho.quercus.env.QuercusClass; +import com.caucho.quercus.env.Value; +import com.caucho.quercus.expr.CallExpr; +import com.caucho.quercus.expr.Expr; +import com.caucho.quercus.function.AbstractFunction; import com.caucho.quercus.parser.QuercusParser; +import com.caucho.quercus.program.Arg; +import com.caucho.quercus.program.ClassDef; +import com.caucho.quercus.program.ClassDef.FieldEntry; +import com.caucho.quercus.program.Function; +import com.caucho.quercus.program.InterpretedClassDef; import com.caucho.quercus.program.QuercusProgram; +import com.caucho.quercus.statement.BlockStatement; +import com.caucho.quercus.statement.ExprStatement; +import com.caucho.quercus.statement.Statement; import com.caucho.vfs.FilePath; import com.caucho.vfs.FileReadStream; +import com.caucho.vfs.NullPath; import com.caucho.vfs.ReadStream; +import com.caucho.vfs.StringStream; +import com.mostc.pftt.results.ISerializer; +import com.mostc.pftt.results.TestCaseCodeCoverage; + +/** parses php code into a tree which can be navigated using this API or + * serialized to XML. + * + * enables easily navigating php code with simple tools like xpath or xquery, which + * can also navigate test results and code coverage data (since they're all xml trees) + * + * The hard part of code analysis (static or runtime/dynamic) is navigating all the data. + * This class can help because it lets you easily navigate the code that was analyzed. + * + * @see TestCaseCodeCoverage + * + */ public class PhpParser { - /* - public static PhptTestCase toPhpt(PhpCodeCoverage cc, PhpProgram prog) { - - } - public static void toSingleFile(PhpCodeCoverage cc, PhpProgram prog) { - PrintStream ps; - ps.println("Test Name"); - ps.println("--CREDITS--"); - ps.println("--FILE--"); - ps.println("--EXPECT--"); - SinglePhpClass clazz; - for ( FunctionCall func : getBuiltinFunctions(cc, prog) ) { - ps.print(clazz.ensureDefined(func)); - ps.print('('); - for ( VariableValue val : func.getArguments() ) { - Value last = func.getLastVal(val); - if (last instanceof ArrayValue) { - // do this recursively - } else if (last instanceof FunctionCall) { - // NOTE: FunctionDefinition and ClassDefinition won't be here - // there will be a FunctionCall (even to instantiate a class) - // - // ensure defined - // add function call - // TODO do this recursively too - ps.print(clazz.ensureDefined(func)); - } else if (last instanceof VariableValue) { - // include value - } - - val.getName(); - } - ps.println(");"); + /** parses file into PhpScript + * + * @param file + * @return + */ + public static PhpScript parseScript(File file) { + // + // with all the `non-technical` obstacles around developing PFTT(nearly preventing it + // from working, trying to limit its features, trying to limit its Linux support, + // calling it an `automation tool` instead of `test tool`, resistance to adding PHPT + // support, resistance to adding PhpUnit support, resistance to UI testing support, + // resistance to adding Apache support and even resistance to IIS support, etc...) + // and the time/energy I have to spend dealing with those `non-technical` obstacles, + // I just don't have the time to develop a simple PHP parser here... have to use Quercus to get this done + // + try { + QuercusContext qctx = new QuercusContext(); + QuercusParser p = new QuercusParser( + qctx, + new FilePath(file.getAbsolutePath()), + new ReadStream(new FileReadStream(new FileInputStream(file))) + ); + return new PhpScript(file, new Env(qctx), p.parse()); + } catch ( Exception ex ) { + //ex.printStackTrace(); + } + return new PhpScript(file, null, null); + } // end public static PhpScript parseScript + + /** parses PhpScript from string containing php code + * + * @param php_code_str + * @param filename - filename the string is from + * @return + */ + public static PhpScript parseScript(String php_code_str, String filename) { + try { + QuercusContext qctx = new QuercusContext(); + QuercusParser p = new QuercusParser( + qctx, + filename == null ? new NullPath("<unknown>") : new FilePath(filename), + StringStream.open(php_code_str) + ); + return new PhpScript(php_code_str, new Env(qctx), p.parse()); + } catch ( Exception ex ) { + //ex.printStackTrace(); } + return new PhpScript(php_code_str, null, null); + } + + public static PhpScript parseScript(String php_code_str) { + return parseScript(php_code_str, null); } + + public static abstract class PhpCode implements ISerializer { + protected final Env env; + + protected PhpCode(Env env) { + this.env = env; + } + + @Override + public abstract int hashCode(); + + @Override + public abstract boolean equals(Object o); + + @Override + public abstract String toString(); + + } // end public static abstract class PhpCode - protected static class SinglePhpClass { - protected final HashMap<FunctionDefinition,String> single_file_names; + public static class PhpScript extends PhpCode { + protected final QuercusProgram prog; + protected final Object source; + + protected PhpScript(Object source, Env env, QuercusProgram prog) { + super(env); + this.source = source; + this.prog = prog; + } + + /** returns the classes defined by this php script + * + * @return + */ + public ClassDefinition[] getClasses() { + if (prog==null) { + return new ClassDefinition[]{}; + } else { + Collection<InterpretedClassDef> clazzes = prog.getClasses(); + ClassDefinition[] defs = new ClassDefinition[clazzes.size()]; + int i=0; + for ( InterpretedClassDef clazz : clazzes ) + defs[i++] = new ClassDefinition(env, clazz); + return defs; + } + } + + public List<String> getUserFunctionNames() { + return getBuiltinFunctionNames(null); + } + + public List<String> getUserFunctionNames(TestCaseCodeCoverage cc) { + List<UserFunctionCall> in = getUserFunctions(cc); + ArrayList<String> out = new ArrayList<String>(in.size()); + for ( FunctionCall func : in ) + out.add(func.getFunctionName()); + return out; + } + + public List<String> getFunctionNames() { + return getBuiltinFunctionNames(null); + } + + public List<String> getFunctionNames(TestCaseCodeCoverage cc) { + List<FunctionCall> in = getFunctions(cc); + ArrayList<String> out = new ArrayList<String>(in.size()); + for ( FunctionCall func : in ) + out.add(func.getFunctionName()); + return out; + } + + public List<String> getBuiltinFunctionNames() { + return getBuiltinFunctionNames(null); + } + + public List<String> getBuiltinFunctionNames(TestCaseCodeCoverage cc) { + List<BuiltinFunctionCall> in = getBuiltinFunctions(cc); + ArrayList<String> out = new ArrayList<String>(in.size()); + for ( BuiltinFunctionCall func : in ) + out.add(func.getFunctionName()); + return out; + } + + /** returns the builtin functions called by this php script + * + * this returns all the builtin functions the script could call. + * + * @return + */ + public List<BuiltinFunctionCall> getBuiltinFunctions() { + return getBuiltinFunctions(null); + } - public SinglePhpClass() { - single_file_names = new HashMap<FunctionDefinition,String>(); + /** returns the builtin functions that were actually called by this php script + * when it was run (based on the code coverage data). + * + * @param cc + * @return + */ + public List<BuiltinFunctionCall> getBuiltinFunctions(TestCaseCodeCoverage cc) { + LinkedList<BuiltinFunctionCall> out = new LinkedList<BuiltinFunctionCall>(); + readFunctions(true, false, out, cc); + return out; } - public String ensureDefined(FunctionDefinition func) { - String func_name = func.getName(); - if (single_file_names.containsValue(func.getName())) { - // generate unique function name - for ( int i=2 ; ; i++ ) { - if (!single_file_names.containsValue(func_name+i)) { - func_name = func_name+i; - break; + /** get all function calls to user defined functions in this PhpScript + * + * @return + */ + public List<UserFunctionCall> getUserFunctions() { + return getUserFunctions(null); + } + /** get all function calls to user defined functions in this PhpScript, + * called from this code coverage data + * + * @param cc + * @return + */ + public List<UserFunctionCall> getUserFunctions(TestCaseCodeCoverage cc) { + LinkedList<UserFunctionCall> out = new LinkedList<UserFunctionCall>(); + readFunctions(false, false, out, cc); + return out; + } + /** get all functions called from this PhpScript + * + * @return + */ + public List<FunctionCall> getFunctions() { + return getFunctions(null); + } + /** get all function calls from this PhpScript that were called from this code coverage data + * + * @param cc + * @return + */ + public List<FunctionCall> getFunctions(TestCaseCodeCoverage cc) { + LinkedList<FunctionCall> out = new LinkedList<FunctionCall>(); + readFunctions(true, true, out, cc); + return out; + } + + @SuppressWarnings("unchecked") + protected void readFunctions(boolean builtin, boolean both, @SuppressWarnings("rawtypes") List out, TestCaseCodeCoverage cc) { + for ( ClassDefinition clazz : getClasses() ) { + for ( FunctionDefinition def : clazz.getFunctions() ) { + for ( FunctionCall call : def.getCalls() ) { + if ( + (both||builtin==call.isBuiltin()) + && (cc==null||cc.isExecuted(call.getFileName(), call.getLineNumber())) + ) { + out.add(call); + } } } } - single_file_names.put(func, func_name); - return func_name; } - } - - public static List<String> getBuiltinFunctionNames(PhpProgram prog) { - return getBuiltinFunctionNames(null, prog); - } + + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "php"); + for ( ClassDefinition clazz : getClasses() ) + clazz.serial(serial); + serial.endTag("pftt", "php"); + } + + @Override + public int hashCode() { + return source.hashCode(); + } + + @Override + public boolean equals(Object o) { + return o == this ? true : toString().equals(o.toString()); + } + + @Override + public String toString() { + return source.toString(); + } + + } // end public static class PhpScript - public static List<String> getBuiltinFunctionNames(PhpCodeCoverage cc, PhpProgram prog) { - List<FunctionCall> in = getBuiltinFunctions(cc, prog); - ArrayList<String> out = new ArrayList<String>(in.size()); - for ( FunctionCall func : in ) - out.add(func.getName()); - return out; - } + public static class Field extends CodeBlock { + protected final String name; + protected final VariableValue value; + + protected Field(Env env, String name, VariableValue value) { + super(env); + this.name = name; + this.value = value; + } + + /** variable this class field stores + * + * @return + */ + public VariableValue getValue() { + return value; + } + + /** name of this class field + * + */ + @Override + public String getName() { + return name; + } + + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "field"); + serial.attribute("pftt", "name", getName()); + getValue().serial(serial); + serial.endTag("pftt", "field"); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public String toString() { + return "{"+name+"="+value+"}"; + } + + @Override + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof Field) + return ((Field)o).getName().equals(name); + else + return toString().equals(o.toString()); + } + + } // end public static class Field - public static List<FunctionCall> getBuiltinFunctions(PhpProgram prog) { - return getBuiltinFunctions(null, prog); - } + public static abstract class VariableValue extends PhpCode { + protected final FunctionDefinition fdef; // can be null, if this is a class field + + protected VariableValue(Env env, FunctionDefinition fdef) { + super(env); + this.fdef = fdef; + } + + /** function that contains this variable + * + * @return + */ + public FunctionDefinition getParent() { + return fdef; + } + + protected static VariableValue toVariableValue(Env env, FunctionDefinition fdef, com.caucho.quercus.env.Value v) { + if (v instanceof com.caucho.quercus.env.NumberValue) { + return new NumericValue(env, fdef, (com.caucho.quercus.env.NumberValue)v); + } else if (v instanceof com.caucho.quercus.env.StringValue) { + return new StringValue(env, fdef, (com.caucho.quercus.env.StringValue)v); + } else if (v instanceof AbstractFunction) { + // TODO null - get location of this call + return new UserFunctionCall(env, fdef, null, new FunctionDefinition(env, fdef.getClassDefinition(), (AbstractFunction)v)); + } else if (v instanceof com.caucho.quercus.env.ArrayValue) { + com.caucho.quercus.env.ArrayValue av = (com.caucho.quercus.env.ArrayValue) v; + // PHP arrays can either be a list/array or a map + return av.entrySet().isEmpty() ? new ArrayValue(env, fdef, av) : new MapValue(env, fdef, av); + } else if (v instanceof com.caucho.quercus.env.BooleanValue) { + return new BooleanValue(env, fdef, v); + } else { + return new ObjectValue(env, fdef, v); + } + } + + public static VariableValue toVariableValue(Env env, FunctionDefinition fdef, Expr e) { + return toVariableValue(env, fdef, e.eval(env)); + } + + } // end public static abstract class VariableValue - public static List<FunctionCall> getBuiltinFunctions(PhpCodeCoverage cc, PhpProgram prog) { - LinkedList<FunctionCall> out = new LinkedList<FunctionCall>(); - for ( ClassDefinition clazz : prog.getClasses() ) { - for ( FunctionCall call : clazz.getFunctions() ) { - if (call.isBuiltin()) { - if (cc==null||cc.isExecuted(call.getFileName(), call.getLineNumber())) { - out.add(call); - } - } + public static class MapValue extends VariableValue implements Iterable<String> { + protected final HashMap<String,VariableValue> map; + + public MapValue(Env env, FunctionDefinition fdef, com.caucho.quercus.env.ArrayValue v) { + super(env, fdef); + map = new HashMap<String,VariableValue>(); + for ( java.util.Map.Entry<Value, Value> e : v.entrySet() ) { + map.put( + e.getKey().toJavaString(), + VariableValue.toVariableValue(env, fdef, e.getValue()) + ); } } - return out; - } + + @Override + public Iterator<String> iterator() { + return getNames(); + } + + public Iterator<String> getNames() { + return map.keySet().iterator(); + } + + public VariableValue getValue(String name) { + return map.get(name); + } + + public Iterator<VariableValue> getValues() { + return map.values().iterator(); + } + + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "map"); + Iterator<String> it = getNames(); + String name; + while (it.hasNext()) { + name = it.next(); + + serial.startTag("pftt", "value"); + serial.attribute("pftt", "name", name); + getValue(name).serial(serial); + serial.endTag("pftt", "value"); + } + serial.endTag("pftt", "map"); + } + + @Override + public int hashCode() { + return map.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof MapValue) + return ((MapValue)o).map.equals(this.map); + else + return toString().equals(o.toString()); + } + + @Override + public String toString() { + return map.toString(); + } + + } // end public static class MapValue - public static PhpProgram parseProgram(File file) { - QuercusParser p = new QuercusParser(qctx, new FilePath(file.getAbsolutePath()), new ReadStream(new FileReadStream(fin))); - QuercusProgram prog = p.parse(); - } + public static class ObjectValue extends VariableValue { + protected final com.caucho.quercus.env.Value v; + + public ObjectValue(Env env, FunctionDefinition fdef, com.caucho.quercus.env.Value v) { + super(env, fdef); + this.v = v; + } + + public com.caucho.quercus.env.Value value() { + return v; + } - public static abstract class PhpSerial { - public abstract void serialize(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException; - } - public static class PhpProgram extends PhpSerial { - public ClassDefinition[] getClasses() { - + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "object"); + serial.text(v.toJavaString()); + serial.endTag("pftt", "object"); } - } - public abstract static class Value extends PhpSerial { - public String getName() { - + + @Override + public int hashCode() { + return v.hashCode(); } - public FunctionDefinition getParent() { - + @Override + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof ObjectValue) + return ((ObjectValue)o).equals(v); + else + return toString().equals(o.toString()); + } + public String toString() { + return v.toJavaString(); } - } - public static abstract class VariableValue extends Value { - } + } // end public static class ObjectValue + + public static class BooleanValue extends VariableValue { + protected final boolean b; + + public BooleanValue(Env env, FunctionDefinition fdef, com.caucho.quercus.env.Value v) { + super(env, fdef); + b = v.toJavaBoolean().booleanValue(); + } + + public boolean value() { + return b; + } + + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "boolean"); + serial.text(Boolean.toString(b)); + serial.endTag("pftt", "boolean"); + } + + @Override + public int hashCode() { + return b ? 1 : 0; + } + + @Override + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof BooleanValue) + return ((BooleanValue)o).b == this.b; + else + return toString().equals(o.toString()); + } + + @Override + public String toString() { + return Boolean.toString(b); + } + + } // end public static class BooleanValue + public static class NumericValue extends VariableValue { + protected final double v; + + public NumericValue(Env env, FunctionDefinition fdef, NumberValue v) { + super(env, fdef); + this.v = v.toDouble(); + } + + public double value() { + return v; + } + @Override - public void serialize(XmlSerializer serial) { - + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "numeric"); + serial.text(Double.toString(v)); + serial.endTag("pftt", "numeric"); } - } + + @Override + public int hashCode() { + return (int) v; + } + + @Override + public String toString() { + return Double.toString(v); + } + + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof NumericValue) + return ((NumericValue)o).v == v; + else + return toString().equals(o.toString()); + } + + } // end public static class NumericValue + public static class StringValue extends VariableValue { + protected final String v; + + public StringValue(Env env, FunctionDefinition fdef, com.caucho.quercus.env.StringValue v) { + super(env, fdef); + this.v = v.toJavaString(); + } + + public String value() { + return v; + } + @Override - public void serialize(XmlSerializer serial) { - + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "string"); + serial.text(v); + serial.endTag("pftt", "string"); } - } - public static class ArrayValue extends VariableValue { - public VariableValue[] getVariables() { - + + @Override + public int hashCode() { + return v.hashCode(); } + @Override - public void serialize(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { - serial.startTag(null, "array"); - for ( VariableValue var : getVariables() ) { - var.serialize(serial); + public String toString() { + return v; + } + + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof StringValue) + return ((StringValue)o).v.equals(v); + else + return toString().equals(o.toString()); + } + + } // end public static class StringValue + + public static class ArrayValue extends VariableValue implements Iterable<VariableValue> { + protected final com.caucho.quercus.env.ArrayValue def; + + protected ArrayValue(Env env, FunctionDefinition fdef, com.caucho.quercus.env.ArrayValue def) { + super(env, fdef); + this.def = def; + } + + public VariableValue[] value() { + ArrayList<VariableValue> out = new ArrayList<VariableValue>(2); + for ( com.caucho.quercus.env.Value v : def.values() ) { + out.add(toVariableValue(env, fdef, v)); } - serial.endTag(null, "array"); + return out.toArray(new VariableValue[out.size()]); } - } - public static abstract class CodeBlock extends Value { - } - public static abstract class ClassDefinition extends CodeBlock { - public abstract boolean isBuiltin(); - public boolean isAbstract() { - + @Override + public Iterator<VariableValue> iterator() { + ArrayList<VariableValue> out = new ArrayList<VariableValue>(2); + for ( com.caucho.quercus.env.Value v : def.values() ) { + out.add(toVariableValue(env, fdef, v)); + } + return out.iterator(); } - public boolean isInterface() { - + + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "array"); + for ( VariableValue var : value() ) { + var.serial(serial); + } + serial.endTag("pftt", "array"); + } + + @Override + public int hashCode() { + return def.hashCode(); + } + + @Override + public String toString() { + return def.toJavaString(); + } + + @Override + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof ArrayValue) + return ((ArrayValue)o).def.eq(this.def); + else + return toString().equals(o.toString()); + } + + } // end public static class ArrayValue + + public static abstract class CodeBlock extends PhpCode { + + protected CodeBlock(Env env) { + super(env); + } + + public abstract String getName(); + + } // end public static abstract class CodeBlock + + public static class ClassDefinition extends CodeBlock { + protected final ClassDef clazz; + + protected ClassDefinition(Env env, ClassDef clazz) { + super(env); + this.clazz = clazz; + } + + @Override + public String getName() { + return clazz.getName(); } + + /** returns fields in this class + * + * @return + */ + public Field[] getFields() { + ArrayList<Field> out = new ArrayList<Field>(); + for (Map.Entry<com.caucho.quercus.env.StringValue,FieldEntry> e : clazz.fieldSet()) { + out.add(new Field( + env, + e.getKey().toJavaString(), + VariableValue.toVariableValue( + env, + null, + e.getValue().getValue() + ) + )); + } + return out.toArray(new Field[out.size()]); + } + + /** returns functions defined by this class + * + * @return + */ public FunctionDefinition[] getFunctions() { - + ArrayList<FunctionDefinition> out = new ArrayList<FunctionDefinition>(); + for (Map.Entry<String, AbstractFunction> e : clazz.functionSet()) { + out.add(new FunctionDefinition(env, this, e.getValue())); + } + return out.toArray(new FunctionDefinition[out.size()]); } - } - public static class UserClassDefinition extends ClassDefinition { - } - public static class BuiltinClassDefinition extends ClassDefinition { + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "class"); + serial.startTag("pftt", "fields"); + for ( Field f : getFields() ) + f.serial(serial); + serial.endTag("pftt", "fields"); + serial.startTag("pftt", "functions"); + for ( FunctionDefinition fd : getFunctions() ) + fd.serial(serial); + serial.endTag("pftt", "functions"); + serial.endTag("pftt", "class"); + } + + public boolean isInterface() { + return clazz.isInterface(); + } + + public boolean isAbstract() { + return clazz.isAbstract(); + } - } - public static abstract class Function extends CodeBlock { + public boolean isFinal() { + return clazz.isFinal(); + } + + @Override + public int hashCode() { + return getName().hashCode(); + } + @Override + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof ClassDefinition) + return ((ClassDefinition)o).clazz.equals(this.clazz); + else + return toString().equals(o.toString()); + } + + @Override + public String toString() { + return clazz.toString(); + } + + } // end public static class ClassDefinition + + public static class UserFunctionCall extends FunctionCall { + protected final FunctionDefinition target_func; + + protected UserFunctionCall(Env env, FunctionDefinition fdef, Location loc, FunctionDefinition target_func) { + super(env, fdef, loc, target_func.getName()); + this.target_func = target_func; + } + + public FunctionDefinition getTarget() { + return target_func; + } + + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "userCall"); + serial.attribute("pftt", "name", getFunctionName()); + serial.attribute("pftt", "class", getClassName()); + serial.attribute("pftt", "file", getFileName()); + serial.attribute("pftt", "line", Integer.toString(getLineNumber())); + serial.endTag("pftt", "userCall"); + } + + @Override + public boolean isBuiltin() { + return false; + } } - public static class FunctionCall extends Function { + + public static class BuiltinFunctionCall extends FunctionCall { + + protected BuiltinFunctionCall(Env env, FunctionDefinition fdef, Location loc, String func_name) { + super(env, fdef, loc, func_name); + } @Override - public void serialize(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { - serial.startTag(null, "call"); - serial.endTag(null, "call"); + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "builtinCall"); + serial.attribute("pftt", "name", getFunctionName()); + serial.endTag("pftt", "builtinCall"); } - public Value getLastVal(VariableValue val) { + @Override + public boolean isBuiltin() { + return true; + } + + } // end public static class BuiltinFunctionCall + + public static abstract class FunctionCall extends VariableValue { + protected final String func_name; + protected final Location loc; + + protected FunctionCall(Env env, FunctionDefinition fdef, Location loc, String func_name) { + super(env, fdef); + this.loc = loc; + this.func_name = func_name; + } + + public String getClassName() { + return loc.getClassName(); + } + + public String getFunctionName() { + return func_name; } public String getFileName() { - + return loc.getFileName(); } public int getLineNumber() { + return loc.getLineNumber(); + } + + public abstract boolean isBuiltin(); + + @Override + public int hashCode() { + return getFunctionName().hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof FunctionCall) + return ((FunctionCall)o).func_name.equals(this.func_name); + else + return toString().equals(o.toString()); + } + + @Override + public String toString() { + return func_name; + } + + } // end public static abstract class FunctionCall + + public static class FunctionDefinition extends CodeBlock { + protected final AbstractFunction def; + protected final ClassDefinition clazz; + + protected FunctionDefinition(Env env, ClassDefinition clazz, AbstractFunction def) { + super(env); + this.def = def; + this.clazz = clazz; + } + + public int getLineNumber() { + return def.getLocation().getLineNumber(); + } + + public String getClassName() { + return def.getLocation().getClassName(); + } + + public String getFileName() { + return def.getLocation().getFileName(); + } + public ClassDefinition getClassDefinition() { + return clazz; } - } - public static class FunctionDefinition extends Function { - public boolean isBuiltin() { - + /** the fully qualified name of the function, including class name and \\ + * + * @return + */ + @Override + public String getName() { + return def.getName(); } + public int getArgumentCount() { - + return def.getArgs().length; } + + /** returns the arguments this function accepts + * + * @return + */ public VariableValue[] getArguments() { - + Arg[] args = def.getArgs(); + VariableValue[] val = new VariableValue[args.length]; + int i=0; + for ( Arg arg : args ) + val[i++] = VariableValue.toVariableValue( + env, + this, + arg.getDefault() + ); + return val; } + + /** returns all the other functions that this function calls + * + * @return + */ public FunctionCall[] getCalls() { - + ArrayList<FunctionCall> out = new ArrayList<FunctionCall>(2); + if (def instanceof Function && ((Function)def)._statement instanceof BlockStatement) { + String call_name, class_name; + int i; + AbstractFunction f; + QuercusClass qc; + for ( Statement st : ((BlockStatement)((Function)def)._statement)._statements ) { + if (st instanceof ExprStatement) { + Expr expr = ((ExprStatement)st)._expr; + if (expr instanceof CallExpr) { + CallExpr call = (CallExpr) expr; + call_name = call.getName(); + + if (call_name.contains("\\")) { + i = call_name.lastIndexOf('\\'); + class_name = call_name.substring(0, i); + call_name = call_name.substring(i+1); + + qc = env.findClass(class_name); + if (qc!=null) { + f = qc.findFunction(call_name); + if (f!=null) { + out.add( + new UserFunctionCall(env, this, call.getLocation(), + // TODO lookup the class this function actually belongs to + new FunctionDefinition(env, new ClassDefinition(env, qc.getClassDef()), f) + )); + continue; + } + } + } else { + f = env.findFunction(call_name); + if (f!=null) { + out.add( + new UserFunctionCall(env, this, call.getLocation(), + // TODO lookup the class this function actually belongs to + new FunctionDefinition(env, getClassDefinition(), f) + )); + continue; + } + } + // assume builtin + out.add(new BuiltinFunctionCall(env, this, call.getLocation(), call.getName())); + } + } + } + } + return out.toArray(new FunctionCall[out.size()]); } - } + + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial.startTag("pftt", "function"); + serial.attribute("pftt", "name", getName()); + serial.startTag("pftt", "arguments"); + for ( VariableValue arg : getArguments() ) + arg.serial(serial); + serial.endTag("pftt", "arguments"); + serial.startTag("pftt", "calls"); + for ( FunctionCall fc : getCalls() ) + fc.serial(serial); + serial.endTag("pftt", "calls"); + serial.endTag("pftt", "function"); + } + + @Override + public int hashCode() { + return def.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (o==this) + return true; + else if (o instanceof FunctionDefinition) + return ((FunctionDefinition)o).def.eq(this.def); + else + return toString().equals(o.toString()); + } + + @Override + public String toString() { + return getName(); + } + + } // end public static class FunctionDefinition -*/ -} +} // end public class PhpParser diff --git a/src/com/mostc/pftt/results/AbstractPhpUnitRW.java b/src/com/mostc/pftt/results/AbstractPhpUnitRW.java index 823c3ea..6caeb47 100644 --- a/src/com/mostc/pftt/results/AbstractPhpUnitRW.java +++ b/src/com/mostc/pftt/results/AbstractPhpUnitRW.java @@ -22,7 +22,7 @@ public abstract class AbstractPhpUnitRW extends AbstractTestResultRW { public abstract List<String> getTestNames(EPhpUnitTestStatus status); @Override public float passRate() { - return 100.0f * ((float)count(EPhpUnitTestStatus.PASS))/((float)getTestCount()); + return PhpResultPack.round1(100.0f * ((float)count(EPhpUnitTestStatus.PASS))/((float)getTestCount())); } public abstract PhpIni getPhpIni(); diff --git a/src/com/mostc/pftt/results/AbstractPhptRW.java b/src/com/mostc/pftt/results/AbstractPhptRW.java index 193b4d1..6b4b365 100644 --- a/src/com/mostc/pftt/results/AbstractPhptRW.java +++ b/src/com/mostc/pftt/results/AbstractPhptRW.java @@ -10,7 +10,7 @@ public abstract class AbstractPhptRW extends AbstractTestResultRW { public abstract String getTestPackVersion(); @Override public float passRate() { - return 100.0f * ((float)count(EPhptTestStatus.PASS)) / ((float)(count(EPhptTestStatus.PASS) + count(EPhptTestStatus.CRASH) + count(EPhptTestStatus.FAIL) + count(EPhptTestStatus.TIMEOUT))); + return PhpResultPack.round1(100.0f * ((float)count(EPhptTestStatus.PASS)) / ((float)(count(EPhptTestStatus.PASS) + count(EPhptTestStatus.CRASH) + count(EPhptTestStatus.FAIL) + count(EPhptTestStatus.TIMEOUT)))); } public abstract int count(EPhptTestStatus status); public abstract List<String> getTestNames(EPhptTestStatus status); diff --git a/src/com/mostc/pftt/results/CodeCoverageSummary.java b/src/com/mostc/pftt/results/CodeCoverageSummary.java index 9a2e24d..0a507e0 100644 --- a/src/com/mostc/pftt/results/CodeCoverageSummary.java +++ b/src/com/mostc/pftt/results/CodeCoverageSummary.java @@ -10,7 +10,7 @@ import org.xmlpull.v1.XmlSerializer; import com.mostc.pftt.scenario.ScenarioSet; -public class CodeCoverageSummary { +public class CodeCoverageSummary implements ISerializer { protected final HashMap<ScenarioSet,ScenarioSetCoverage> cov_map; protected final String test_pack_name_and_version; @@ -73,6 +73,7 @@ public class CodeCoverageSummary { cov.add(cc); } + @Override public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { ScenarioSetCoverage cov; serial.startTag("pftt", "codeCoverageSummary"); diff --git a/src/com/mostc/pftt/results/ConsoleManager.java b/src/com/mostc/pftt/results/ConsoleManager.java index a40cc46..5fb6951 100644 --- a/src/com/mostc/pftt/results/ConsoleManager.java +++ b/src/com/mostc/pftt/results/ConsoleManager.java @@ -62,6 +62,8 @@ public interface ConsoleManager { */ @Nonnegative public int getRunTestTimesAll(); + @Nonnegative + public int getRunTestPack(); /** should test cases be run in (TRUE) random order (different order every time) or normal order (FALSE) * * @see -randomize_order console option diff --git a/src/com/mostc/pftt/results/ISerializer.java b/src/com/mostc/pftt/results/ISerializer.java new file mode 100644 index 0000000..4b930b7 --- /dev/null +++ b/src/com/mostc/pftt/results/ISerializer.java @@ -0,0 +1,9 @@ +package com.mostc.pftt.results; + +import java.io.IOException; + +import org.xmlpull.v1.XmlSerializer; + +public interface ISerializer { + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException; +} diff --git a/src/com/mostc/pftt/results/LocalConsoleManager.java b/src/com/mostc/pftt/results/LocalConsoleManager.java index e2dc118..636d248 100644 --- a/src/com/mostc/pftt/results/LocalConsoleManager.java +++ b/src/com/mostc/pftt/results/LocalConsoleManager.java @@ -26,7 +26,7 @@ import com.mostc.pftt.util.ErrorUtil; public class LocalConsoleManager implements ConsoleManager { protected final boolean overwrite, debug_all, results_only, show_gui, disable_debug_prompt, dont_cleanup_test_pack, phpt_not_in_place, pftt_debug, no_result_file_for_pass_xskip_skip, randomize_order, thread_safety, skip_smoke_tests, restart_each_test_all, no_restart_all, ini_actual_all; - protected final int run_test_times_all, run_test_times_list_times, run_group_times, run_group_times_list_times, max_test_read_count, thread_count, delay_between_ms, suspend_seconds, run_count; + protected final int run_test_times_all, run_test_pack, run_test_times_list_times, run_group_times, run_group_times_list_times, max_test_read_count, thread_count, delay_between_ms, suspend_seconds, run_count; protected String source_pack; protected PhpDebugPack debug_pack; protected PhptDebuggerFrame phpt_gui; @@ -35,10 +35,10 @@ public class LocalConsoleManager implements ConsoleManager { protected List<String> debug_list, run_test_times_list, run_group_times_list, skip_list; public LocalConsoleManager() { - this(null, null, false, false, false, false, true, false, true, false, true, false, 1, true, 1, 1, 1, null, null, null, null, true, 0, 0, false, false, 0, 0, 0, false); + this(null, null, false, false, false, false, true, false, true, false, true, false, 1, 1, true, 1, 1, 1, null, null, null, null, true, 0, 0, false, false, 0, 0, 0, false); } - public LocalConsoleManager(String source_pack, PhpDebugPack debug_pack, boolean overwrite, boolean debug_all, boolean results_only, boolean show_gui, boolean disable_debug_prompt, boolean dont_cleanup_test_pack, boolean phpt_not_in_place, boolean pftt_debug, boolean no_result_file_for_pass_xskip_skip, boolean randomize_order, int run_test_times_all, boolean thread_safety, int run_test_times_list_times, int run_group_times, int run_group_times_list_times, List<String> debug_list, List<String> run_test_times_list, List<String> run_group_times_list, List<String> skip_list, boolean skip_smoke_tests, int max_test_read_count, int thread_count, boolean restart_each_test_all, boolean no_restart_all, int delay_between_ms, int run_count, int suspend_seconds, boolean ini_actual_all) { + public LocalConsoleManager(String source_pack, PhpDebugPack debug_pack, boolean overwrite, boolean debug_all, boolean results_only, boolean show_gui, boolean disable_debug_prompt, boolean dont_cleanup_test_pack, boolean phpt_not_in_place, boolean pftt_debug, boolean no_result_file_for_pass_xskip_skip, boolean randomize_order, int run_test_times_all, int run_test_pack, boolean thread_safety, int run_test_times_list_times, int run_group_times, int run_group_times_list_times, List<String> debug_list, List<String> run_test_times_list, List<String> run_group_times_list, List<String> skip_list, boolean skip_smoke_tests, int max_test_read_count, int thread_count, boolean restart_each_test_all, boolean no_restart_all, int delay_between_ms, int run_count, int suspend_seconds, boolean ini_actual_all) { this.source_pack = source_pack; this.debug_pack = debug_pack; this.overwrite = overwrite; @@ -53,6 +53,7 @@ public class LocalConsoleManager implements ConsoleManager { this.randomize_order = randomize_order; this.run_test_times_all = run_test_times_all; this.thread_safety = thread_safety; + this.run_test_pack = run_test_pack; this.run_test_times_list_times = run_test_times_list_times; this.run_group_times = run_group_times; this.run_group_times_list_times = run_group_times_list_times; @@ -290,6 +291,11 @@ public class LocalConsoleManager implements ConsoleManager { public int getRunTestTimesAll() { return run_test_times_all; } + + @Override + public int getRunTestPack() { + return run_test_pack; + } @Override public boolean isRandomizeTestOrder() { diff --git a/src/com/mostc/pftt/results/PhpResultPackWriter.java b/src/com/mostc/pftt/results/PhpResultPackWriter.java index 43dfcf7..ddd34bf 100644 --- a/src/com/mostc/pftt/results/PhpResultPackWriter.java +++ b/src/com/mostc/pftt/results/PhpResultPackWriter.java @@ -14,6 +14,7 @@ import java.util.concurrent.LinkedBlockingQueue; import com.github.mattficken.io.StringUtil; import com.mostc.pftt.host.AHost; import com.mostc.pftt.host.LocalHost; +import com.mostc.pftt.main.Config; import com.mostc.pftt.model.TestCase; import com.mostc.pftt.model.app.EPhpUnitTestStatus; import com.mostc.pftt.model.app.PhpUnitSourceTestPack; @@ -53,6 +54,7 @@ public class PhpResultPackWriter extends PhpResultPack implements ITestResultRec protected final String test_pack_version; protected final Thread writer_thread; protected final boolean first_for_build; + protected final Config config; protected class UITestScenarioSetGroup { protected final HashMap<String,HashMap<ScenarioSetSetup,UITestWriter>> map; @@ -101,13 +103,15 @@ public class PhpResultPackWriter extends PhpResultPack implements ITestResultRec } } - public PhpResultPackWriter(LocalHost local_host, LocalConsoleManager cm, File telem_base_dir, PhpBuild build) throws Exception { - this(local_host, cm, telem_base_dir, build, null); + public PhpResultPackWriter(LocalHost local_host, LocalConsoleManager cm, File telem_base_dir, PhpBuild build, Config config) throws Exception { + this(local_host, cm, telem_base_dir, build, null, config); } - public PhpResultPackWriter(LocalHost local_host, LocalConsoleManager cm, File telem_base_dir, PhpBuild build, PhptSourceTestPack src_test_pack) throws Exception { + public PhpResultPackWriter(LocalHost local_host, LocalConsoleManager cm, File telem_base_dir, PhpBuild build, PhptSourceTestPack src_test_pack, Config config) throws Exception { super(local_host); + this.config = config; + ui_test_writer_map = new HashMap<AHost,HashMap<String,UITestScenarioSetGroup>>(16); phpt_writer_map = new HashMap<AHost,HashMap<ScenarioSetSetup,PhptResultWriter>>(16); phpunit_writer_map = new HashMap<AHost,HashMap<String,PhpUnitScenarioSetGroup>>(16); @@ -328,9 +332,12 @@ public class PhpResultPackWriter extends PhpResultPack implements ITestResultRec @Override public void handle() throws IOException { + config.processPhptTestResult(cm, this_result); PhptResultWriter w = getCreatePhptResultWriter(this_host, this_scenario_set_setup); w.writeResult(cm, this_host, this_scenario_set_setup, this_result); + this_result.extra = null; + this_result.code_coverage = null; // show on console // TODO @@ -374,6 +381,7 @@ public class PhpResultPackWriter extends PhpResultPack implements ITestResultRec @Override public void handle() throws IllegalArgumentException, IllegalStateException, IOException { + config.processPhpUnitTestResult(cm, this_result); PhpUnitScenarioSetGroup sg = getCreatePhpUnitResultWriter( this_host, this_scenario_set_setup, @@ -385,6 +393,8 @@ public class PhpResultPackWriter extends PhpResultPack implements ITestResultRec PhpUnitResultWriter w = sg.getWriter(this_scenario_set_setup); w.writeResult(cm.phpunit_gui!=null, this_result); + this_result.extra = null; + this_result.code_coverage = null; // show on console // TODO diff --git a/src/com/mostc/pftt/results/PhpUnitResultWriter.java b/src/com/mostc/pftt/results/PhpUnitResultWriter.java index 2510892..3270310 100644 --- a/src/com/mostc/pftt/results/PhpUnitResultWriter.java +++ b/src/com/mostc/pftt/results/PhpUnitResultWriter.java @@ -177,6 +177,7 @@ public class PhpUnitResultWriter extends AbstractPhpUnitRW { // write result itself result.serial(serial); + result.extra = null; // store name, status and run-time in CSV format all_csv_pw.print("'"); diff --git a/src/com/mostc/pftt/results/PhpUnitTestResult.java b/src/com/mostc/pftt/results/PhpUnitTestResult.java index c106b93..fc1f7a9 100644 --- a/src/com/mostc/pftt/results/PhpUnitTestResult.java +++ b/src/com/mostc/pftt/results/PhpUnitTestResult.java @@ -1,6 +1,9 @@ package com.mostc.pftt.results; import java.io.IOException; + +import javax.annotation.Nullable; + import org.xmlpull.v1.XmlSerializer; import com.github.mattficken.io.StringUtil; @@ -16,7 +19,7 @@ import com.mostc.pftt.scenario.ScenarioSetSetup; * */ -public class PhpUnitTestResult { +public class PhpUnitTestResult implements ISerializer { public final PhpUnitTestCase test_case; public final EPhpUnitTestStatus status; public final ScenarioSetSetup scenario_set; @@ -26,7 +29,8 @@ public class PhpUnitTestResult { protected String sapi_output, sapi_config; public PhpIni ini; public final float run_time_micros; - public final TestCaseCodeCoverage code_coverage; + public TestCaseCodeCoverage code_coverage; + @Nullable public ISerializer extra; public PhpUnitTestResult(PhpUnitTestCase test_case, EPhpUnitTestStatus status, ScenarioSetSetup scenario_set, Host host, String output, float run_time_micros, TestCaseCodeCoverage code_coverage) { this.test_case = test_case; @@ -80,6 +84,7 @@ public class PhpUnitTestResult { } } + @Override public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { serial(serial, shouldStoreAllInfo(status)); } @@ -166,9 +171,16 @@ public class PhpUnitTestResult { if (code_coverage!=null) { code_coverage.serial(serial); } + } else { + if (code_coverage!=null) { + code_coverage.serial(serial); + } } // + if (extra!=null) + extra.serial(serial); + serial.endTag(null, "testcase"); } // end public void serial diff --git a/src/com/mostc/pftt/results/PhptResultWriter.java b/src/com/mostc/pftt/results/PhptResultWriter.java index 8fabc44..cbc6379 100644 --- a/src/com/mostc/pftt/results/PhptResultWriter.java +++ b/src/com/mostc/pftt/results/PhptResultWriter.java @@ -193,7 +193,8 @@ public class PhptResultWriter extends AbstractPhptRW { // write result info in XML format serial.startDocument(null, null); // write result and reference to the XSL stylesheet - result.serialize(serial, store_all, StringUtil.repeat("../", 1+AHost.countUp(test_case_base_name, dir.getAbsolutePath()))+"/phptresult.xsl"); + result.serial(serial, store_all, StringUtil.repeat("../", 1+AHost.countUp(test_case_base_name, dir.getAbsolutePath()))+"/phptresult.xsl"); + result.extra = null; serial.endDocument(); serial.flush(); diff --git a/src/com/mostc/pftt/results/PhptTestResult.java b/src/com/mostc/pftt/results/PhptTestResult.java index e7a080d..242913a 100644 --- a/src/com/mostc/pftt/results/PhptTestResult.java +++ b/src/com/mostc/pftt/results/PhptTestResult.java @@ -36,7 +36,7 @@ import com.mostc.pftt.model.core.PhptTestCase; * */ -public class PhptTestResult { +public class PhptTestResult implements ISerializer { public Charset actual_cs; public AHost host; /** the result of the test case */ @@ -82,6 +82,7 @@ public class PhptTestResult { /** microseconds it took to run the test */ public float run_time_micros; public TestCaseCodeCoverage code_coverage; + @Nullable public ISerializer extra; protected PhptTestResult() { @@ -226,11 +227,12 @@ public class PhptTestResult { return c; } - public void serialize(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { - serialize(serial, shouldStoreAllInfo(status)||status==EPhptTestStatus.SKIP, null); + @Override + public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { + serial(serial, shouldStoreAllInfo(status)||status==EPhptTestStatus.SKIP, null); } - public void serialize(XmlSerializer serial, boolean include_all, String stylesheet) throws IllegalArgumentException, IllegalStateException, IOException { + public void serial(XmlSerializer serial, boolean include_all, String stylesheet) throws IllegalArgumentException, IllegalStateException, IOException { if (StringUtil.isNotEmpty(stylesheet)) { serial.processingInstruction("xml-stylesheet type=\"text/xsl\" href=\""+stylesheet+"\""); } @@ -374,8 +376,11 @@ public class PhptTestResult { if (test_case!=null) test_case.serialize(serial); + if (extra!=null) + extra.serial(serial); + serial.endTag(null, "phptResult"); - } // end public void serialize + } // end public void serial public static boolean shouldStoreAllInfo(EPhptTestStatus status) { return status==EPhptTestStatus.FAIL||status==EPhptTestStatus.XFAIL_WORKS||status==EPhptTestStatus.CRASH||status==EPhptTestStatus.BORK||status==EPhptTestStatus.UNSUPPORTED||status==EPhptTestStatus.TEST_EXCEPTION; diff --git a/src/com/mostc/pftt/results/TestCaseCodeCoverage.java b/src/com/mostc/pftt/results/TestCaseCodeCoverage.java index 9e12948..513244e 100644 --- a/src/com/mostc/pftt/results/TestCaseCodeCoverage.java +++ b/src/com/mostc/pftt/results/TestCaseCodeCoverage.java @@ -2,13 +2,22 @@ package com.mostc.pftt.results; import java.io.IOException; import java.util.HashMap; +import java.util.Set; import org.xmlpull.v1.XmlSerializer; import com.github.mattficken.io.StringUtil; import com.mostc.pftt.host.AHost; -public class TestCaseCodeCoverage { +/** Stores information about what parts of the code were executed, etc... + * + * @see PhpParser + * + * @author Matt Ficken + * + */ + +public class TestCaseCodeCoverage implements ISerializer { public static enum ELineState { EXECUTED, @@ -32,6 +41,10 @@ public class TestCaseCodeCoverage { this(host, null); } + public Set<String> getFileNames() { + return file_map.keySet(); + } + public int getTotalClassCount() { return class_count; } @@ -101,6 +114,7 @@ public class TestCaseCodeCoverage { setLineState(filename, line_num, ELineState.NOT_EXECUTABLE); } + @Override public void serial(XmlSerializer serial) throws IllegalArgumentException, IllegalStateException, IOException { // TODO XSL stylesheet serial.startTag("pftt", "codeCoverage"); @@ -115,7 +129,7 @@ public class TestCaseCodeCoverage { for ( String filename : file_map.keySet() ) { FileInfo file_info = file_map.get(filename); serial.startTag("pftt", "file"); - serial.attribute("pft", "filename", filename); + serial.attribute("pftt", "filename", filename); String[] lines = StringUtil.splitLines(file_info.php_code); for ( int line_num : file_info.line_map.keySet() ) { String line_tag; diff --git a/src/com/mostc/pftt/runner/AbstractLocalTestPackRunner.java b/src/com/mostc/pftt/runner/AbstractLocalTestPackRunner.java index a220bec..f0dce90 100644 --- a/src/com/mostc/pftt/runner/AbstractLocalTestPackRunner.java +++ b/src/com/mostc/pftt/runner/AbstractLocalTestPackRunner.java @@ -296,7 +296,21 @@ public abstract class AbstractLocalTestPackRunner<A extends ActiveTestPack, S ex protected abstract void setupStorageAndTestPack(ITestPackStorageDir storage_dir, List<T> test_cases) throws IOException, Exception; + public void forceClose() { + if (runner_state.get()==ETestPackRunnerState.NOT_RUNNING) + return; // already closed + close(); + + for ( TestPackThread<?> t : threads ) { + t.stopThisThread(); + } + } + public void close() { + if (runner_state.get()==ETestPackRunnerState.NOT_RUNNING) + return; // already closed + runner_state.set(ETestPackRunnerState.NOT_RUNNING); + if (sapi_scenario!=null) { // don't kill procs we're debugging sapi_scenario.close(cm, cm.isDebugAll() || cm.isDebugList()); @@ -728,26 +742,28 @@ public abstract class AbstractLocalTestPackRunner<A extends ActiveTestPack, S ex } protected void exec_jobs(TestCaseGroupKey group_key, LinkedBlockingQueue<T> jobs) { - if (cm.getRunCount() > 0 && test_count.get() > cm.getRunCount() ) { - // run maximum number of tests, don't run any more - return; - } - this.group_key = group_key; LinkedList<T> completed_tests = new LinkedList<T>(); this.jobs = jobs; while (shouldRun()) { + /*if (test_count.get() > 100 ) {//cm.getRunCount() > 0 && test_count.get() > cm.getRunCount() ) { + // run maximum number of tests, don't run any more + System.exit(0); + runner_state.set(ETestPackRunnerState.NOT_RUNNING); + break; + }*/ + // test_case = null; try { test_case = jobs.poll(5, TimeUnit.SECONDS); } catch ( InterruptedException ex ) {} - if (test_case==null && shouldRun()) { - if (jobs.isEmpty()) - break; - else + if (test_case==null) { + if (shouldRun() && !jobs.isEmpty()) continue; + else + break; } completed_tests.add(test_case); test_run_start_time.set(System.currentTimeMillis()); @@ -887,7 +903,11 @@ public abstract class AbstractLocalTestPackRunner<A extends ActiveTestPack, S ex } // end for test_run_start_time.set(0); - test_count.incrementAndGet(); + if (test_count.incrementAndGet() > cm.getRunCount() && cm.getRunCount() > 0 ) { + // run maximum number of tests, don't run any more + forceClose(); + break; + } Thread.yield(); } // end while } // end protected void exec_jobs diff --git a/src/com/mostc/pftt/runner/LocalPhpUnitTestPackRunner.java b/src/com/mostc/pftt/runner/LocalPhpUnitTestPackRunner.java index a11ecad..6ed2dc3 100644 --- a/src/com/mostc/pftt/runner/LocalPhpUnitTestPackRunner.java +++ b/src/com/mostc/pftt/runner/LocalPhpUnitTestPackRunner.java @@ -233,6 +233,7 @@ public class LocalPhpUnitTestPackRunner extends AbstractLocalTestPackRunner<PhpU for ( EPhpUnitTestStatus status : EPhpUnitTestStatus.values() ) { cm.println(EPrintType.CLUE, getClass(), status+" "+phpunit.count(status)+" tests"); } + cm.println(EPrintType.CLUE, getClass(), "Pass Rate(%): "+phpunit.passRate()); } @Override diff --git a/src/com/mostc/pftt/runner/LocalPhptTestPackRunner.java b/src/com/mostc/pftt/runner/LocalPhptTestPackRunner.java index 74012d7..d96794e 100644 --- a/src/com/mostc/pftt/runner/LocalPhptTestPackRunner.java +++ b/src/com/mostc/pftt/runner/LocalPhptTestPackRunner.java @@ -229,6 +229,7 @@ public class LocalPhptTestPackRunner extends AbstractLocalTestPackRunner<PhptAct for ( EPhptTestStatus status : EPhptTestStatus.values() ) { cm.println(EPrintType.CLUE, getClass(), status+" "+phpt.count(status)+" tests"); } + cm.println(EPrintType.CLUE, getClass(), "Pass Rate(%): "+phpt.passRate()); } @Override diff --git a/src/com/mostc/pftt/scenario/BuiltinWebServerScenario.java b/src/com/mostc/pftt/scenario/BuiltinWebServerScenario.java index 0bd5f92..aed134f 100644 --- a/src/com/mostc/pftt/scenario/BuiltinWebServerScenario.java +++ b/src/com/mostc/pftt/scenario/BuiltinWebServerScenario.java @@ -70,7 +70,10 @@ public class BuiltinWebServerScenario extends WebServerScenario { @Override public boolean isExpectedCrash(PhptTestCase test_case) { - return test_case.isNamed(BUILTIN_WEB_EXPECTED_CRASHES) || isSlowTest(test_case) || super.isExpectedCrash(test_case); + return test_case.isExtension("pdo") + || test_case.isNamed(BUILTIN_WEB_EXPECTED_CRASHES) + || isSlowTest(test_case) + || super.isExpectedCrash(test_case); } public static final Trie BUILTIN_WEB_EXPECTED_CRASHES = PhptTestCase.createNamed( @@ -224,7 +227,8 @@ public class BuiltinWebServerScenario extends WebServerScenario { @Override public boolean isSlowTest(PhptTestCase test_case) { - return test_case.isExtension(BUILTIN_WEB_SLOW_TESTS) || super.isSlowTest(test_case); + return test_case.isExtension(BUILTIN_WEB_SLOW_TESTS) + || super.isSlowTest(test_case); } public static final Trie BUILTIN_WEB_SLOW_TESTS = PhptTestCase.createExtensions("mbstring", diff --git a/src/com/mostc/pftt/scenario/SAPIScenario.java b/src/com/mostc/pftt/scenario/SAPIScenario.java index 752a4aa..e345992 100644 --- a/src/com/mostc/pftt/scenario/SAPIScenario.java +++ b/src/com/mostc/pftt/scenario/SAPIScenario.java @@ -164,6 +164,7 @@ public abstract class SAPIScenario extends AbstractSerialScenario { "zend/tests/unset_cv10.phpt", // TODO // + "ext/pdo_mysql/tests/pdo_mysql___construct_ini.phpt", "ext/pcre/tests/backtrack_limit.phpt", "ext/pcre/tests/recursion_limit.phpt", "ext/phar/tests/bug45218_slowtest.phpt", diff --git a/src/com/mostc/pftt/scenario/XDebugProfilingScenario.java b/src/com/mostc/pftt/scenario/XDebugProfilingScenario.java new file mode 100644 index 0000000..2854283 --- /dev/null +++ b/src/com/mostc/pftt/scenario/XDebugProfilingScenario.java @@ -0,0 +1,37 @@ +package com.mostc.pftt.scenario; + +import com.mostc.pftt.host.Host; +import com.mostc.pftt.model.core.PhpBuild; +import com.mostc.pftt.model.core.PhpIni; +import com.mostc.pftt.results.ConsoleManager; + +/** + * + * @see http://xdebug.org/docs/profiler + * + */ + +public class XDebugProfilingScenario extends XDebugScenario { + + @Override + public String getName() { + return "XDebug-Profiling"; + } + + @Override + public IScenarioSetup setup(ConsoleManager cm, Host host, PhpBuild build, PhpIni ini) { + if (super.setup(cm, host, build, ini)==SETUP_FAILED) + return SETUP_FAILED; + + ini.putSingle("xdebug.profiler_append", "0"); + ini.putSingle("xdebug.profiler_enable", "1"); + // TODO include test name + // include script name in file name + ini.putSingle("xdebug.profiler_output_name", "pftt.%p.%s.cachegrind"); + // TODO store in test-pack + ini.putSingle("xdebug.profiler_output_dir", host.getTempDir()); + + return SETUP_SUCCESS; + } + +} diff --git a/src/com/mostc/pftt/scenario/XDebugScenario.java b/src/com/mostc/pftt/scenario/XDebugScenario.java index 64aee57..a5c6ee7 100644 --- a/src/com/mostc/pftt/scenario/XDebugScenario.java +++ b/src/com/mostc/pftt/scenario/XDebugScenario.java @@ -29,7 +29,7 @@ import com.mostc.pftt.results.ConsoleManager; */ public class XDebugScenario extends DebugScenario { - + @Override public IScenarioSetup setup(ConsoleManager cm, Host host, PhpBuild build, PhpIni ini) { // TODO temp diff --git a/src/com/mostc/pftt/util/ValgrindMemoryCheckManager.java b/src/com/mostc/pftt/util/ValgrindMemoryCheckManager.java index 272d208..68a04be 100644 --- a/src/com/mostc/pftt/util/ValgrindMemoryCheckManager.java +++ b/src/com/mostc/pftt/util/ValgrindMemoryCheckManager.java @@ -48,6 +48,7 @@ public class ValgrindMemoryCheckManager extends DebuggerManager { } public class ValgrindDebugger extends Debugger { + protected AHost host; protected AHost.ExecHandle process; public ValgrindDebugger(AHost host, String command_line) throws Exception { @@ -84,20 +85,17 @@ public class ValgrindMemoryCheckManager extends DebuggerManager { @Override public RunRequest createRunRequest(ConsoleManager cm, String ctx_str) { - // TODO Auto-generated method stub - return null; + return host.createRunRequest(cm, ctx_str); } @Override public ExecOutput execOut(RunRequest req) { - // TODO Auto-generated method stub - return null; + req.setCommandline("valgrind --leak-check=yes "+req.getCommandline()); } @Override public ExecHandle execThread(RunRequest req) { - // TODO Auto-generated method stub - return null; + req.setCommandline("valgrind --leak-check=yes "+req.getCommandline()); } }