[php-src] master: Run tests in parallel by default (#22939)

Author: NickSdot (NickSdot) Committer: GitHub (web-flow) Pusher: arnaud-lb Date: 2026-08-10T11:33:43+02:00 Commit: https://github.com/php/php-src/commit/1d2ea5ce22cd941d1bbf073d08af2b04be99ecf3 Raw diff: https://github.com/php/php-src/commit/1d2ea5ce22cd941d1bbf073d08af2b04be99ecf3.diff Run tests in parallel by default (#22939) Runs the test suite in parallel by default; automatically selects up to 10 workers. Explicit -j1 remains the opt-out to run tests serial. Smaller batches improve balancing. Changed paths: A tests/run-test/automatic_worker_limit.phpt M .gitignore M NEWS M README.md M UPGRADING.INTERNALS M docs/source/miscellaneous/writing-tests.rst M ext/gd/tests/createfromwbmp2.phpt M ext/gd/tests/createfromwbmp2_extern.phpt M ext/zip/tests/oo_addglob_leak.phpt M ext/zlib/tests/readgzfile_basic.phpt M ext/zlib/tests/readgzfile_basic2.phpt M run-tests.php M sapi/cli/tests/010-2.phpt M sapi/cli/tests/010.phpt Diff: diff --git a/.gitignore b/.gitignore index b76b5a787caa..769ac62cc5d8 100644 --- a/.gitignore +++ b/.gitignore @@ -252,7 +252,7 @@ php # Test results generated by ./run-tests.php php_test_results_*.txt -# Temporary test information generated by ./run-tests.php +# Temporary test information generated by ./run-tests.php (kept for BC; now lives in tmp dir) /run-test-info.php # Temporary POST data placeholder files generated by ./run-tests.php diff --git a/NEWS b/NEWS index 8ea376a2c247..ae9a19c0cc87 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,9 @@ PHP NEWS operation depth. (iliaal) - Core: + . Changed run-tests.php to run in parallel by default, using up to 10 + automatically detected workers. Pass -j1 for sequential execution. + (NickSdot) . Changed run-tests.php to run test subprocesses without a shell where possible. (NickSdot) . Fixed GH-23083 (SEGV build_trace_args in zend_exceptions.c with diff --git a/README.md b/README.md index d83203d74c0b..80618f2d6f94 100644 --- a/README.md +++ b/README.md @@ -97,15 +97,15 @@ can be determined using nproc. PHP ships with an extensive test suite, the command make test is used after successful compilation of the sources to run this test suite. -It is possible to run tests using multiple cores by setting -jN in -TEST_PHP_ARGS or TESTS: +Tests run in parallel by default, using up to 10 detected logical processors. +Set -jN in TEST_PHP_ARGS or TESTS to override the worker count: ```shell make TEST_PHP_ARGS=-j4 test ``` -Shall run make test with a maximum of 4 concurrent jobs: Generally the maximum -number of jobs should not exceed the number of cores available. +This runs make test with a maximum of 4 concurrent jobs. Alternatively, +use -j1 to run tests sequentially. Use the TEST_PHP_ARGS or TESTS variable to test only specific directories: diff --git a/UPGRADING.INTERNALS b/UPGRADING.INTERNALS index 28cefb1e0afb..3c0d3b4f80fa 100644 --- a/UPGRADING.INTERNALS +++ b/UPGRADING.INTERNALS @@ -223,6 +223,9 @@ PHP 8.6 INTERNALS UPGRADE NOTES ======================== - Abstract: + . run-tests.php now runs in parallel by default, using up to 10 automatically + detected workers. Pass -j1 for sequential execution. --asan, --msan, and + Valgrind default to at most two workers. . Minimum required PHP version found on the host system for running scripts like build/gen_stub.php during development has been updated from 7.4 to 8.1. . build/gen_stub.php may now generate a _decl.h file in addition to diff --git a/docs/source/miscellaneous/writing-tests.rst b/docs/source/miscellaneous/writing-tests.rst index 8e17674ae481..4339e042e6a4 100644 --- a/docs/source/miscellaneous/writing-tests.rst +++ b/docs/source/miscellaneous/writing-tests.rst @@ -192,6 +192,11 @@ When you are testing your test case it's really important to make sure that you temporary resources (eg files) that you used in the test. There is a special `--CLEAN--` section to help you do this — see here <#clean>_. +Tests run in parallel by default. Mutable resources such as files, directories, ports, database +objects, and IPC identifiers must therefore be unique to each test. Read-only fixtures may be +shared. If a resource cannot be isolated, declare the narrowest applicable conflict using +`--CONFLICTS-- or a CONFLICTS` file. + Another good check is to look at what lines of code in the PHP source your test case covers. This is easy to do, there are some instructions on the `PHP Wiki <https://wiki.php.net/doc/articles/writing-tests>`_. diff --git a/ext/gd/tests/createfromwbmp2.phpt b/ext/gd/tests/createfromwbmp2.phpt index 4608c861323f..7007ff545d74 100644 --- a/ext/gd/tests/createfromwbmp2.phpt +++ b/ext/gd/tests/createfromwbmp2.phpt @@ -8,7 +8,7 @@ gd ?> --FILE-- <?php -$filename = __DIR__ . '/_tmp.wbmp'; +$filename = __DIR__ . '/_tmp_createfromwbmp2.wbmp'; $fp = fopen($filename,"wb"); if (!$fp) { exit("Failed to create <$filename>"); diff --git a/ext/gd/tests/createfromwbmp2_extern.phpt b/ext/gd/tests/createfromwbmp2_extern.phpt index 68895f9a3570..711f2e8ca3ed 100644 --- a/ext/gd/tests/createfromwbmp2_extern.phpt +++ b/ext/gd/tests/createfromwbmp2_extern.phpt @@ -4,7 +4,7 @@ imagecreatefromwbmp with invalid wbmp gd --FILE-- <?php -$filename = __DIR__ . '/_tmp.wbmp'; +$filename = __DIR__ . '/_tmp_createfromwbmp2_extern.wbmp'; $fp = fopen($filename,"wb"); if (!$fp) { exit("Failed to create <$filename>"); @@ -41,4 +41,4 @@ unlink($filename); --EXPECTF-- Warning: imagecreatefromwbmp(): %croduct of memory allocation multiplication would exceed INT_MAX, failing operation gracefully%win %s on line %d -Warning: imagecreatefromwbmp(): "%s_tmp.wbmp" is not a valid WBMP file in %s on line %d +Warning: imagecreatefromwbmp(): "%s_tmp_createfromwbmp2_extern.wbmp" is not a valid WBMP file in %s on line %d diff --git a/ext/zip/tests/oo_addglob_leak.phpt b/ext/zip/tests/oo_addglob_leak.phpt index 9040c5565f84..be7f92dccb90 100644 --- a/ext/zip/tests/oo_addglob_leak.phpt +++ b/ext/zip/tests/oo_addglob_leak.phpt @@ -12,7 +12,7 @@ if(!defined("GLOB_BRACE")) die ('skip requires GLOB_BRACE'); $dirname = __DIR__ . '/'; include $dirname . 'utils.inc'; -$dirname = __DIR__ . '/__tmp_oo_addglob2/'; +$dirname = __DIR__ . '/__tmp_oo_addglob_leak/'; $file = $dirname . 'test.zip'; @mkdir($dirname); @@ -38,7 +38,7 @@ var_dump($zip->addGlob($dirname . 'bar.*', GLOB_BRACE, $options)); <?php $dirname = __DIR__ . '/'; include $dirname . 'utils.inc'; -rmdir_rf(__DIR__ . '/__tmp_oo_addglob2/'); +rmdir_rf(__DIR__ . '/__tmp_oo_addglob_leak/'); ?> --EXPECTF-- array(1) { diff --git a/ext/zlib/tests/readgzfile_basic.phpt b/ext/zlib/tests/readgzfile_basic.phpt index ae829406e0d4..cf579879e2fc 100644 --- a/ext/zlib/tests/readgzfile_basic.phpt +++ b/ext/zlib/tests/readgzfile_basic.phpt @@ -10,7 +10,7 @@ is a very common test for all languages EOT; -$dirname = 'readgzfile_temp'; +$dirname = 'readgzfile_basic_temp'; $filename = $dirname.'/readgzfile_basic.txt.gz'; mkdir($dirname); $h = gzopen($filename, 'w'); @@ -19,9 +19,11 @@ gzclose($h); var_dump(readgzfile( $filename ) ); - -unlink($filename); -rmdir($dirname); +?> +--CLEAN-- +<?php +@unlink('readgzfile_basic_temp/readgzfile_basic.txt.gz'); +@rmdir('readgzfile_basic_temp'); ?> --EXPECT-- hello world diff --git a/ext/zlib/tests/readgzfile_basic2.phpt b/ext/zlib/tests/readgzfile_basic2.phpt index ff19bcd35e93..6edb4c85784a 100644 --- a/ext/zlib/tests/readgzfile_basic2.phpt +++ b/ext/zlib/tests/readgzfile_basic2.phpt @@ -10,7 +10,7 @@ is a very common test for all languages EOT; -$dirname = 'readgzfile_temp'; +$dirname = 'readgzfile_basic2_temp'; $filename = $dirname.'/readgzfile_basic2.txt'; mkdir($dirname); $h = fopen($filename, 'w'); @@ -19,9 +19,11 @@ fclose($h); var_dump(readgzfile( $filename ) ); - -unlink($filename); -rmdir($dirname); +?> +--CLEAN-- +<?php +@unlink('readgzfile_basic2_temp/readgzfile_basic2.txt'); +@rmdir('readgzfile_basic2_temp'); ?> --EXPECT-- hello world diff --git a/run-tests.php b/run-tests.php index 6eed6649aec7..ce6bd94db1e0 100755 --- a/run-tests.php +++ b/run-tests.php @@ -34,9 +34,9 @@ function show_usage(): void php run-tests.php [options] [files] [directories] Options: - -j<workers> Run up to <workers> simultaneous testing processes in parallel for - quicker testing on systems with multiple logical processors. - Note that this is experimental feature. + -j<workers> Run up to <workers> simultaneous testing processes. By default, + the worker count is detected automatically. Use -j1 to run + tests sequentially. -l <file> Read the testfiles to be executed from <file>. After the test has finished all failed tests are written to the same <file>. @@ -356,6 +356,7 @@ function main(): void $shuffle = false; $bless = false; $workers = null; + $workersExplicit = false; $context_line_count = 3; $num_repeats = 1; $show_progress = true; @@ -417,6 +418,7 @@ function main(): void switch ($switch) { case 'j': + $workersExplicit = true; $workers = substr($argv[$i], 2); if ($workers == 0 || !preg_match('/^\d+$/', $workers)) { error("'$workers' is not a valid number of workers, try e.g. -j16 for 16 workers"); @@ -646,6 +648,17 @@ function main(): void } } + if (!$workersExplicit && (!$selected_tests || count($test_files) > 1)) { + $workers = get_default_worker_count(); + if ($workers !== null + && ($valgrind !== null || isset($environment['SKIP_ASAN']))) { + $workers = min($workers, 2); + } + if ($workers !== null && !can_create_parallel_worker_socket()) { + $workers = null; + } + } + if ($online === null && !isset($environment['SKIP_ONLINE_TESTS'])) { $online = false; } @@ -808,6 +821,53 @@ function main(): void } } +function get_default_worker_count(): ?int +{ + if (IS_WINDOWS) { + $workerCount = getenv('NUMBER_OF_PROCESSORS'); + return is_string($workerCount) ? parse_default_worker_count($workerCount) : null; + } + + $commands = [ + 'nproc 2>/dev/null', + 'getconf _NPROCESSORS_ONLN 2>/dev/null', + 'getconf NPROCESSORS_ONLN 2>/dev/null', + 'sysctl -n hw.logicalcpu 2>/dev/null', + 'sysctl -n hw.ncpu 2>/dev/null', + ]; + foreach ($commands as $command) { + $workerCount = shell_exec($command); + if (is_string($workerCount) + && ($workerCount = parse_default_worker_count($workerCount)) !== null) { + return $workerCount; + } + } + + return null; +} + +function parse_default_worker_count(string $workerCount): ?int +{ + $workerCount = trim($workerCount); + if (preg_match('/^[0-9]+$/D', $workerCount) !== 1) { + return null; + } + + $workerCount = (int) $workerCount; + return $workerCount >= 2 ? min($workerCount, 10) : null; +} + +function can_create_parallel_worker_socket(): bool +{ + $socket = @stream_socket_server('tcp://127.0.0.1:0'); + if ($socket === false) { + return false; + } + + fclose($socket); + return true; +} + function verify_config(string $php): void { if (empty($php) || !file_exists($php)) { @@ -830,7 +890,7 @@ function write_information(array $user_tests, $phpdbg): void $escaped_no_file_cache = escaped_shell_string_from($no_file_cache); // Get info from php - $info_file = __DIR__ . '/run-test-info.php'; + $info_file = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'run-test-info-' . getmypid() . '.php'; @unlink($info_file); $php_info = '<?php echo " PHP_SAPI : " , PHP_SAPI , " @@ -1676,10 +1736,10 @@ function run_all_tests_parallel(array $test_files, array $env, ?string $redir_te // - If this is running a small enough number of tests, // reduce the batch size to give batches to more workers. $files = []; - $maxBatchSize = $valgrind ? 1 : ($shuffle ? 4 : 32); + $maxBatchSize = $valgrind ? 1 : 4; $averageFilesPerWorker = max(1, (int) ceil($totalFileCount / count($workerProcs))); $batchSize = min($maxBatchSize, $averageFilesPerWorker); - while (count($files) <= $batchSize && $file = array_pop($test_files)) { + while (count($files) < $batchSize && $file = array_pop($test_files)) { foreach ($fileConflictsWith[$file] as $conflictKey) { if (isset($activeConflicts[$conflictKey])) { $waitingTests[$conflictKey][] = $file; diff --git a/sapi/cli/tests/010-2.phpt b/sapi/cli/tests/010-2.phpt index 88fe1c832a11..ddf7315c8298 100644 --- a/sapi/cli/tests/010-2.phpt +++ b/sapi/cli/tests/010-2.phpt @@ -12,7 +12,7 @@ if (substr(PHP_OS, 0, 3) == 'WIN') { $php = getenv('TEST_PHP_EXECUTABLE_ESCAPED'); -$filename_txt = __DIR__."/010.test.txt"; +$filename_txt = __DIR__."/010-R.test.txt"; $filename_txt_escaped = escapeshellarg($filename_txt); $txt = ' diff --git a/sapi/cli/tests/010.phpt b/sapi/cli/tests/010.phpt index 356b69bebf91..80758f368ab9 100644 --- a/sapi/cli/tests/010.phpt +++ b/sapi/cli/tests/010.phpt @@ -14,7 +14,7 @@ $php = getenv('TEST_PHP_EXECUTABLE_ESCAPED'); $filename = __DIR__."/010.test.php"; $filename_escaped = escapeshellarg($filename); -$filename_txt = __DIR__."/010.test.txt"; +$filename_txt = __DIR__."/010-F.test.txt"; $filename_txt_escaped = escapeshellarg($filename_txt); $code = ' @@ -37,7 +37,7 @@ var_dump(shell_exec("cat $filename_txt_escaped | $php -n -F $filename_escaped")) --CLEAN-- <?php @unlink(__DIR__."/010.test.php"); -@unlink(__DIR__."/010.test.txt"); +@unlink(__DIR__."/010-F.test.txt"); ?> --EXPECT-- string(25) " diff --git a/tests/run-test/automatic_worker_limit.phpt b/tests/run-test/automatic_worker_limit.phpt new file mode 100644 index 000000000000..5bf6df07365e --- /dev/null +++ b/tests/run-test/automatic_worker_limit.phpt @@ -0,0 +1,107 @@ +--TEST-- +Automatic worker detection is capped for regular and instrumented runs +--SKIPIF-- +<?php +if (PHP_OS_FAMILY === 'Windows') { + die('skip requires a POSIX shell'); +} +?> +--ENV-- +TEST_PHP_FORK_SERVER=0 +--FILE-- +<?php +$root = __DIR__ . '/automatic_worker_limit_' . getmypid(); +$bin = $root . '/bin'; +$tests = $root . '/tests'; +mkdir($bin, recursive: true); +mkdir($tests); + +$nproc = $bin . '/nproc'; +file_put_contents($nproc, "#!/bin/sh\nprintf '64\\n'\n"); +chmod($nproc, 0755); + +$testFiles = []; +for ($i = 0; $i < 11; $i++) { + $testFiles[] = $file = $tests . "/$i.phpt"; + file_put_contents($file, <<<PHPT + --TEST-- + worker cap $i + --FILE-- + <?php echo "ok\\n"; ?> + --EXPECT-- + ok + PHPT); +} + +$environment = [ + 'PATH' => $bin . PATH_SEPARATOR . getenv('PATH'), + 'TEST_PHP_EXECUTABLE' => getenv('TEST_PHP_EXECUTABLE'), + 'TEST_PHP_FORK_SERVER' => '0', +]; +foreach (['TEMP', 'TMPDIR'] as $name) { + if (($value = getenv($name)) !== false) { + $environment[$name] = $value; + } +} + +$runTests = static function (array $arguments) use ($environment, $testFiles): array { + $process = proc_open( + [ + getenv('TEST_PHP_EXECUTABLE'), + dirname(__DIR__, 2) . '/run-tests.php', + '-q', + '--no-progress', + ...$arguments, + ...$testFiles, + ], + [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['redirect', 1], + ], + $pipes, + null, + $environment, + ); + fclose($pipes[0]); + $output = stream_get_contents($pipes[1]); + fclose($pipes[1]); + + return [proc_close($process), $output]; +}; + +[$exitCode, $output] = $runTests([]); +var_dump($exitCode); +var_dump(str_contains($output, 'Spawning 10 workers...')); +var_dump(str_contains($output, 'Spawning 11 workers...')); + +[$exitCode, $output] = $runTests(['--asan']); +var_dump($exitCode); +var_dump(str_contains($output, 'Spawning 2 workers...')); +var_dump(str_contains($output, 'Spawning 10 workers...')); + +[$exitCode, $output] = $runTests(['--asan', '-j3']); +var_dump($exitCode); +var_dump(str_contains($output, 'Spawning 3 workers...')); +?> +--CLEAN-- +<?php +foreach (glob(__DIR__ . '/automatic_worker_limit_*') ?: [] as $root) { + foreach (glob($root . '/tests/*.phpt') ?: [] as $file) { + unlink($file); + } + @unlink($root . '/bin/nproc'); + @rmdir($root . '/tests'); + @rmdir($root . '/bin'); + @rmdir($root); +} +?> +--EXPECT-- +int(0) +bool(true) +bool(false) +int(0) +bool(true) +bool(false) +int(0) +bool(true)