com web/rmtools: refactor the snapshot script and related stuff: include/Branch.php script/snap.php

Commit: 68598445eb4e7050c66d4cb690ac3f90fd056c91 Author: Anatol Belski <ab@php.net> Fri, 9 Dec 2016 16:43:55 +0100 Parents: 44c32405778c561b7dc6163e97688467926ae3c5 Branches: master Link: http://git.php.net/?p=web/rmtools.git;a=commitdiff;h=68598445eb4e7050c66d4cb690ac3f90fd056c91 Log: refactor the snapshot script and related stuff Changed paths: M include/Branch.php M script/snap.php

diff --git a/include/Branch.php b/include/Branch.php index 8e7e33d..506a430 100644 --- a/include/Branch.php +++ b/include/Branch.php @@ -11,7 +11,6 @@ class Branch { public $config; private $repo; public $db_path; - private $has_new_revision; public $data = NULL; private $required_build_runs = 2; @@ -25,20 +24,34 @@ class Branch { $this->repo->setModule($this->config->getModule()); $this->repo->setBranch($this->config->getRepoBranch()); $this->db_path = __DIR__ . '/../data/db/' . $this->config->getName() . '.json'; + + $this->data = $this->readdata(); + /*if ($this->requiredBuldRunsReached()) { + $this->data->build_run = 0; + }*/ + + $this->addBuildList(); + } + + protected function readData() + { if (file_exists($this->db_path)) { - $this->data = json_decode(file_get_contents($this->db_path)); - if ($this->data->build_run == $this->required_build_runs) { - $this->data->build_run = 0; - } + $data = json_decode(file_get_contents($this->db_path)); } else { $data = new \StdClass; $data->revision_last = NULL; $data->revision_previous = NULL; $data->revision_last_exported = NULL; $data->build_run = 0; - $this->data = $data; } - $this->addBuildList(); + + return $data; + } + + protected function writeData() + { + $json = json_encode($this->data, JSON_PRETTY_PRINT); + return file_put_contents($this->db_path, $json); } private function addBuildList() @@ -57,28 +70,47 @@ class Branch { public function update() { - $this->data->build_run++; - $last_id = $this->repo->getLastCommitId(); - /* Either there's no db file at all yet, or this is the last required build run. */ - if ($this->requiredBuldRunsReached() && (strcasecmp($last_id, (string)$this->data->revision_last) != 0 || strcasecmp($last_id, $this->data->revision_previous) != 0) - || NULL == $this->data->revision_last && NULL == $this->data->revision_previous) { + + if (!$last_id) { + // XXX throw here + echo "last revision id is empty\n"; + return false; + } + + if ($this->data->build_run > 0 && $this->hasUnfinishedBuild()) { + $this->data->build_run++; + } else if ($this->hasNewRevision()) { $this->data->revision_previous = $this->data->revision_last; $this->data->revision_last = $last_id; - $json = json_encode($this->data); - file_put_contents($this->db_path, $json); - $this->has_new_revision = true; + $this->data->build_run = 1; + } else { + return false; } + + $this->writeData(); + + return true; } + public function hasUnfinishedBuild() + { + $exported = $this->getLastRevisionExported(); + $last = $this->getLastRevisionId(); + + return !$this->requiredBuldRunsReached() || substr_compare($last, $exported, 0, strlen($exported)) != 0; + } + public function requiredBuldRunsReached() { - return $this->data->build_run == $this->required_build_runs; + return $this->data->build_run >= $this->required_build_runs; } public function hasNewRevision() { - return $this->has_new_revision || $this->data->revision_previous == NULL; + $last = $this->repo->getLastCommitId(); + + return $last && !$this->isLastRevisionExported($last); } public function export($revision = false, $build_type = false, $zip = false, $is_zip = false) @@ -131,8 +163,7 @@ class Branch { Only set the revision exported, if we're on last required build run. */ if ($this->requiredBuldRunsReached()) { $this->data->revision_last_exported = $last_rev; - $json = json_encode($this->data); - file_put_contents($this->db_path, $json); + $this->writeData(); } } @@ -140,12 +171,16 @@ class Branch { { return $this->data->revision_last_exported; } + + public function isLastRevisionExported($rev) + { + $last_exported = $this->getLastRevisionExported(); + + return $last_exported && substr_compare($rev, $last_exported, 0, strlen($last_exported)) === 0; + } public function getLastRevisionId() { - if (!$this->data->revision_last) { - $this->update(); - } return $this->data->revision_last; } diff --git a/script/snap.php b/script/snap.php index f93349d..69f2343 100644 --- a/script/snap.php +++ b/script/snap.php @@ -5,15 +5,15 @@ include __DIR__ . '/../include/Tools.php'; use rmtools as rm; -if ($argc < 2 || $argc >3) { - echo "Usage: snapshot <config name> [force 1/0]\n"; +if ($argc < 2 || $argc > 4) { + echo "Usage: snapshot <branch> <config name> [force 1/0]\n"; exit(); } -$new_rev = false; +$have_build_run = false; $branch_name = $argv[1]; -$build_type = "all"; /* Build both ts and NTS for one given arch. This also complies with the top level script, otherwise it'd need a special SDK setup. */ -$force = isset($argv[2]) ? true : false; +$build_type = "all"; /* $argv[2] */ /* Build both ts and NTS for one given arch. This also complies with the top level script, otherwise it'd need a special SDK setup. */ +$force = isset($argv[3]) && $argv[3] ? true : false; $sdk_arch = getenv("PHP_SDK_ARCH"); if (!$sdk_arch) { throw new \Exception("Arch is empty, the SDK might not have been setup. "); @@ -21,17 +21,16 @@ if (!$sdk_arch) { $config_path = __DIR__ . '/../data/config/branch/' . $sdk_arch . '/' . $branch_name . '.ini'; $branch = new rm\Branch($config_path); -$branch->update(); $branch_name = $branch->config->getName(); $branch_name_short = $branch->config->getBranch(); -$last_rev = $branch->getLastRevisionId(); echo "Running <" . realpath($config_path) . ">\n"; echo "\t$branch_name\n"; -echo "\tprevious revision was: " . $branch->getPreviousRevision() . "\n"; -echo "\tlast revision is: " . $branch->getLastRevisionId() . "\n"; -if ($force || $branch->hasNewRevision()) { + +if ($branch->hasNewRevision() || $branch->hasUnfinishedBuild() || $force) { + + /* Prepared rewritten part, which might be helpful for PGO integration later. */ /*$builds_top = $branch->getBuildList('windows'); for ($i = 0; $i < count($builds_top) && ($force || $branch->hasNewRevision()); $i++) { @@ -41,180 +40,193 @@ for ($i = 0; $i < count($builds_top) && ($force || $branch->hasNewRevision()); $ echo "Unknown build type '{$builds_top[$i]}', skip\n"; continue; }*/ + + if (!$branch->update()) { + goto out_here; + } - if ($force || substr_compare($last_rev, $branch->getLastRevisionExported(), 0, 7) != 0) { - $new_rev = true; - echo "processing revision $last_rev\n"; - if ($branch->config->getPGO() == 1) { // Check revision to maintain concurrent builds - $fh = fopen(getenv("PHP_RMTOOLS_LOCK_PATH") . "/snaps_".$build_type.".lock", "a"); - fwrite($fh, "$last_rev"); - fclose($fh); - if (strcmp($build_type, 'nts') == 0) { - if (!file_exists(getenv("PHP_RMTOOLS_LOCK_PATH") . "/snaps_ts.lock")) { - echo "Waiting for thread-safe build, exiting.\n"; + $last_rev = $branch->getLastRevisionId(); + $prev_rev = $branch->getPreviousRevision(); + + echo "\tprevious revision was: $prev_rev\n"; + echo "\tlast revision is: $last_rev\n"; + + $have_build_run = true; + echo "processing revision $last_rev\n"; + + if ($branch->config->getPGO() == 1) { // Check revision to maintain concurrent builds + $fh = fopen(getenv("PHP_RMTOOLS_LOCK_PATH") . "/snaps_".$build_type.".lock", "a"); + fwrite($fh, "$last_rev"); + fclose($fh); + if (strcmp($build_type, 'nts') == 0) { + if (!file_exists(getenv("PHP_RMTOOLS_LOCK_PATH") . "/snaps_ts.lock")) { + echo "Waiting for thread-safe build, exiting.\n"; + exit(0); + } + else { + $fh = fopen(getenv("PHP_RMTOOLS_LOCK_PATH") . "/snaps_ts.lock", "r"); + $data = fread($fh, filesize(getenv("PHP_RMTOOLS_LOCK_PATH") . "/snaps_ts.lock")); + fclose($fh); + if (!preg_match("/$last_rev/", $data)) { + echo "Revision mismatch on concurrent builds, waiting for ts build to complete\n"; exit(0); } - else { - $fh = fopen(getenv("PHP_RMTOOLS_LOCK_PATH") . "/snaps_ts.lock", "r"); - $data = fread($fh, filesize(getenv("PHP_RMTOOLS_LOCK_PATH") . "/snaps_ts.lock")); - fclose($fh); - if (!preg_match("/$last_rev/", $data)) { - echo "Revision mismatch on concurrent builds, waiting for ts build to complete\n"; - exit(0); - } - } } } + } - if (strlen($last_rev) == 40) { - $last_rev = substr($last_rev, 0, 7); - } - $src_original_path = $branch->createSourceSnap($build_type); + if (strlen($last_rev) == 40) { + $last_rev = substr($last_rev, 0, 7); + } + $src_original_path = $branch->createSourceSnap($build_type); - $build_dir_parent = $branch->config->getBuildLocation(); + $build_dir_parent = $branch->config->getBuildLocation(); - if (!is_dir($build_dir_parent)) { - echo "Invalid build location <$build_dir_parent>\n"; - exit(-1); - } + if (!is_dir($build_dir_parent)) { + echo "Invalid build location <$build_dir_parent>\n"; + exit(-1); + } - $toupload_dir = TMP_DIR . '/' . $branch_name . '/r' . $last_rev . '-builds/'; - if (!is_dir($toupload_dir)) { - mkdir($toupload_dir, 0655, true); - } + $toupload_dir = TMP_DIR . '/' . $branch_name . '/r' . $last_rev . '-builds/'; + if (!is_dir($toupload_dir)) { + mkdir($toupload_dir, 0655, true); + } - if (!is_dir($toupload_dir . '/logs')) { - mkdir($toupload_dir . '/logs', 0655, true); - } + if (!is_dir($toupload_dir . '/logs')) { + mkdir($toupload_dir . '/logs', 0655, true); + } - copy($src_original_path . '.zip', $toupload_dir . '/' . $branch_name . '-src-r'. $last_rev . '.zip'); - $builds = $branch->getBuildList('windows'); + copy($src_original_path . '.zip', $toupload_dir . '/' . $branch_name . '-src-r'. $last_rev . '.zip'); + $builds = $branch->getBuildList('windows'); - $has_build_errors = false; - $build_errors = array(); + $has_build_errors = false; + $build_errors = array(); - foreach ($builds as $build_name) { - if (strcmp($build_type, 'all') != 0) { - if (substr_compare($build_name, $build_type, 0, 2) != 0) { // i.e. nts-windows-vc9-x86 - continue; - } - else { - echo "Starting build for $build_name\n"; - } + foreach ($builds as $build_name) { + if (strcmp($build_type, 'all') != 0) { + if (substr_compare($build_name, $build_type, 0, 2) != 0) { // i.e. nts-windows-vc9-x86 + continue; } - - $build_src_path = realpath($build_dir_parent) . DIRECTORY_SEPARATOR . $build_name; - $log = rm\exec_single_log('mklink /J ' . $build_src_path . ' ' . $src_original_path); - if (!file_exists($build_src_path)) { - throw new \Exception("Couldn't link '$src_original_path' to '$build_src_path'"); + else { + echo "Starting build for $build_name\n"; } + } - $build = $branch->createBuildInstance($build_name); - try { - $build->setSourceDir($build_src_path); + $build_src_path = realpath($build_dir_parent) . DIRECTORY_SEPARATOR . $build_name; + $log = rm\exec_single_log('mklink /J ' . $build_src_path . ' ' . $src_original_path); + if (!file_exists($build_src_path)) { + throw new \Exception("Couldn't link '$src_original_path' to '$build_src_path'"); + } - echo "Updating dependencies\n"; - /* XXX Pass stability from script arg. */ - $ret = $build->updateDeps("staging"); - echo $ret["log"] . "\n"; + $build = $branch->createBuildInstance($build_name); + try { + $build->setSourceDir($build_src_path); - echo "running build in <$build_src_path>\n"; - $build->buildconf(); - if ($branch->config->getPGO() == 1) { - echo "Creating PGI build\n"; - $build->configure(' "--enable-pgi" '); - } - else { - $build->configure(); - } - $build->make(); - $html_make_log = $build->getMakeLogParsed(); - $build->makeArchive(); - } catch (Exception $e) { - echo $e->getMessage() . "\n"; - echo $build->log_buildconf; - } + echo "Updating dependencies\n"; + /* XXX Pass stability from script arg. */ + $ret = $build->updateDeps("staging"); + echo $ret["log"] . "\n"; + + echo "running build in <$build_src_path>\n"; + $build->buildconf(); if ($branch->config->getPGO() == 1) { - if ($build->archive_path) { - echo "Running pgo_controller.ps1 with PGI build at $build->archive_path, ver=$branch_name, opcache=0\n"; - $cmd = 'c:\windows\system32\WindowsPowerShell\v1.0\powershell.exe -NonInteractive -Command C:\php-sdk\pgo-build\pgo_controller.ps1 -PHPBUILD '. $build->archive_path . ' -PHPVER ' . $branch_name; + echo "Creating PGI build\n"; + $build->configure(' "--enable-pgi" '); + } + else { + $build->configure(); + } + $build->make(); + $html_make_log = $build->getMakeLogParsed(); + $build->makeArchive(); + } catch (Exception $e) { + echo $e->getMessage() . "\n"; + echo $build->log_buildconf; + } + if ($branch->config->getPGO() == 1) { + if ($build->archive_path) { + echo "Running pgo_controller.ps1 with PGI build at $build->archive_path, ver=$branch_name, opcache=0\n"; + $cmd = 'c:\windows\system32\WindowsPowerShell\v1.0\powershell.exe -NonInteractive -Command C:\php-sdk\pgo-build\pgo_controller.ps1 -PHPBUILD '. $build->archive_path . ' -PHPVER ' . $branch_name; + $pgolog = rm\exec_single_log($cmd); + print_r($pgolog); + if ( preg_match('/5\.5/', $branch_name) ) { + echo "Running pgo_controller.ps1 with PGI build at $build->archive_path, ver=$branch_name, opcache=1\n"; + $cmd = 'c:\windows\system32\WindowsPowerShell\v1.0\powershell.exe -NonInteractive -Command C:\php-sdk\pgo-build\pgo_controller.ps1 -PHPBUILD '. $build->archive_path . ' -PHPVER ' . $branch_name . ' -OPCACHE 1'; $pgolog = rm\exec_single_log($cmd); print_r($pgolog); - if ( preg_match('/5\.5/', $branch_name) ) { - echo "Running pgo_controller.ps1 with PGI build at $build->archive_path, ver=$branch_name, opcache=1\n"; - $cmd = 'c:\windows\system32\WindowsPowerShell\v1.0\powershell.exe -NonInteractive -Command C:\php-sdk\pgo-build\pgo_controller.ps1 -PHPBUILD '. $build->archive_path . ' -PHPVER ' . $branch_name . ' -OPCACHE 1'; - $pgolog = rm\exec_single_log($cmd); - print_r($pgolog); - } - - echo "Creating PGO build\n"; - try { - $build->make(' clean-pgo'); - $build->configure(' "--with-pgo" ', false); - $build->make(); - $html_make_log = $build->getMakeLogParsed(); - $build->makeArchive(); - } catch (Exception $e) { - echo $e->getMessage() . "\n"; - echo $build->log_buildconf; - } } - } - if ($build->archive_path) { - copy($build->archive_path, $toupload_dir . '/php-' . $branch_name_short . '-' . $build_name . '-r'. $last_rev . '.zip'); - } - if ($build->archive_path) { - copy($build->devel_path, $toupload_dir . '/php-devel-pack-' . $branch_name_short . '-' . $build_name . '-r'. $last_rev . '.zip'); - } - if ($build->archive_path) { - copy($build->debug_path, $toupload_dir . '/php-debug-pack-' . $branch_name_short . '-' . $build_name . '-r'. $last_rev . '.zip'); - } - if ($build->test_path) { - $tmp = str_replace(array('-ts', '-nts'), array('',''), $build_name); - copy($build->test_path, $toupload_dir . '/php-test-pack-' . $branch_name_short . '-' . $tmp . '-r'. $last_rev . '.zip'); + echo "Creating PGO build\n"; + try { + $build->make(' clean-pgo'); + $build->configure(' "--with-pgo" ', false); + $build->make(); + $html_make_log = $build->getMakeLogParsed(); + $build->makeArchive(); + } catch (Exception $e) { + echo $e->getMessage() . "\n"; + echo $build->log_buildconf; + } } + } - file_put_contents($toupload_dir . '/logs/buildconf-' . $build_name . '-r'. $last_rev . '.txt', $build->log_buildconf); - file_put_contents($toupload_dir . '/logs/configure-' . $build_name . '-r'. $last_rev . '.txt', $build->log_configure); - file_put_contents($toupload_dir . '/logs/make-' . $build_name . '-r'. $last_rev . '.txt', $build->log_make); - file_put_contents($toupload_dir . '/logs/archive-' . $build_name . '-r'. $last_rev . '.txt', $build->log_archive); + if ($build->archive_path) { + copy($build->archive_path, $toupload_dir . '/php-' . $branch_name_short . '-' . $build_name . '-r'. $last_rev . '.zip'); + } + if ($build->archive_path) { + copy($build->devel_path, $toupload_dir . '/php-devel-pack-' . $branch_name_short . '-' . $build_name . '-r'. $last_rev . '.zip'); + } + if ($build->archive_path) { + copy($build->debug_path, $toupload_dir . '/php-debug-pack-' . $branch_name_short . '-' . $build_name . '-r'. $last_rev . '.zip'); + } + if ($build->test_path) { + $tmp = str_replace(array('-ts', '-nts'), array('',''), $build_name); + copy($build->test_path, $toupload_dir . '/php-test-pack-' . $branch_name_short . '-' . $tmp . '-r'. $last_rev . '.zip'); + } - $html_make_log = $build->getMakeLogParsed(); - file_put_contents($toupload_dir . '/logs/make-' . $build_name . '-r'. $last_rev . '.html', $html_make_log); - copy(__DIR__ . '/../template/log_style.css', $toupload_dir . '/logs/log_style.css'); + file_put_contents($toupload_dir . '/logs/buildconf-' . $build_name . '-r'. $last_rev . '.txt', $build->log_buildconf); + file_put_contents($toupload_dir . '/logs/configure-' . $build_name . '-r'. $last_rev . '.txt', $build->log_configure); + file_put_contents($toupload_dir . '/logs/make-' . $build_name . '-r'. $last_rev . '.txt', $build->log_make); + file_put_contents($toupload_dir . '/logs/archive-' . $build_name . '-r'. $last_rev . '.txt', $build->log_archive); - $stats = $build->getStats(); + $html_make_log = $build->getMakeLogParsed(); + file_put_contents($toupload_dir . '/logs/make-' . $build_name . '-r'. $last_rev . '.html', $html_make_log); + copy(__DIR__ . '/../template/log_style.css', $toupload_dir . '/logs/log_style.css'); - $json_filename = $build_name . '.json'; + $stats = $build->getStats(); - $json_data = array( - 'stats' => $stats, - 'has_php_pkg' => file_exists($build->archive_path), - 'has_debug_pkg' => file_exists($build->debug_path), - 'has_devel_pkg' => file_exists($build->devel_path), - 'has_test_pkg' => file_exists($build->test_path), - ); + $json_filename = $build_name . '.json'; - if ($stats['error'] > 0) { - $has_build_errors = true; - $build_errors[$build_name] = $build->compiler_log_parser->getErrors(); - $json_data['build_error'] = $build_errors[$build_name]; - } + $json_data = array( + 'stats' => $stats, + 'has_php_pkg' => file_exists($build->archive_path), + 'has_debug_pkg' => file_exists($build->debug_path), + 'has_devel_pkg' => file_exists($build->devel_path), + 'has_test_pkg' => file_exists($build->test_path), + ); - $json = json_encode($json_data); - file_put_contents($toupload_dir . '/' . $json_filename, $json); - rm\upload_build_result_ftp_curl($toupload_dir, $branch_name . '/r' . $last_rev); -// $build->clean(); - rmdir($build_src_path); + if ($stats['error'] > 0) { + $has_build_errors = true; + $build_errors[$build_name] = $build->compiler_log_parser->getErrors(); + $json_data['build_error'] = $build_errors[$build_name]; } - $branch->setLastRevisionExported($last_rev); + $json = json_encode($json_data); + file_put_contents($toupload_dir . '/' . $json_filename, $json); + +// $build->clean(); + rmdir($build_src_path); } -} + $src_dir = $branch_name . '/r' . $last_rev; + rm\upload_build_result_ftp_curl($toupload_dir, $src_dir); + + $branch->setLastRevisionExported($last_rev); + +} -if (!$new_rev) { +out_here: +if (!$have_build_run) { echo "no new revision.\n"; }