cvs: SVNROOT / run-conversion.php

gwynne Sat Jun 20 09:50:55 2009 UTC Modified files: /SVNROOT run-conversion.php Log: - Heavily commented everything - run_command() can now return exit status to caller if desired - cp() uses DIR_SEP instead of hardcoded / - Added -d/--noclean option to leave temporary files in place - Rewrite symlink->svn:externals code to hopefully work correctly

http://cvs.php.net/viewvc.cgi/SVNROOT/run-conversion.php?r1=1.60&r2=1.61&diff_format=u Index: SVNROOT/run-conversion.php diff -u SVNROOT/run-conversion.php:1.60 SVNROOT/run-conversion.php:1.61 --- SVNROOT/run-conversion.php:1.60 Wed Dec 17 17:44:19 2008 +++ SVNROOT/run-conversion.php Sat Jun 20 09:50:55 2009 @@ -7,7 +7,7 @@ // ----------------------------------------------------------------------------------------------------------------------------- // Constants -$version = substr('$Revision: 1.60 $', strlen('$Revision: '), -2); +$version = substr('$Revision: 1.61 $', strlen('$Revision: '), -2); $passes = array( 'processcvs', // Process CVS modules @@ -26,6 +26,8 @@ // ----------------------------------------------------------------------------------------------------------------------------- // Utility functions + +// Print a verbose message if the current verbosity level allows function v($level, $message) { if ($GLOBALS['options']['verbose'] >= $level) { @@ -33,12 +35,14 @@ } } +// Note a fatal error and exit. Just die(), really. function error($message, $status = 1) { print $message; exit($status); } +// Returns true if either 1) We're running all passes, or 2) running the specific pass given function is_running_pass($pass) { return $GLOBALS['options']['pass'] === NULL || $GLOBALS['options']['pass'] == $pass; @@ -49,6 +53,7 @@ return !($value === '.' || $value === '..'); } +// A version of scandir() which doesn't return the . and .. entries function scandir_no_meta($directory) { if (func_num_args() == 3) { @@ -61,30 +66,38 @@ return array_filter($results, 'scandir_is_meta'); } -function run_command($command, $ignoreErrors = FALSE) +// Run a shell command with verbose commentary. Does NOT escape its argument. Can be made to ignore errors. +function run_command($command, $ignoreErrors = FALSE, &$exitStatus = NULL) { v(2, "Running '{$command}'..."); - exec($command, $output, $exitstatus); - if ($exitstatus != 0 && !$ignoreErrors) { + exec($command, $output, $lExitStatus); + if ($lExitStatus != 0 && !$ignoreErrors) { error("\nAn error occurred. Exit status was {$exitstatus}. Output:\n" . implode("\n", $output) . "\n"); - } else if ($exitstatus != 0) { + } else if ($lExitStatus != 0) { v(2, " warning: an error occurred.\n"); } else { v(2, " done.\n"); } + if ($exitStatus !== NULL) { + $exitStatus = $lExitStatus; + } return $output; } +// Recursive premissions-preserving verbose copy which appends to temp_path/cp-output function cp($source, $dest, $ignoreErrors = FALSE) { - run_command("cp -Rpv " . escapeshellarg($source) . ' ' . escapeshellarg($dest) . ' >> ' . escapeshellarg("{$GLOBALS['temp_path']}/cp-output"), $ignoreErrors); + run_command('cp -Rpv ' . escapeshellarg($source) . ' ' . escapeshellarg($dest) . + ' >> ' . escapeshellarg($GLOBALS['temp_path'] . DIR_SEP . 'cp-output'), $ignoreErrors); } +// Rename from source to dest. function mv($source, $dest) { run_command("mv " . escapeshellarg($source) . ' ' . escapeshellarg($dest)); } +// Recursive forced unlink function rm($path) { run_command("rm -Rf " . escapeshellarg($path)); @@ -133,6 +146,12 @@ 'default' => './phpsvn.options.skel', 'description' => 'Skeleton file to use for cvs2svn conversions.', 'action' => 'StoreString')); +$cmdline_parser->addOption('noclean', array( + 'short_name' => '-d', + 'long_name' => '--noclean', + 'description' => 'Don\'t remove temporary files when conversion is done.', + 'default' => FALSE, + 'action' => 'StoreTrue')); try { $result = $cmdline_parser->parse(); @@ -173,7 +192,8 @@ private $optionsFileContents = NULL; private $outputPath = NULL; private $seed = 0; - + + // Read options file, replacing temporary directory reference immediately (not variant per run). public function __construct($seed = NULL) { $this->optionsFileContents = file_get_contents($GLOBALS['options']['skeleton']); @@ -181,23 +201,32 @@ $this->seed = ($seed === NULL ? mt_rand() : $seed); } + // Update the SVN repository path public function setOutputPath($path) { $this->outputPath = $path; } + // Add a CVS module to the options. Special-cases several problematic modules. public function addCVSModule($moduleName, $isMetaModule = false) { + // Path to the module in the original CVS repository $cvspath = "{$GLOBALS['temp_path']}/cvsrepo/{$moduleName}/"; + // A meta-module is one which contains a list of other modules that need to be individually handled. + // Currently this includes only PEAR's modules and PECL's extensions if ($isMetaModule) { + // Prefix of module $modprefix = $moduleName . '/'; + // For each submodule in the meta-module... foreach(scandir_no_meta($cvspath) as $module) { + // Special case for problematic modules if (!is_dir($cvspath . $module) || count(scandir($cvspath . $module)) == 2 || $module == 'Attic' || ($moduleName == 'php/pecl' && $module == 'libextractor') || ($moduleName == 'pear/modules' && in_array($module, array('HTML_QuickForm_ComboBox', 'Services_Compete', 'XML_HTMLSax3')))) { continue; // empty dir } + // Add the submodule as a subdirectory of the meta-module. Notice the meta-module itself is never added. $this->optionsFileContents .= <<<EOEXTRA run_options.add_project( '{$cvspath}{$module}', @@ -211,6 +240,7 @@ EOEXTRA; } } else { + // Add the plain old module as a root directory $this->optionsFileContents .= <<<EOEXTRA run_options.add_project( '{$cvspath}', @@ -227,10 +257,13 @@ public function run() { + // Replace output repository path here now that it can't change again. $this->optionsFileContents = str_replace('@@@OUTPUT_PATH@@@', $this->outputPath, $this->optionsFileContents); + // Choose a random filename for the options file, which has to exist on disk (grumble) $filename = $GLOBALS['temp_path'] . DIR_SEP . 'cvs2svn.options.' . $this->seed; file_put_contents($filename, $this->optionsFileContents); + // Run the cvs2svn command with various pipes. $command = "exec cvs2svn --options=" . escapeshellarg($filename); v(2, "Running: '{$command}'..."); $cvs2svn_process = proc_open($command, array( @@ -240,6 +273,7 @@ ), $pipes, NULL, NULL); fclose($pipes[0]); $procinfo = array(); + // Sleep and let cvs2svn do its work. 500,000us is 1/2 second. Could probably push that even higher without noticable penalty. do { usleep(500000); $procinfo = proc_get_status($cvs2svn_process); @@ -285,9 +319,17 @@ $converter = new CVS2SVNConverter; + // Copy the real CVSROOT to a temporary staging area, since we'll be modding it heavily. $cvsPath = "{$GLOBALS['temp_path']}/cvsrepo"; cp($GLOBALS['options']['cvsroot'], $cvsPath, TRUE); + // Some renames: + // cvs/phpdoc => cvs/doc-base ; Will be moved into phpdoc/ later + // cvs/phpdoc/en => cvs/phpdoc-en ; Will be renamed to "en" and moved into phpdoc/ later + // cvs/pear => cvs/modules ; This will be moved into a "pear" directory later + // cvs/TSRM => cvs/php-src/TSRM ; Merge TSRM into php-src + // cvs/ZendEngine2 => cvs/php-src/ZendEngine2 ; Merge ZE2 into php-src + // cvs/pdo-specs => cvs/pecl/pdo/specs ; This didn't belong as a separate module to begin with and will be merged into PECL's PDO mv("{$cvsPath}/phpdoc", "{$cvsPath}/doc-base"); mv("{$cvsPath}/doc-base/en", "{$cvsPath}/phpdoc-en"); mv("{$cvsPath}/pear", "{$cvsPath}/modules"); @@ -295,14 +337,17 @@ mv("{$cvsPath}/ZendEngine2", "{$cvsPath}/php-src/ZendEngine2"); mv("{$cvsPath}/pdo-specs", "{$cvsPath}/pecl/pdo/specs"); + // There are several symlinks in php-src to PECL extensions which confuse cvs2svn, so remove them. These will be properly restored later. $links = array('bz2', 'hash', 'oci8', 'pdo', 'pdo_dblib', 'pdo_firebird', 'spl', 'soap', 'sqlite', 'tidy', 'xmlreader', 'xmlwriter', 'filter', 'ircg', 'json', 'pdo_mysql', 'pdo_oci', 'pdo_odbc', 'pdo_pgsql', 'pdo_sqlite', 'simplexml', 'fileinfo'); foreach ($links as $link) { rm("{$cvsPath}/php-src/ext/{$link}"); } - + + // Get a base list of CVS modules. $cvs_modules = scandir_no_meta($cvsPath); - + + // Setup the basic directory structure in the CVS root for restructure purposes mkdir("{$cvsPath}/gtk"); mkdir("{$cvsPath}/php"); mkdir("{$cvsPath}/pear"); @@ -310,23 +355,32 @@ mkdir("{$cvsPath}/phpdoc"); foreach ($cvs_modules as $cvs_module) { + // CVSROOT is not converted. Period. if ($cvs_module == 'CVSROOT') { continue; } + // Raw files in the cvs repo, empty directories (count(scandir()) == 2 (./..)), and selected obsolete modules are not converted. if (!is_dir("{$cvsPath}/{$cvs_module}") || count(scandir("{$cvsPath}/{$cvs_module}")) == 2 || in_array($cvs_module, array('smarty', 'smarty-web', 'php4.fubar', 'php4.unused', 'peardoc.backup', 'php3', 'phpfi', 'livedocs', 'Zend', 'functable', 'dialin', 'docstuff', 'jpgraph', 'phpoc_de', 'phpdoc-fa_IR', 'lxr', 'portal', 'pres', 'embed', 'imapd'))) { rm("{$cvsPath}/{$cvs_module}"); + // Modules belonging to PHP-GTK go into cvs/gtk/<module> } else if (in_array($cvs_module, array('php-gtk', 'php-gtk-doc', 'old-php-gtk-modules', 'php-gtk-web'))) { mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/gtk/{$cvs_module}"); $converter->addCVSModule("gtk/{$cvs_module}"); + // Modules belonging to the PHP base go into cvs/php/<module>. Note that this includes PECL! + // PECL is a meta-module. (see CVS2SVNConverter::addCVSModule) } else if (in_array($cvs_module, array('ZendAPI', 'php-src', 'pecl', 'php-objc', 'php-lang', 'win-installer', 'bindlib_w32', 'zlib', 'pdo-specs', 'php-internals-win', 'fastcgi-isapi'))) { mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/php/{$cvs_module}"); $converter->addCVSModule("php/{$cvs_module}", $cvs_module == 'pecl'); + // Modules belonging to PEAR (especially the "modules" directory which contains all PEAR modules) go into cvs/pear/<module> + // PEAR/modules is a meta-module. (see CVS2SVNConverter::addCVSModule) } else if (in_array($cvs_module, array('modules', 'pearbot', 'pear-core', 'peardoc', 'pearweb'))) { mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/pear/{$cvs_module}"); $converter->addCVSModule("pear/{$cvs_module}", $cvs_module == 'modules'); + // Modules belonging to websites, including the presentation, bugtracker, and pecl4win modules, go into cvs/web/<module> + // All modules that end in web are matched, and added with the "-?web" suffix stripped. } else if (substr($cvs_module, -3) == "web" || in_array($cvs_module, array('pres2', 'presentations', 'bugtracker', 'pecl4win'))) { if (substr($cvs_module, -4) == '-web') { $dest_cvs_module = substr($cvs_module, 0, -4); @@ -337,6 +391,8 @@ } mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/web/{$dest_cvs_module}"); $converter->addCVSModule("web/{$dest_cvs_module}"); + // Modules belonging to phpdoc (especially doc-base, see renamings) go into cvs/phpdoc/<module> + // All modules matching "phpdoc-(.*)" are matched, and added with the "phpdoc-" prefix stripped. } else if (strncmp($cvs_module, "phpdoc", 6) == 0 || $cvs_module == 'doc-base') { if (substr($cvs_module, 0, 7) == 'phpdoc-') { $dest_cvs_module = substr($cvs_module, 7); @@ -345,11 +401,13 @@ } mv("{$cvsPath}/{$cvs_module}", "{$cvsPath}/phpdoc/{$dest_cvs_module}"); $converter->addCVSModule("phpdoc/{$dest_cvs_module}"); + // All other modules go into cvs/<module>. There shouldn't be any of these. } else { $converter->addCVSModule($cvs_module); } } + // Strip any .svn dirs that might've been accidentally committed to CVS from other checkouts to avoid confusing the heck out of cvs2svn run_command('find ' . escapeshellarg("{$cvsPath}/pear/modules") . " -name '.svn' -type d -print0 | xargs -0 rm -Rf"); } @@ -397,6 +455,8 @@ // ignores pass function pass_ignores() { + // Recursively processes the entire converted SVN repository, converting .cvsignore files to svn:ignore properties. + // XXX THIS MAY NO LONGER BE NECESSARY AS OF cvs2svn 2.2.0! $svnPath = "{$GLOBALS['options']['svnroot']}/repository"; $coDir = "{$GLOBALS['temp_path']}/ignore_co"; $dirStack = array($svnPath); @@ -425,13 +485,17 @@ // link pass function pass_link() { + // Re-link the PECL extensions to php-src using svn:externals. + // XXX THERE IS A BUG HERE WHICH IS CAUSING EXTENSIONS TO BE LINKED INTO BRANCHES THEY SHOULDN'T BE $svnPath = "{$GLOBALS['options']['svnroot']}/repository/php/php-src"; - + + // Get a complete list of tags and branches in the converted SVN repository $rawTagList = run_command('exec svn ls file:///' . escapeshellarg("{$svnPath}/tags")); $rawBranchList = run_command('exec svn ls file:///' . escapeshellarg("{$svnPath}/branches")); $oldcwd = getcwd(); + // Creates a sparse checkout of the top levels ONLY of php/php-src/trunk, php/php-src/branches/*, and php/php-src/tags/* $sparseCoPath = "{$GLOBALS['temp_path']}/php-src-sparse-co"; $tmpfilePath = "{$GLOBALS['temp_path']}/sparse-co-output"; if (!file_exists($sparseCoPath)) { @@ -456,52 +520,47 @@ chdir($sparseCoPath); - $tagList = array(); - $branchList = array(); - foreach ($rawTagList as $tag) { - $realTag = substr($tag, 0, -1); - $xml = run_command('exec svn info --xml file:///' . escapeshellarg("{$svnPath}/tags/{$realTag}")); - $xml = implode("\n", $xml); - $tagList[$realTag] = strtotime(substr($xml, strpos($xml, '<date>') + 6, strlen('0000-00-00T00:00:00.000000Z'))); - run_command('exec svn propdel svn:externals ' . escapeshellarg('tags' . DIR_SEP . $realTag . DIR_SEP . 'ext') . ' 2>&1'); - } - foreach ($rawBranchList as $branch) { - $realBranch = substr($branch, 0, -1); - $xml = run_command('exec svn info --xml file:///' . escapeshellarg("{$svnPath}/branches/{$realBranch}")); - $xml = implode("\n", $xml); - $branchList[$realBranch] = strtotime(substr($xml, strpos($xml, '<date>') + 6, strlen('0000-00-00T00:00:00.000000Z'))); - run_command('exec svn propdel svn:externals ' . escapeshellarg('branches' . DIR_SEP . $realBranch . DIR_SEP . 'ext') . ' 2>&1'); - } - run_command('exec svn propdel svn:externals ' . escapeshellarg('trunk' . DIR_SEP . 'ext') . ' 2>&1'); - + // List of extensions that should be linked from PECL. + // Note: This list is accurate as of 2009-06-20 $links = array('bz2', 'hash', 'oci8', 'pdo', 'pdo_dblib', 'pdo_firebird', 'spl', 'soap', 'sqlite', 'tidy', 'xmlreader', 'xmlwriter', 'filter', 'json', 'pdo_mysql', 'pdo_oci', 'pdo_odbc', 'pdo_pgsql', 'pdo_sqlite', 'simplexml', 'fileinfo'); + // For each link, get its creation date from the ORIGINAL CVS root. + // Then, for each tag and branch, see whether the given link has a same-named directory. If so, make a link. Always make a link for trunk. $props = array(); - foreach ($links as $link) { - $info = lstat("{$GLOBALS['options']['cvsroot']}/php-src/ext/{$link}"); // Use the original CVS root here so we get proper timestamps - $createDate = $info['ctime']; - foreach ($tagList as $tag => $tagDate) { - $path = 'tags' . DIR_SEP . $tag . DIR_SEP . 'ext'; - if ($createDate <= $tagDate) { - $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/tags/{$tag} {$link}\n"; + foreach ($rawTagList as $tag) { + $tag = substr($tag, 0, -1); + $externalPath = $svnPath . "/php/pecl/{$link}/tags/{$tag}"; + run_command("exec svn ls file:///" . escapeshellarg($externalPath) . " 2>/dev/null", TRUE, $status); + if ($status === 0) { + v(2, "Tag {$tag} exists in ^/php/pecl/{$link}/tags. Adding external.\n"); + $path = 'tags' . DIR_SEP . $tag . DIR_SEP . 'ext'; + $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/tags/{$tag} {$link}"; + } else { + v(2, "Tag {$tag} does NOT exist in ^/php/pecl/{$link}/tags. NOT adding external.\n"); } } - foreach ($branchList as $branch => $branchDate) { - $path = 'branches' . DIR_SEP . $branch . DIR_SEP . 'ext'; - if ($createDate <= $branchDate) { - $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/branches/{$branch} {$link}\n"; + foreach ($rawBranchList as $branch) { + $branch = substr($branch, 0, -1); + $externalPath = $svnPath . "/php/pecl/{$link}/branches/{$branch}"; + run_command("exec svn ls file:///" . escapeshellarg($externalPath) . " 2>/dev/null", TRUE, $status); + if ($status === 0) { + v(2, "Branch {$branch} exists in ^/php/pecl/{$link}/branches. Adding external.\n"); + $path = 'branches' . DIR_SEP . $branch . DIR_SEP . 'ext'; + $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/branches/{$branch} {$link}"; + } else { + v(2, "Branch {$branch} does NOT exist in ^/php/pecl/{$link}/branches. NOT adding external.\n"); } } $path = 'trunk' . DIR_SEP . 'ext'; - $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/trunk {$link}\n"; + $props[$path] = (isset($props[$path]) ? $props[$path] : '') . "^/php/pecl/{$link}/trunk {$link}"; } - + foreach ($props as $path => $value) { run_command('exec svn propset svn:externals ' . escapeshellarg($value) . ' ' . escapeshellarg($path)); } - run_command('exec svn commit -m "[SVN CONVERSION] Reorganization in repository php-src."'); + run_command('exec svn commit -m "[SVN CONVERSION] Reorganization in repository php-src (committing externals in php-src)."'); chdir($oldcwd); } @@ -532,7 +591,7 @@ // ----------------------------------------------------------------------------------------------------------------------------- // Cleanup -if ($options['pass'] === NULL) { +if ($options['pass'] === NULL && $options['noclean'] === FALSE) { v(2, "Removing temporary files.\n"); system("rm -Rf " . escapeshellarg($temp_path)); }