cvs: docweb /cron/weekly checkent /include lib_url_entities.inc.php /scripts checkent.php

sean Sat Jan 22 12:09:03 2005 EDT Modified files: /docweb/cron/weekly checkent /docweb/include lib_url_entities.inc.php /docweb/scripts checkent.php Log: more checkent cleaning && now with (optional) forking http://cvs.php.net/diff.php/docweb/cron/weekly/checkent?r1=1.3&r2=1.4&ty=u Index: docweb/cron/weekly/checkent diff -u docweb/cron/weekly/checkent:1.3 docweb/cron/weekly/checkent:1.4 --- docweb/cron/weekly/checkent:1.3 Sat Jan 22 01:56:59 2005 +++ docweb/cron/weekly/checkent Sat Jan 22 12:09:03 2005 @@ -2,6 +2,8 @@ . dirname $0/../../build-ops +NUMFORKS=10 + cd ${SCRIPTSDIR} ${PHP} checkent.php phpdoc > /dev/null ${PHP} checkent.php peardoc > /dev/null http://cvs.php.net/diff.php/docweb/include/lib_url_entities.inc.php?r1=1.2&r2=1.3&ty=u Index: docweb/include/lib_url_entities.inc.php diff -u docweb/include/lib_url_entities.inc.php:1.2 docweb/include/lib_url_entities.inc.php:1.3 --- docweb/include/lib_url_entities.inc.php:1.2 Sat Jan 22 10:15:26 2005 +++ docweb/include/lib_url_entities.inc.php Sat Jan 22 12:09:03 2005 @@ -19,7 +19,7 @@ | Mehdi Achour <didou@php.net> | | Sean Coates <sean@php.net> | +----------------------------------------------------------------------+ -$Id: lib_url_entities.inc.php,v 1.2 2005/01/22 15:15:26 sean Exp $ +$Id: lib_url_entities.inc.php,v 1.3 2005/01/22 17:09:03 sean Exp $ */ // user agent @@ -73,10 +73,30 @@ $schemes[] = 'ftp'; } +// timeout +define('URL_CONNECT_TIMEOUT', 10); + +// allow forking? +define('URL_ALLOW_FORK', function_exists('pcntl_fork') && isset($_ENV['NUMFORKS'])); +define('NUM_ALLOWED_FORKS', URL_ALLOW_FORK ? $_ENV['NUMFORKS'] : 0); + // SQLite DB file define('URL_ENT_SQLITE_FILE', SQLITE_DIR . "checkent_{$entType}.sqlite"); /** + * Opens a new SQLite connection + * + * @return resource SQLite connection + */ +function url_ent_sqlite_open() +{ + if (!($sqlite = sqlite_open(URL_ENT_SQLITE_FILE, 0666))) { + echo "Error creating database.\n"; + } + return $sqlite; +} + +/** * Handles relative HTTP URLs * * @param string $url URL to handle @@ -152,11 +172,17 @@ } $port = isset($url['port']) ? $url['port'] : $port; - if (!$fp = @fsockopen($scheme . $url['host'], $port)) { + $junk = ''; + if (!$fp = @fsockopen($scheme . $url['host'], $port, $junk, $junk, URL_CONNECT_TIMEOUT)) { return array(HTTP_CONNECT, array($num)); } else { - fputs($fp, "HEAD {$url['path']} HTTP/1.0\r\nHost: {$url['host']}\r\nUser-agent: ". DOCWEB_CRAWLER_USER_AGENT ."\r\nConnection: close\r\n\r\n"); + $query = "HEAD {$url['path']} HTTP/1.0\r\n" + ."Host: {$url['host']}\r\n" + ."User-agent: ". DOCWEB_CRAWLER_USER_AGENT ."\r\n" + ."Connection: close\r\n" + ."\r\n"; + fputs($fp, $query); $str = ''; while (!feof($fp)) { @@ -226,8 +252,11 @@ * @param array $result result of check_url() * @return void */ -function url_store_result(&$sqlite, $num, $name, $url, $result) +function url_store_result($sqlite, $num, $name, $url, $result) { + if (!$sqlite) { + $sqlite = url_ent_sqlite_open(); + } $return_val = isset($result[1][1]) ? $result[1][1] : ''; $sql = " INSERT http://cvs.php.net/diff.php/docweb/scripts/checkent.php?r1=1.15&r2=1.16&ty=u Index: docweb/scripts/checkent.php diff -u docweb/scripts/checkent.php:1.15 docweb/scripts/checkent.php:1.16 --- docweb/scripts/checkent.php:1.15 Sat Jan 22 10:15:26 2005 +++ docweb/scripts/checkent.php Sat Jan 22 12:09:03 2005 @@ -19,10 +19,11 @@ | Mehdi Achour <didou@php.net> | | Sean Coates <sean@php.net> | +----------------------------------------------------------------------+ -$Id: checkent.php,v 1.15 2005/01/22 15:15:26 sean Exp $ +$Id: checkent.php,v 1.16 2005/01/22 17:09:03 sean Exp $ */ set_time_limit(0); +$scriptBegin = time(); $inCli = true; require_once '../include/init.inc.php'; @@ -50,14 +51,15 @@ require_once '../include/lib_url_entities.inc.php'; +echo "DocWeb URL Entity Checker.\n"; +echo "Using forks? ". (NUM_ALLOWED_FORKS ? 'yes: '. NUM_ALLOWED_FORKS : 'no') ."\n\n"; + // create a new database (remove old first, if exists) if (is_file(URL_ENT_SQLITE_FILE) && !unlink(URL_ENT_SQLITE_FILE)) { echo "Error removing old database.\n"; die(); } -if (!($sqlite = sqlite_open(URL_ENT_SQLITE_FILE, 0666))) { - echo "Error creating database.\n"; -} +$sqlite = url_ent_sqlite_open(); // Table creation $sqlCreateMeta = " @@ -100,7 +102,7 @@ $entity_names = $entities_found[1]; $entity_urls = $entities_found[3]; -echo "Found: ". count($entity_urls) ."URLs\n"; +echo "Found: ". count($entity_urls) ." URLs\n"; // log start time && schemes in DB $sql = " @@ -112,10 +114,49 @@ "; sqlite_query($sqlite, $sql); -// Walk through entities found -foreach ($entity_urls as $num => $entity_url) { - echo "Checking: $entity_url\n"; - url_store_result($sqlite, $num, $entity_names[$num], $entity_url, check_url($num, $entity_url)); +if (URL_ALLOW_FORK) { + // use the forking method ... MUCH faster + declare(ticks=1); + $children = 0; + for ($num=0; $num<count($entity_urls); $num++) { + $url = $entity_urls[$num]; + $name = $entity_names[$num]; + if ($children < NUM_ALLOWED_FORKS) { + $pid = pcntl_fork(); + if ($pid) { + // parent + //echo "Forked: $pid\n"; + ++$children; + } else { + // child + echo "[$num] (". getmypid() .") Checking: $url\n"; + url_store_result(FALSE, $num, $name, $url, check_url($num, $url)); + exit(); + } + } else { + // enough $children + $status = 0; + $child = pcntl_wait($status); + --$children; + echo "Child: $child exited with status $status ($children remain)\n"; + } + } + + while ($children) { + $status = 0; + $child = pcntl_wait($status); + --$children; + echo "Child: $child exited with status $status ($children remain)\n"; + } + +} else { + // no forking + // walk through entities found + foreach ($entity_urls as $num => $entity_url) { + echo "[$num] Checking: $entity_url\n"; + url_store_result($sqlite, $num, $entity_names[$num], $entity_url, check_url($num, $entity_url)); + } + ++$num; // (for the count) } // log end time in DB @@ -127,6 +168,10 @@ "; sqlite_query($sqlite, $sql); -echo "Done.\n"; +$elapsed = time() - $scriptBegin; + +echo "\n"; +echo "Checked $num URLs.\n"; +echo "Completed in $elapsed seconds.\n"; ?>