note 60255 deleted from function.fsockopen by nlopess
| From: | nlopess@php.net | Date: | Sat, 31 Dec 2005 17:26:10 +0000 |
| Subject: | note 60255 deleted from function.fsockopen by nlopess | ||
| References: | 1 | Groups: | php.notes |
| Request: | Send a blank email to php-notes+get-101050@lists.php.net to get a copy of this message | ||
Note Submitter: mike at nott dot org
Reason: bad code
----
For a singlethreaded crawl:
function mycrawler_single($url, $timeout=10, $maxredirs=1)
{
$urlinfo = parse_url($url);
if (empty($urlinfo['scheme'])) {$urlinfo = parse_url('http://'.$url);}
if (empty($urlinfo["path"])) {$urlinfo["path"]="/";}
if (empty($urlinfo['port']))
{
switch($urlinfo['scheme'])
{
case "http":
$urlinfo['port'] = 80;
break;
case "https":
$urlinfo['port'] = 443;
break;
}
}
if (isset($urlinfo["query"]))
{
$request = "GET
".$urlinfo["path"]."?".$urlinfo["query"]." ";
} else {
$request = "GET ".$urlinfo["path"]." ";
}
$request .= "HTTP/1.0\r\n";
$request .= "Host: ".$urlinfo['host']."\r\n";
$request .= "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)\r\n";
$request .= "Connection: close\r\n\r\n";
$fp = fsockopen($urlinfo['host'], $urlinfo['port'], $errno, $errstr,
$timeout);
if (!$fp)
{
echo "(".$errno.")".$errstr."\n";
}
else
{
fwrite($fp, $request);
while (!feof($fp))
{
$data .= fgets($fp, 4096);
}
fclose($fp);
$tmp = explode("\r\n\r\n", $data, 2);
$urlinfo['header'] = $tmp[0];
$urlinfo['html'] = $tmp[1];
if ((stripos($urlinfo['header'], "location:")) &&
($maxredirs > 0))
{
preg_match("/\r\nlocation:(.*)/i", $urlinfo['header'], $match);
if ($match)
{
$redirect = trim($match[1]);
echo "Redirecting to ".$redirect."\n";
$maxredirs--;
return mycrawler_single($redirect, $timeout, $maxredirs);
}
}
return $urlinfo;
}
}
Mike Nott
http://www.nott.org/blog/php-crawler.html