note 60256 added to function.fsockopen
| From: | mike at nott dot org | Date: | Sat, 31 Dec 2005 17:01:57 +0000 |
| Subject: | note 60256 added to function.fsockopen | ||
| Groups: | php.notes | ||
| Request: | Send a blank email to php-notes+get-101049@lists.php.net to get a copy of this message | ||
For multi-threaded crawling:
function mycrawler_multi($urls, $timeout=10, $maxredirects=1)
{
for ($i=0; $i<count($urls); $i++)
{
$urlinfo[$i] = parse_url($urls[$i]);
$maxredirs[$i] = $maxredirects;
if (empty($urlinfo[$i]['scheme'])) {$urlinfo[$i] = parse_url('http://'.$url);}
if (empty($urlinfo[$i]["path"]))
{$urlinfo[$i]["path"]="/";}
if (empty($urlinfo[$i]['port']))
{
switch($urlinfo[$i]['scheme'])
{
case "http":
$urlinfo[$i]['port'] = 80;
break;
case "https":
$urlinfo[$i]['port'] = 443;
break;
}
}
if (isset($urlinfo[$i]["query"]))
{
$request[$i] = "GET
".$urlinfo[$i]["path"]."?".$urlinfo[$i]["query"]." ";
} else {
$request[$i] = "GET ".$urlinfo[$i]["path"]." ";
}
$request[$i] .= "HTTP/1.0\r\n";
$request[$i] .= "Host: ".$urlinfo[$i]['host']."\r\n";
$request[$i] .= "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT
5.1)\r\n";
$request[$i] .= "Connection: close\r\n\r\n";
$fp[$i] = fsockopen($urlinfo[$i]['host'], $urlinfo[$i]['port'],
$urlinfo[$i]['errno'], $urlinfo[$i]['errstr'], $timeout);
socket_set_blocking($fp[$i], false);
if (!$fp[$i])
{
echo
"(".$urlinfo[$i]['errno'].")".$urlinfo[$i]['errstr']."\n";
}
else
{
fwrite($fp[$i], $request[$i]);
}
}
$done = false;
$numdone = array();
while (!$done)
{
for ($i=0; $i<count($urls); $i++)
{
if (!feof($fp[$i]))
{
$data[$i] .= fgets($fp[$i], 4096);
}
elseif (empty($numdone[$i]))
{
$numdone[$i] = 1;
$tmp[$i] = explode("\r\n\r\n", $data[$i], 2);
$urlinfo[$i]['header'] = $tmp[$i][0];
$urlinfo[$i]['html'] = $tmp[$i][1];
if ((stripos($urlinfo[$i]['header'], "location:"))
&& ($maxredirs[$i] > 0))
{
preg_match("/\r\nlocation:(.*)/i",
$urlinfo[$i]['header'], $match[$i]);
if ($match[$i])
{
$redirect[$i] = trim($match[$i][1]);
echo "Redirecting to ".$redirect[$i]."\n";
$maxredirs[$i]--;
$urlinfo[$i] = mycrawler_single($redirect[$i], $timeout,
$maxredirs[$i]);
}
}
}
}
$done = (array_sum($numdone) == count($urls));
}
for ($i=0; $i<count($urls); $i++)
{
fclose($fp[$i]);
}
return $urlinfo;
}
Mike Nott
http://www.nott.org/blog/php-crawler.html
----
Manual Page -- http://www.php.net/manual/en/function.fsockopen.php
Edit -- http://master.php.net/manage/user-notes.php?action=edit+60256
Delete: added to the manual -- http://master.php.net/manage/user-notes.php?action=delete+60256&report=yes&reason=added+to+the+manual
Delete: bad code -- http://master.php.net/manage/user-notes.php?action=delete+60256&report=yes&reason=bad+code
Delete: spam -- http://master.php.net/manage/user-notes.php?action=delete+60256&report=yes&reason=spam
Delete: useless -- http://master.php.net/manage/user-notes.php?action=delete+60256&report=yes&reason=useless
Delete: non-english -- http://master.php.net/manage/user-notes.php?action=delete+60256&report=yes&reason=non-english
Delete: already in docs -- http://master.php.net/manage/user-notes.php?action=delete+60256&report=yes&reason=already+in+docs
Delete: other reasons -- http://master.php.net/manage/user-notes.php?action=delete+60256&report=yes
Reject -- http://master.php.net/manage/user-notes.php?action=reject+60256&report=yes
Search -- http://master.php.net/manage/user-notes.php