note 60255 added to function.fsockopen

For a singlethreaded crawl: function mycrawler_single($url, $timeout=10, $maxredirs=1) { $urlinfo = parse_url($url); if (empty($urlinfo['scheme'])) {$urlinfo = parse_url('http://'.$url);} if (empty($urlinfo["path"])) {$urlinfo["path"]="/";} if (empty($urlinfo['port'])) { switch($urlinfo['scheme']) { case "http": $urlinfo['port'] = 80; break; case "https": $urlinfo['port'] = 443; break; } } if (isset($urlinfo["query"])) { $request = "GET ".$urlinfo["path"]."?".$urlinfo["query"]." "; } else { $request = "GET ".$urlinfo["path"]." "; } $request .= "HTTP/1.0\r\n"; $request .= "Host: ".$urlinfo['host']."\r\n"; $request .= "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)\r\n"; $request .= "Connection: close\r\n\r\n"; $fp = fsockopen($urlinfo['host'], $urlinfo['port'], $errno, $errstr, $timeout); if (!$fp) { echo "(".$errno.")".$errstr."\n"; } else { fwrite($fp, $request); while (!feof($fp)) { $data .= fgets($fp, 4096); } fclose($fp); $tmp = explode("\r\n\r\n", $data, 2); $urlinfo['header'] = $tmp[0]; $urlinfo['html'] = $tmp[1]; if ((stripos($urlinfo['header'], "location:")) && ($maxredirs > 0)) { preg_match("/\r\nlocation:(.*)/i", $urlinfo['header'], $match); if ($match) { $redirect = trim($match[1]); echo "Redirecting to ".$redirect."\n"; $maxredirs--; return mycrawler_single($redirect, $timeout, $maxredirs); } } return $urlinfo; } } Mike Nott http://www.nott.org/blog/php-crawler.html ---- Manual Page -- http://www.php.net/manual/en/function.fsockopen.php Edit -- http://master.php.net/manage/user-notes.php?action=edit+60255 Delete: added to the manual -- http://master.php.net/manage/user-notes.php?action=delete+60255&report=yes&reason=added+to+the+manual Delete: bad code -- http://master.php.net/manage/user-notes.php?action=delete+60255&report=yes&reason=bad+code Delete: spam -- http://master.php.net/manage/user-notes.php?action=delete+60255&report=yes&reason=spam Delete: useless -- http://master.php.net/manage/user-notes.php?action=delete+60255&report=yes&reason=useless Delete: non-english -- http://master.php.net/manage/user-notes.php?action=delete+60255&report=yes&reason=non-english Delete: already in docs -- http://master.php.net/manage/user-notes.php?action=delete+60255&report=yes&reason=already+in+docs Delete: other reasons -- http://master.php.net/manage/user-notes.php?action=delete+60255&report=yes Reject -- http://master.php.net/manage/user-notes.php?action=reject+60255&report=yes Search -- http://master.php.net/manage/user-notes.php