note 88772 modified in function.curl-multi-exec by danbrown
| From: | danbrown@php.net | Date: | Sat, 07 Feb 2009 00:00:16 +0000 |
| Subject: | note 88772 modified in function.curl-multi-exec by danbrown | ||
| References: | 1 | Groups: | php.notes |
| Request: | Send a blank email to php-notes+get-150327@lists.php.net to get a copy of this message | ||
I noticed some may still have a bit of a trouble using mcurl to continuously download stuff (adding
and removing stuff from a queue) despite my earlier example, so here's a more complete example:
<?php
// Urls to download
$urls = array();
for ($i=0; $i<20; $i++)
$urls[] = 'http://www.example.com/search?q='.base_convert(mt_rand(0,999999),
10, 36);
$threads = 5;
$timeout = 30;
$mcurl = curl_multi_init();
$threadsRunning = 0;
$urls_id = 0;
for(;;) {
// Fill up the slots
while ($threadsRunning < $threads && $urls_id < count($urls)) {
echo 'Adding download link: '.$urls[$urls_id].'<br>';
flush();
$ch = curl_init();
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_URL, $urls[$urls_id++]);
curl_multi_add_handle($mcurl, $ch);
$threadsRunning++;
}
// Check if done
if ($threadsRunning == 0 && $urls_id >= count($urls))
break;
// Let mcurl do it's thing
curl_multi_select($mcurl);
while(($mcRes = curl_multi_exec($mcurl, $mcActive)) == CURLM_CALL_MULTI_PERFORM) usleep(100000);
if($mcRes != CURLM_OK) break;
while($done = curl_multi_info_read($mcurl)) {
$ch = $done['handle'];
$done_url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
$done_content = curl_multi_getcontent($ch);
if(curl_errno($ch) == 0) {
$count = (preg_match('#of about <b>([^>]+)</b>#i', $done_content, $g) ?
$g[1] : 'many');
echo "Link <a href='$done_url'>$done_url</a> found $count
links.<br>\n";
flush();
} else {
echo "Link <a href='$done_url'>$done_url</a> failed:
".curl_error($ch)."<br>\n";
flush();
}
curl_multi_remove_handle($mcurl, $ch);
curl_close($ch);
$threadsRunning--;
}
}
curl_multi_close($mcurl);
echo 'Done.';
?>
[EDITED BY danbrown AT php DOT net: Edited with RFC 2606 in mind.]
--was--
I noticed some may still have a bit of a trouble using mcurl to continuously download stuff (adding
and removing stuff from a queue) despite my earlier example, so here's a more complete example:
<?php
// Urls to download
$urls = array();
for ($i=0; $i<20; $i++)
$urls[] = 'http://www.google.com/search?q='.base_convert(mt_rand(0,999999),
10, 36);
$threads = 5;
$timeout = 30;
$mcurl = curl_multi_init();
$threadsRunning = 0;
$urls_id = 0;
for(;;) {
// Fill up the slots
while ($threadsRunning < $threads && $urls_id < count($urls)) {
echo 'Adding download link: '.$urls[$urls_id].'<br>';
flush();
$ch = curl_init();
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_URL, $urls[$urls_id++]);
curl_multi_add_handle($mcurl, $ch);
$threadsRunning++;
}
// Check if done
if ($threadsRunning == 0 && $urls_id >= count($urls))
break;
// Let mcurl do it's thing
curl_multi_select($mcurl);
while(($mcRes = curl_multi_exec($mcurl, $mcActive)) == CURLM_CALL_MULTI_PERFORM) usleep(100000);
if($mcRes != CURLM_OK) break;
while($done = curl_multi_info_read($mcurl)) {
$ch = $done['handle'];
$done_url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
$done_content = curl_multi_getcontent($ch);
if(curl_errno($ch) == 0) {
$count = (preg_match('#of about <b>([^>]+)</b>#i', $done_content, $g) ?
$g[1] : 'many');
echo "Link <a href='$done_url'>$done_url</a> found $count
links.<br>\n";
flush();
} else {
echo "Link <a href='$done_url'>$done_url</a> failed:
".curl_error($ch)."<br>\n";
flush();
}
curl_multi_remove_handle($mcurl, $ch);
curl_close($ch);
$threadsRunning--;
}
}
curl_multi_close($mcurl);
echo 'Done.';
?>
Here's to hoping google doesn't sue me for using them in my example.
http://php.net/manual/en/function.curl-multi-exec.php