note 87320 modified in class.domdocument by danbrown
| From: | danbrown@php.net | Date: | Sun, 30 Nov 2008 18:06:29 +0000 |
| Subject: | note 87320 modified in class.domdocument by danbrown | ||
| References: | 1 | Groups: | php.notes |
| Request: | Send a blank email to php-notes+get-147442@lists.php.net to get a copy of this message | ||
Here is a simple web scraping example using the PHP DOM that tries to get the largest text body of a
HTML document. I needed it for a spider that had to show a short description for a page. It assumes
that document annotation can be the largest <div>, <td> or <p> element in the
page.
In the example I show a way to prevent a bug in the DOM as it sometimes just doesn't recognize
html encoding. It seems to work if you put charset meta tag right after the head tag of the
document.
<?php
$ch= curl_init();
curl_setopt ($ch, CURLOPT_URL, '...put url here...' );
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch,CURLOPT_VERBOSE,1);
curl_setopt($ch, CURLOPT_USERAGENT, 'set sth...');
curl_setopt ($ch, CURLOPT_REFERER, '...set sth...'); //just a fake referer
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch,CURLOPT_POST,0);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 20);
$html= curl_exec($ch);
$html1= curl_getinfo($ch);
//try to get page encoding as it was sent from server
if ($html1['content_type']){
$arr= explode('charset=',$html1['content_type']);
$csethdr= strtolower(trim($arr[1]));
} else {
$csethdr= false;
}
$cset= false;
$arr= array();
//This has to replace page meta tags for charset with utf-8, but it doesn't actually help(see
the bug info).
if (preg_match_all(
'/(<meta\s*http-equiv="Content-Type"\s*content="[^;]*;
\s*charset=([^"]*?)(?:"|\;)[^>]*>)/' //merge this line
,$html,$arr,PREG_PATTERN_ORDER)){
$cset= strtolower(trim($arr[2][0]));
if ($cset!='utf-8'||$cset!=$csethdr){
$new= str_replace($arr[2][0],'utf-8',$arr[1][0]);
$html= str_replace($arr[1][0],$new,$html);
$cset= $csethdr;
} else {
$cset= false;
}
if ($cset=='utf-8'){
$cset= false;
}
}
unset($arr);
if ($cset){
$html= iconv($cset,'utf-8',$html);
}
unset($cset);
//solve dom bug
$html=preg_replace('/<head[^>]*>/','<head>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=utf-8">
',$html);
$dom= new DOMDocument();
$dom->loadHTML($html);
$dom->preserveWhiteSpace = false;
function getMaxTextBody($dom){
$content = $dom->getElementsByTagname('div');
$content2= $dom->getElementsByTagname('td');
$content3= $dom->getElementsByTagname('p');
$new= array();
foreach ($content as $value) {
$new[]= $value;
unset($value);
}
unset($content);
foreach ($content2 as $value) {
$new[]= $value;
unset($value);
}
unset($content2);
foreach ($content3 as $value) {
$new[]= $value;
unset($value);
}
unset($content3);
$maxlen= 0;
$result= '';
foreach ($new as $item)
{
$str= $item->nodeValue;
if (strlen($str)>$maxlen){
$content1= $item->getElementsByTagName('div');
$content2= $item->getElementsByTagname('td');
$content3= $item->getElementsByTagname('p');
$contentnew= array();
foreach ($content1 as $value) {
$contentnew[]= $value;
unset($value);
}
unset($content1);
foreach ($content2 as $value) {
$contentnew[]= $value;
unset($value);
}
unset($content2);
foreach ($content3 as $value) {
$contentnew[]= $value;
unset($value);
}
unset($content3);
if (count($contentnew)==0){
$result= $str;
} else {
foreach ($contentnew as $value) {
$str1= getMaxTextBody($value);
$str2= $value->nodeValue;
//let's say largest body has more than 50% of the text in its parent
if (strlen($str1)*2<strlen($str2)){
$str1= $str2;
}
if (strlen($str1)*2>strlen($str)&&strlen($str1)>$maxlen){
$result= $str1;
} elseif (strlen($str1)>$maxlen){
$result= $str1;
}
$maxlen= strlen($result);
}
}
$maxlen= strlen($result);
unset($contnentnew);
}
}
unset($new);
return $result;
}
print getMaxTextBody($dom);
?>
--was--
Here is a simple web scraping example using the PHP DOM that tries to get the largest text body of a
HTML document. I needed it for a spider that had to show a short description for a page. It assumes
that document annotation can be the largest <div>, <td> or <p> element in the
page.
In the example I show a way to prevent a bug in the DOM as it sometimes just doesn't recognize
html encoding. It seems to work if you put charset meta tag right after the head tag of the
document.
<?
$ch= curl_init();
curl_setopt ($ch, CURLOPT_URL, '...put url here...' );
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch,CURLOPT_VERBOSE,1);
curl_setopt($ch, CURLOPT_USERAGENT, 'set sth...');
curl_setopt ($ch, CURLOPT_REFERER, '...set sth...'); //just a fake referer
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch,CURLOPT_POST,0);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 20);
$html= curl_exec($ch);
$html1= curl_getinfo($ch);
//try to get page encoding as it was sent from server
if ($html1['content_type']){
$arr= explode('charset=',$html1['content_type']);
$csethdr= strtolower(trim($arr[1]));
} else {
$csethdr= false;
}
$cset= false;
$arr= array();
//This has to replace page meta tags for charset with utf-8, but it doesn't actually help(see
the bug info).
if (preg_match_all(
'/(<meta\s*http-equiv="Content-Type"\s*content="[^;]*;
\s*charset=([^"]*?)(?:"|\;)[^>]*>)/' //merge this line
,$html,$arr,PREG_PATTERN_ORDER)){
$cset= strtolower(trim($arr[2][0]));
if ($cset!='utf-8'||$cset!=$csethdr){
$new= str_replace($arr[2][0],'utf-8',$arr[1][0]);
$html= str_replace($arr[1][0],$new,$html);
$cset= $csethdr;
} else {
$cset= false;
}
if ($cset=='utf-8'){
$cset= false;
}
}
unset($arr);
if ($cset){
$html= iconv($cset,'utf-8',$html);
}
unset($cset);
//solve dom bug
$html=preg_replace('/<head[^>]*>/','<head>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=utf-8">
',$html);
$dom= new DOMDocument();
$dom->loadHTML($html);
$dom->preserveWhiteSpace = false;
function getMaxTextBody($dom){
$content = $dom->getElementsByTagname('div');
$content2= $dom->getElementsByTagname('td');
$content3= $dom->getElementsByTagname('p');
$new= array();
foreach ($content as $value) {
$new[]= $value;
unset($value);
}
unset($content);
foreach ($content2 as $value) {
$new[]= $value;
unset($value);
}
unset($content2);
foreach ($content3 as $value) {
$new[]= $value;
unset($value);
}
unset($content3);
$maxlen= 0;
$result= '';
foreach ($new as $item)
{
$str= $item->nodeValue;
if (strlen($str)>$maxlen){
$content1= $item->getElementsByTagName('div');
$content2= $item->getElementsByTagname('td');
$content3= $item->getElementsByTagname('p');
$contentnew= array();
foreach ($content1 as $value) {
$contentnew[]= $value;
unset($value);
}
unset($content1);
foreach ($content2 as $value) {
$contentnew[]= $value;
unset($value);
}
unset($content2);
foreach ($content3 as $value) {
$contentnew[]= $value;
unset($value);
}
unset($content3);
if (count($contentnew)==0){
$result= $str;
} else {
foreach ($contentnew as $value) {
$str1= getMaxTextBody($value);
$str2= $value->nodeValue;
//let's say largest body has more than 50% of the text in its parent
if (strlen($str1)*2<strlen($str2)){
$str1= $str2;
}
if (strlen($str1)*2>strlen($str)&&strlen($str1)>$maxlen){
$result= $str1;
} elseif (strlen($str1)>$maxlen){
$result= $str1;
}
$maxlen= strlen($result);
}
}
$maxlen= strlen($result);
unset($contnentnew);
}
}
unset($new);
return $result;
}
print getMaxTextBody($dom);
?>
http://php.net/manual/en/class.domdocument.php