note 12882 deleted from preface by jimw

Modification of my script above to remove a couple of bugs. To get a local copy of the annotated manual: Note 1: tested on RH Linux 6.1, will probably work on other systems with/without tweaking). Note 2: Several log files are created for debugging. Anything not under the Manual directory can be deleted. Note 3: the script is not perfect, but it does the job :) #!/bin/bash # Name: get_php_annotated_manual.sh # Task: Get all pages of the PHP Annotated Manual for local use. # By: Saied Tahaghoghi saied_listNOSPAM@altavista.net # Date: 14th May 2001 # Path to the manual Server_URL=http://www.php.net Manual_URL=$Server_URL/manual/en # Path to the images Images_URL=http://www.php.net/gifs mkdir Manual cd Manual echo ==================================== echo Getting index file echo ==================================== lynx -source -nolist -error_file=../access_errors.txt -dump http://www.php.net/manual/en | sed "s/\/gifs/gifs/g" | sed "s/\/manual/manual/g" | sed "s/\/html/html/g" > index.html echo ==================================== echo Getting top-level files echo ==================================== grep -i HREF index.html | sed "s/.\{0,\}[Hh][Rr][Ee][Ff]=\"//" | cut -f1 -d"\"" | cut -f1 -d"#" | cut -f1 -d"?" | grep -v "&" | grep -v ":" | grep -vi ftp | grep -vi http | grep -vi mailto | grep -iv window\.location | sort | uniq > tier_1_files.txt for RemoteFile in cat tier_1_files.txt do LocalFile=$RemoteFile # if the URL starts with /, remove it when writing if(test echo $RemoteFile | cut -c1 = /) then LocalFile=echo $RemoteFile | cut -c2- fi # if the URL has a directory in it, make the directory # Handles a single directory only! if(test echo $LocalFile | grep \/) then mkdir echo $LocalFile | cut -f1 -d"/" fi # if the URL starts with /, don't use the Manual_URL if(test echo $RemoteFile | cut -c1 = /) then echo Getting $Server_URL$RemoteFile lynx -source -nolist -error_file=../access_errors.txt -dump $Server_URL/$RemoteFile | sed "s/=\"\//=\"/g" > $LocalFile else echo Getting $Manual_URL/$RemoteFile lynx -source -nolist -error_file=../access_errors.txt -dump $Manual_URL/$RemoteFile | sed "s/=\"\//=\"/g" > $LocalFile fi # Wait a bit to avoid being accused of swamping the server sleep 60 done #/bin/rm tier_1_files.txt mv tier_1_files.txt .. echo ==================================== echo Getting tier_2 files echo ==================================== grep -i HREF * | cut -f2 -d"\"" | cut -f1 -d"#" | grep -v "&" | cut -f1 -d"?" | grep -v ":" | grep -vi ftp | grep -vi http | grep -vi mailto | grep -iv window\.location | sort | uniq > tier_2_files.txt for RemoteFile in cat tier_2_files.txt do LocalFile=$RemoteFile # if the URL starts with /, remove it when writing if(test echo $RemoteFile | cut -c1 = /) then LocalFile=echo $RemoteFile | cut -c2- fi # if the URL has a directory in it, make the directory # Handles a single directory only! if(test echo $LocalFile | grep \/) then mkdir echo $LocalFile | cut -f1 -d"/" fi # if the URL starts with /, don't use the Manual_URL if(test echo $RemoteFile | cut -c1 = /) then echo Getting $Server_URL$RemoteFile lynx -source -nolist -error_file=../access_errors.txt -dump $Server_URL/$RemoteFile | sed "s/=\"\//=\"/g" > $LocalFile else echo Getting $Manual_URL/$RemoteFile lynx -source -nolist -error_file=../access_errors.txt -dump $Manual_URL/$RemoteFile | sed "s/=\"\//=\"/g" > $LocalFile fi # Wait a bit to avoid being accused of swamping the server sleep 60 done #/bin/rm tier_2_files.txt mv tier_2_files.txt .. echo ==================================== echo Getting image files echo ==================================== mkdir gifs lynx -source -nolist -error_file=../access_errors.txt -dump $Images_URL > image_files.txt for RemoteFile in grep -i HREF image_files.txt | sed "s/.\\{1,\\}[Hh][Rr][Ee][Ff]=\"//g" | cut -f1 -d"\"" | grep -i gif do LocalFile=$RemoteFile echo Getting $Images_URL/$RemoteFile lynx -error_file=../access_errors.txt -dump $Images_URL/$RemoteFile > gifs/$LocalFile done for RemoteFile in grep -i HREF image_files.txt | sed "s/.\\{1,\\}[Hh][Rr][Ee][Ff]=\"//g" | cut -f1 -d"\"" | grep -i jpg do LocalFile=$RemoteFile echo Getting $Images_URL/$RemoteFile lynx -error_file=../access_errors.txt -dump $Images_URL/$RemoteFile > gifs/$LocalFile # Wait a bit to avoid being accused of swamping the server sleep 60 done #/bin/rm image_files.txt mv image_files.txt .. #lynx -source -nolist -error_file=../access_errors.txt -dump http://www.php.net/manual/en/$1 | sed "s/=\"\//=\"/g" > $1 # See which documents were not retrieved. rgrep "<TITLE>403 Forbidden</TITLE>" * | cut -f1 -d":" | sort > ../forbidden.list