note 67735 added to function.zip-open

I had some trouble reading a bunch of Zip-Files in a loop, each containing about 10,000 small files - The original php_zip extension memory consumption was huge and the replacement hack way to slow (it makes one call to "unzip -p" per entry of the zip file). Here is a revised version of the unzip hack. It streams the whole zip content at once (uncompressing on the fly). The interface is exactly the same, only some "byte-arithmetic" within the functions was needed to offset to the proper position of the stream. Compared to the original hack, my own script finishes now after 6 minutes instead of 4.5 hours. <?php function ShellFix($s) { return '"'.str_replace("'", "'\''", $s).'"'; } function zip_open($s) { $fp = @fopen($s, 'rb'); if(!$fp) return false; $lines = Array(); $cmd = 'unzip -v '.ShellFix($s); exec($cmd, $lines); $contents = Array(); $startpos = 0; $ok=false; foreach($lines as $line) { if($line[2]=='-') { $ok=!$ok; continue; } if(!$ok) continue; $length = (int)$line; $fn = trim(substr($line,58)); array_push($contents, Array('name' => $fn, 'length' => $length, 'startpos' => $startpos)); $startpos += $length; // startpos of next file; } //// we already open the pipe for later reading ALL files at once... $cmd = 'unzip -p '.ShellFix($s); $stream = popen($cmd, 'rb'); //////////////////////////////////// return Array('fp' => $fp, 'name' => $s, 'contents' => $contents, 'pointer' => -1, 'stream' => $stream ); } function zip_read(&$fp) { if(!$fp) return false; $next = $fp['pointer'] + 1; if($next >= count($fp['contents'])) return false; // add the length from the (now) previous file to current position. if ($fp['pointer'] >= 0) { $fp['startpos'] += $fp['contents'][$fp['pointer']]['length']; } $fp['pointer'] = $next; return $fp['contents'][$next]; // == $res } function zip_entry_name(&$res) { if(!$res) return false; return $res['name']; } function zip_entry_filesize(&$res) { if(!$res) return false; return $res['length']; } function zip_entry_open(&$fp, &$res) { if(!$res) return false; // now more or less a dummy, stream should be already open... $res['fp'] = $fp['stream']; return !!$fp['stream']; } function zip_entry_read(&$res, $nbytes) { $curpos = ftell($res['fp']); // current bytepos in (unpacked)zip stream... $startpos = $res['startpos']; $length = $res['length']; // length of current file $seekcnt = 0; if ($curpos < $startpos) { $seekcnt = $startpos - $curpos ; fseek($res['fp'],$seekcnt,SEEK_CUR); $curpos = $startpos; } // determine how much is left for reading from current file: $remain = $length - ($curpos - $startpos ); // do we want to read less? if ($nbytes < $remain) {$remain = $nbytes; } while (($remain > 0 ) && ($s = fread($res['fp'],(($remain > 1024)?1024:$remain) ))) { $data .= $s; $remain -= strlen($s) ; } return $data; } function zip_entry_close(&$res) { // do nothing ... (Todo: Some cleanup needed?) } function zip_close(&$fp) { fclose($fp['stream']); unset($fp['stream']); fclose($fp['fp']); unset($fp['fp']); } ?> ---- Server IP: 66.163.161.117 Probable Submitter: 198.240.213.25 ---- X-Spam-Status: No, hits=3.1 required=5.0 tests=DATE_MISSING,FROM_NO_LOWER autolearn=no version=2.64 ---- Manual Page -- http://www.php.net/manual/en/function.zip-open.php Edit -- http://master.php.net/note/edit/67735 Del: integrated -- http://master.php.net/note/delete/67735/integrated Del: useless -- http://master.php.net/note/delete/67735/useless Del: bad code -- http://master.php.net/note/delete/67735/bad+code Del: spam -- http://master.php.net/note/delete/67735/spam Del: non-english -- http://master.php.net/note/delete/67735/non-english Del: in docs -- http://master.php.net/note/delete/67735/in+docs Del: other reasons-- http://master.php.net/note/delete/67735 Reject -- http://master.php.net/note/reject/67735 Search -- http://master.php.net/manage/user-notes.php