Improved transparent compressor

For the consideration of someone who can add code to php4... Following is a patch to php 4.0.4 to add transparent gzip and deflate content encoding to php4. The major advancement in this mechanism over ob_gzhandler is that it compresses the output as it is generated rather than waiting until the entire otuput is finished then compressing the whole file. This allows better overlapping of page generation and page transmission and also avoids the O(N^2) time penalty incurred when buffering very large pages. (We routinely generate output hundreds of kilobytes long the difference is dramatic in this range.) The patch adds several pieces to php4... - a general `body_encoder' concept is added to the output functions in the form of three function pointers to handle writing, flushing, and finalizing output. - calls to these body_encoder functions are added at appropriate points in the code to support flush() and finalizing the stream at exit. - Two functions, encode_gzip() and encode_deflate() are added which turn on content encoding. These must be called before any output is generated as they add a header line. - it should include manual entries, but I forgot that. All logic about when to enable content encoding is left to the calling scripts. This is a complicated mess because of browsers that overstate their abilities. (As noted in a thread here back in Nov.) The decision should also depend on the content type which is to be sent, e.g. it makes no sense to compress jpeg images. They don't get smaller, your cpu just gets hotter. Content encoding is a life saver when your business takes off, you saturate your pipes, and can't get new ones installed for 30 days. :-) Even after that, I think most people find that the CPU dollar cost of compressing is negligible compared to the network costs saved. We use very conservative settings in deciding what to compress, just known browsers that we know can handle it. Fortunately thats the bulk of the requests. Please copy me on replies, (especially if you takes this up to include it in the main source), I am not subscribed to the list. -- Jim Studt, President The Federated Software Group, Inc. --- ./ext/standard/basic_functions.c Wed Jan 10 16:27:28 2001 +++ ./ext/standard/basic_functions.c Wed Dec 13 17:02:09 2000 @@ -1104,7 +1104,6 @@ Flush the output buffer */ PHP_FUNCTION(flush) { - if ( OG(php_body_encode_flush)) OG(php_body_encode_flush)(OG(php_header_write)); sapi_flush(); } /* }}} */ --- ./ext/standard/output.c Wed Jan 10 15:44:44 2001 +++ ./ext/standard/output.c Thu Nov 23 12:43:18 2000 @@ -359,11 +359,7 @@ str_length = new_length; } - if ( OG(php_body_encode)) { - result = OG(php_body_encode)(str, str_length, OG(implicit_flush)?1:0, OG(php_header_write)); - } else { - result = OG(php_header_write)(str, str_length); - } + result = OG(php_header_write)(str, str_length); if (newstr) { free(newstr); --- ./ext/standard/php_output.h Wed Jan 10 15:31:28 2001 +++ ./ext/standard/php_output.h Thu Nov 23 12:43:18 2000 @@ -57,10 +57,6 @@ typedef struct _php_output_globals { int (*php_body_write)(const char *str, uint str_length); /* string output */ int (*php_header_write)(const char *str, uint str_length); /* unbuffer string output */ - int (*php_body_encode)(const char *str, uint str_length, int flush_flag, - int (*output_func)(const char *str, uint str_length)); - int (*php_body_encode_trailer)(int (*output_func)(const char *str, uint str_length)); - int (*php_body_encode_flush)(int (*output_func)(const char *str, uint str_length)); php_ob_buffer active_ob_buffer; unsigned char implicit_flush; char *output_start_filename; --- ./ext/zlib/php_zlib.h Fri Jan 12 11:49:45 2001 +++ ./ext/zlib/php_zlib.h Mon Nov 20 18:40:12 2000 @@ -32,8 +32,6 @@ int compression_coding; z_stream stream; uLong crc; - z_stream encoder; - uLong encoder_crc; } php_zlib_globals; extern zend_module_entry php_zlib_module_entry; @@ -62,8 +60,6 @@ PHP_FUNCTION(gzinflate); PHP_FUNCTION(gzencode); PHP_FUNCTION(ob_gzhandler); -PHP_FUNCTION(encode_deflate); -PHP_FUNCTION(encode_gzip); FILE *zlib_fopen_wrapper(char *path, char *mode, int options, int *issock, int *socketd, char **opened_path); --- ./ext/zlib/zlib.c Fri Jan 12 14:10:01 2001 +++ ./ext/zlib/zlib.c Mon Nov 20 18:40:12 2000 @@ -103,8 +103,6 @@ PHP_FE(gzinflate, NULL) PHP_FE(gzencode, NULL) PHP_FE(ob_gzhandler, NULL) - PHP_FE(encode_gzip, NULL) - PHP_FE(encode_deflate, NULL) {NULL, NULL, NULL} }; @@ -1129,138 +1127,6 @@ zval_copy_ctor(return_value); } } - - -static int php_encode_deflate(const char *str, uint str_length, int flush_flag, - int (*output_func)(const char *str, uint str_length)) -{ - z_stream *e = &ZLIBG(encoder); - int err; - int ff; - - if ( e->total_in == 0) { - /* first call, initialize */ - err = deflateInit2( e, Z_DEFAULT_COMPRESSION, Z_DEFLATED, - -15, 8, Z_DEFAULT_STRATEGY); - if ( err != Z_OK) { - php_error(E_ERROR,"php_encode_deflate failed(%d)", err); - return 0; - } - ZLIBG(encoder_crc) = 0; - } - - e->next_in = (void *)str; - e->avail_in = str_length; - ZLIBG(encoder_crc) = crc32(ZLIBG(encoder_crc), str, str_length); - - ff = 0; - if ( flush_flag == 1) ff = Z_SYNC_FLUSH; - else if ( flush_flag == 2) ff = Z_FINISH; - - for (;;) { - char buffer[1024]; - - e->next_out = buffer; - e->avail_out = sizeof(buffer); - err = deflate(e, ff); - if ( err != Z_OK && err != Z_STREAM_END) { - php_error(E_ERROR,"php_encode_deflate deflat() failed(%d)", err); - return 0; - } - if ( (char *)e->next_out != buffer) output_func(buffer, (char *)e->next_out - buffer); - - if ( ff == Z_FINISH && err != Z_STREAM_END) continue; - if ( e->avail_in <= 0) break; - } - - return str_length; -} - -static int php_encode_deflate_trailer(int (*output_func)(const char *str, uint str_length)) -{ - php_encode_deflate("",0,2,output_func); - return 1; -} - -static int php_encode_deflate_flush(int (*output_func)(const char *str, uint str_length)) -{ - php_encode_deflate("",0,1,output_func); - return 1; -} - - - - -static int php_encode_gzip(const char *str, uint str_length, int flush_flag, - int (*output_func)(const char *str, uint str_length)) -{ - z_stream *e = &ZLIBG(encoder); - - static const char header[] = "\037\213\010\0\0\0\0\0\0\3"; - - if ( e->total_in == 0) { - /* first call, inject header */ - output_func(header, sizeof(header)-1); - } - return php_encode_deflate( str, str_length, flush_flag, output_func); -} - -static int php_encode_gzip_trailer(int (*output_func)(const char *str, uint str_length)) -{ - char buf[8]; - char *b = buf; - - php_encode_gzip("",0,2,output_func); - - *b++ = (char) ZLIBG(encoder_crc) & 0xFF; - *b++ = (char) (ZLIBG(encoder_crc) >> 8) & 0xFF; - *b++ = (char) (ZLIBG(encoder_crc) >> 16) & 0xFF; - *b++ = (char) (ZLIBG(encoder_crc) >> 24) & 0xFF; - *b++ = (char) ZLIBG(encoder).total_in & 0xFF; - *b++ = (char) (ZLIBG(encoder).total_in >> 8) & 0xFF; - *b++ = (char) (ZLIBG(encoder).total_in >> 16) & 0xFF; - *b++ = (char) (ZLIBG(encoder).total_in >> 24) & 0xFF; - - output_func(buf, 8); - - return 1; -} - -static int php_encode_gzip_flush(int (*output_func)(const char *str, uint str_length)) -{ - php_encode_gzip("",0,1,output_func); - return 1; -} - - -PHP_FUNCTION(encode_deflate) -{ - static const char ce[] = "Content-Encoding: deflate"; - if (sapi_add_header(ce, sizeof(ce) - 1, 1)==FAILURE) { - RETURN_LONG(0); - } - OG(php_body_encode) = php_encode_deflate; - OG(php_body_encode_trailer) = php_encode_deflate_trailer; - OG(php_body_encode_flush) = php_encode_deflate_flush; - memset(&ZLIBG(encoder),0,sizeof(ZLIBG(encoder))); - - RETURN_LONG(1); -} - -PHP_FUNCTION(encode_gzip) -{ - static const char ce[] = "Content-Encoding: gzip"; - if (sapi_add_header(ce, sizeof(ce) - 1, 1)==FAILURE) { - RETURN_LONG(0); - } - OG(php_body_encode) = php_encode_gzip; - OG(php_body_encode_trailer) = php_encode_gzip_trailer; - OG(php_body_encode_flush) = php_encode_gzip_flush; - memset(&ZLIBG(encoder),0,sizeof(ZLIBG(encoder))); - - RETURN_LONG(1); -} - /* --- ./main/main.c Fri Jan 12 14:26:20 2001 +++ ./main/main.c Tue Nov 28 19:02:27 2000 @@ -1224,10 +1224,6 @@ V_CHDIR(old_cwd); free_alloca(old_cwd); - if ( OG(php_body_encode_trailer)) { - OG(php_body_encode_trailer)(OG(php_header_write)); - } - return EG(exit_status); }