--- elinks/src/document/session.h Wed May 15 06:40:09 2002 +++ elinks_with_gzip/src/document/session.h Mon May 13 13:55:41 2002 @@ -2,6 +2,15 @@ #ifndef EL__DOCUMENT_SESSION_H #define EL__DOCUMENT_SESSION_H +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#ifdef HAVE_SSL +#include +#endif +#ifdef HAVE_ZLIB_H +#include +#endif /* We need to declare these first :/. Damn cross-dependencies. */ struct session; --- elinks/src/lowlevel/sched.c Wed May 15 06:40:15 2002 +++ elinks_with_gzip/src/lowlevel/sched.c Mon May 13 21:42:33 2002 @@ -5,10 +5,15 @@ #include "config.h" #endif + #include #ifdef HAVE_SSL #include #endif +#ifdef HAVE_ZLIB_H +#include +#endif + #ifdef HAVE_UNISTD_H #include #endif @@ -391,6 +396,12 @@ { del_from_list(c); send_connection_info(c); +#ifdef HAVE_ZLIB_H + if (c->z) { + inflateEnd(c->z); + mem_free(c->z); + } +#endif mem_free(c->url); mem_free(c); } --- elinks/src/lowlevel/sched.h Wed May 15 06:40:15 2002 +++ elinks_with_gzip/src/lowlevel/sched.h Mon May 13 13:41:12 2002 @@ -2,7 +2,15 @@ #ifndef EL__LOWLEVEL_SCHED_H #define EL__LOWLEVEL_SCHED_H - +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#ifdef HAVE_SSL +#include +#endif +#ifdef HAVE_ZLIB_H +#include +#endif #include "links.h" /* tcount, list_head */ #include "document/cache.h" #include "lowlevel/ttime.h" @@ -74,6 +82,10 @@ SSL *ssl; int no_tsl; #endif +#ifdef HAVE_ZLIB_H + int gzip; + z_streamp z; +#endif }; #define S_WAIT 0 --- elinks/src/protocol/http/http.c Wed May 15 06:40:22 2002 +++ elinks_with_gzip/src/protocol/http/http.c Wed May 15 06:32:28 2002 @@ -8,6 +8,9 @@ #ifdef HAVE_SSL #include #endif +#ifdef HAVE_ZLIB_H +#include +#endif #include #include @@ -29,6 +32,7 @@ #include "protocol/url.h" #include "util/base64.h" #include "util/blacklist.h" +#include "util/compress.h" struct http_connection_info { enum blacklist_flags bl_flags; @@ -121,6 +125,7 @@ #endif } } + if (c->info && !((struct http_connection_info *)c->info)->close #ifdef HAVE_SSL && (!c->ssl) /* We won't keep alive ssl connections */ @@ -401,7 +406,9 @@ } add_to_str(&hdr, &l, "Accept: */*\r\n"); - +#ifdef HAVE_ZLIB_H + add_to_str(&hdr, &l, "Accept-Encoding: gzip\r\n"); +#endif if (!accept_charset) { unsigned char *cs, *ac; int aclen = 0; @@ -558,9 +565,27 @@ int l = rb->len; if (info->length >= 0 && info->length < l) l = info->length; c->received += l; - if (add_fragment(c->cache, c->from, rb->data, l) == 1) c->tries = 0; + if (l) { +#ifdef HAVE_ZLIB_H + if (c->gzip) { + int dl; + unsigned char *data = decompress_gzip(&c->z, rb->data, l, &dl); + if (!data) { + setcstate(c, S_OUT_OF_MEM); + abort_connection(c); + return; + } + if (add_fragment(c->cache, c->from, data, dl) == 1) c->tries = 0; + mem_free(data); + c->from += dl; + } else +#endif + { + if (add_fragment(c->cache, c->from, rb->data, l) == 1) c->tries = 0; + c->from += l; + } + } if (info->length >= 0) info->length -= l; - c->from += l; kill_buffer_data(rb, l); if (!info->length && !rb->close) { setcstate(c, S_OK); @@ -604,9 +629,25 @@ int l = info->chunk_remaining; if (l > rb->len) l = rb->len; c->received += l; - if (add_fragment(c->cache, c->from, rb->data, l) == 1) c->tries = 0; info->chunk_remaining -= l; - c->from += l; +#ifdef HAVE_ZLIB_H + if (c->gzip) { + int dl; + unsigned char *data = decompress_gzip(&c->z, rb->data, l, &dl); + if (!data) { + setcstate(c, S_OUT_OF_MEM); + abort_connection(c); + return; + } + if (add_fragment(c->cache, c->from, data, dl) == 1) c->tries = 0; + mem_free(data); + c->from += dl; + } else +#endif + { + if (add_fragment(c->cache, c->from, rb->data, l) == 1) c->tries = 0; + c->from += l; + } kill_buffer_data(rb, l); if (!info->chunk_remaining && rb->len >= 1) { if (rb->data[0] == 10) kill_buffer_data(rb, 1); @@ -846,6 +887,20 @@ if (!e->last_modified && (d = parse_http_header(e->head, "Date", NULL))) e->last_modified = d; if (info->length == -1 || (version < 11 && info->close)) rb->close = 1; +#ifdef HAVE_ZLIB_H + d = parse_http_header(e->head, "Content-Encoding", NULL); + c->gzip = 0; + if (d) { + if (!strcasecmp(d, "gzip") || !strcasecmp(d, "x-gzip")) { + mem_free(d); + d = parse_http_header(e->head, "Content-Type", NULL); + if (d) { + if (!strncasecmp(d, "text", 4)) c->gzip = 1; + mem_free(d); + } + } + } +#endif read_http_data(c, rb); } --- elinks/src/util/compress.c Wed May 15 06:40:24 2002 +++ elinks_with_gzip/src/util/compress.c Wed May 15 06:37:29 2002 @@ -23,7 +23,6 @@ #include "util/compress.h" -#if 0 static void * z_mem_alloc(void *opaque, int items, int size) { @@ -35,7 +34,6 @@ { mem_free(address); } -#endif struct decoding_handlers { @@ -111,74 +109,162 @@ gzclose((gzFile *) stream->data); } -#if 0 -static unsigned char * -decompress_gzip(unsigned char *stream, int cur_size, int *new_size) -{ - z_stream z; - char *stream_pos = stream; - char method, flags; - char *output; - int size; - int ret; +struct decoding_handlers gzip_handlers = { + gzip_open, + gzip_read, + gzip_close, +}; - output = mem_alloc(cur_size * 4); - if (!output) return stream; +#define WMAXBITS 15 +#define ASCII_FLAG 0x01 +#define HEAD_CRC 0x02 +#define EXTRA_FIELD 0x04 +#define ORIG_NAME 0x08 +#define COMMENT 0x10 +#define RESERVED 0xE0 - z.opaque = NULL; - z.zalloc = (alloc_func) z_mem_alloc; - z.zfree = z_mem_free; - z.next_in = stream_pos; - z.next_out = output; - z.avail_out = size = cur_size * 4; - z.avail_in = cur_size + 1; +static z_streamp gzip_init(unsigned char *buf_old, int l) { + +/* check header */ +/* gzip magic */ + unsigned char method; + unsigned char flags; + unsigned char *buf = buf_old; + int len; + int ret; + z_streamp z; + + if (buf[0] != 0x1f || buf[1] != 0x8b) return NULL; + + method = buf[2]; + flags = buf[3]; + + if (method != Z_DEFLATED || (flags & RESERVED) != 0) return NULL; - /* XXX: Why -15? --pasky */ - ret = inflateInit2(&z, -15); +/* Comments are borrowed from gzio.c - zlib */ +/* Discard time, xflags and OS code: */ + buf += 10; + l -= 10; + + if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */ + len = 2 + buf[0] + (buf[1] << 8); + buf += len; + l -= len; + } + if (l <= 0) return NULL; + + if ((flags & ORIG_NAME) != 0) {/* skip the original file name */ + len = strlen(buf) + 1; + buf += len; + l -= len; + } + if (l <= 0) return NULL; + + if ((flags & COMMENT) != 0) {/* skip the .gz file comment */ + len = strlen(buf) + 1; + buf += len; + l -= len; + } + if (l <= 0) return NULL; - while (ret == Z_OK) { - char *output_new; + if ((flags & HEAD_CRC) != 0) { /* skip the header crc */ + buf += 2; + l -= 2; + } + if (l <= 0) return NULL; - ret = inflate(&z, Z_SYNC_FLUSH); +/* initialization of z_stream */ + z = (z_streamp)mem_alloc(sizeof(z_stream)); + if (!z) return NULL; + + z->opaque = NULL; + z->zalloc = (alloc_func)z_mem_alloc; + z->zfree = (free_func)z_mem_free; + z->avail_in = l; + z->next_in = buf; +/* windowBits is passed < 0 to tell that there is no zlib header. + * Note that in this case inflate *requires* an extra "dummy" byte + * after the compressed stream in order to complete decompression and + * return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are + * present after the compressed stream. + */ + ret = inflateInit2(z, -WMAXBITS); + if (ret == Z_OK) return z; + + mem_free(z); + return NULL; +} - if (ret == Z_STREAM_END) { - mem_free(stream); - *new_size = (int) z.total_out; - output = mem_realloc(output, z.total_out); - inflateEnd(&z); - return output; - } +#define OUTPUT_BUFFER_SIZE 65536 - if (ret != Z_OK) { - inflateEnd(&z); - break; +unsigned char *decompress_gzip(z_streamp *z, unsigned char *buf, int l, int *dl) +{ + unsigned char *output; + int cur_size; + int new_size; + int ret; + + if (!*z) { + *z = gzip_init(buf, l); + if (!*z) { + *dl = -1; + return NULL; } + } + else { + (*z)->next_in = buf; + (*z)->avail_in = l; + } - size += cur_size * 4; - - output_new = mem_realloc(output, size); - if (!output_new) { - mem_free(output); - inflateEnd(&z); - return stream; + (*z)->total_out = 0L; + cur_size = OUTPUT_BUFFER_SIZE; + output = mem_alloc(cur_size); /* output will be freed in http_read_data */ + if (!output) { + inflateEnd(*z); + mem_free(*z); + *z = NULL; + *dl = -2; + return NULL; + } + + (*z)->next_out = output; + (*z)->avail_out = 65536; + + ret = inflate(*z, Z_SYNC_FLUSH); + while (ret == Z_OK) { + if (!(*z)->avail_in) { + *dl = (int)(*z)->total_out; + return output; } - - output = output_new; - z.avail_out += cur_size * 4; - z.next_out = output + z.total_out; + + new_size = cur_size + OUTPUT_BUFFER_SIZE; + output = mem_realloc(output, new_size); + if (!output) { + inflateEnd(*z); + mem_free(*z); + *z = NULL; + *dl = -3; + return NULL; + } + + (*z)->next_out = output + cur_size; /* assume that z->avail_out == 0 */ + (*z)->avail_out = OUTPUT_BUFFER_SIZE; + cur_size = new_size; + ret = inflate(*z, Z_SYNC_FLUSH); } - mem_free(output); + if (ret == Z_STREAM_END) *dl = (int)(*z)->total_out; + else { /* something went wrong */ + *dl = -4; + mem_free(output); + output = NULL; + } - return stream; + inflateEnd(*z); + mem_free(*z); + *z = NULL; + return output; } -#endif - -struct decoding_handlers gzip_handlers = { - gzip_open, - gzip_read, - gzip_close, -}; #endif --- elinks/src/util/compress.h Wed May 15 06:40:24 2002 +++ elinks_with_gzip/src/util/compress.h Wed May 15 01:35:11 2002 @@ -3,6 +3,13 @@ #ifndef EL__UTIL_COMPRESS_H #define EL__UTIL_COMPRESS_H +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#ifdef HAVE_ZLIB_H +#include +#endif + enum stream_encoding { ENCODING_NONE, ENCODING_GZIP, @@ -17,5 +24,8 @@ struct stream_encoded *open_encoded(int, enum stream_encoding); int read_encoded(struct stream_encoded *, unsigned char *, int); void close_encoded(struct stream_encoded *); +#ifdef HAVE_ZLIB_H +unsigned char *decompress_gzip(z_streamp *, unsigned char *, int, int *); +#endif #endif