elinks-gzip-http.patch

   1 --- elinks/src/document/session.h       Wed May 15 06:40:09 2002
   2 +++ elinks_with_gzip/src/document/session.h     Mon May 13 13:55:41 2002
   3 @@ -2,6 +2,15 @@
   4
   5  #ifndef EL__DOCUMENT_SESSION_H
   6  #define EL__DOCUMENT_SESSION_H
   7 +#ifdef HAVE_CONFIG_H
   8 +#include "config.h"
   9 +#endif
  10 +#ifdef HAVE_SSL
  11 +#include <openssl/ssl.h>
  12 +#endif
  13 +#ifdef HAVE_ZLIB_H
  14 +#include <zlib.h>
  15 +#endif
  16
  17  /* We need to declare these first :/. Damn cross-dependencies. */
  18  struct session;
  19 --- elinks/src/lowlevel/sched.c Wed May 15 06:40:15 2002
  20 +++ elinks_with_gzip/src/lowlevel/sched.c       Mon May 13 21:42:33 2002
  21 @@ -5,10 +5,15 @@
  22  #include "config.h"
  23  #endif
  24
  25 +
  26  #include <string.h>
  27  #ifdef HAVE_SSL
  28  #include <openssl/ssl.h>
  29  #endif
  30 +#ifdef HAVE_ZLIB_H
  31 +#include <zlib.h>
  32 +#endif
  33 +
  34  #ifdef HAVE_UNISTD_H
  35  #include <unistd.h>
  36  #endif
  37 @@ -391,6 +396,12 @@
  38  {
  39         del_from_list(c);
  40         send_connection_info(c);
  41 +#ifdef HAVE_ZLIB_H
  42 +       if (c->z) {
  43 +               inflateEnd(c->z);
  44 +               mem_free(c->z);
  45 +       }
  46 +#endif
  47         mem_free(c->url);
  48         mem_free(c);
  49  }
  50 --- elinks/src/lowlevel/sched.h Wed May 15 06:40:15 2002
  51 +++ elinks_with_gzip/src/lowlevel/sched.h       Mon May 13 13:41:12 2002
  52 @@ -2,7 +2,15 @@
  53
  54  #ifndef EL__LOWLEVEL_SCHED_H
  55  #define EL__LOWLEVEL_SCHED_H
  56 -
  57 +#ifdef HAVE_CONFIG_H
  58 +#include "config.h"
  59 +#endif
  60 +#ifdef HAVE_SSL
  61 +#include <openssl/ssl.h>
  62 +#endif
  63 +#ifdef HAVE_ZLIB_H
  64 +#include <zlib.h>
  65 +#endif
  66  #include "links.h" /* tcount, list_head */
  67  #include "document/cache.h"
  68  #include "lowlevel/ttime.h"
  69 @@ -74,6 +82,10 @@
  70         SSL *ssl;
  71         int no_tsl;
  72  #endif
  73 +#ifdef HAVE_ZLIB_H
  74 +       int gzip;
  75 +       z_streamp z;
  76 +#endif
  77  };
  78
  79  #define S_WAIT         0
  80 --- elinks/src/protocol/http/http.c     Wed May 15 06:40:22 2002
  81 +++ elinks_with_gzip/src/protocol/http/http.c   Wed May 15 06:32:28 2002
  82 @@ -8,6 +8,9 @@
  83  #ifdef HAVE_SSL
  84  #include <openssl/ssl.h>
  85  #endif
  86 +#ifdef HAVE_ZLIB_H
  87 +#include <zlib.h>
  88 +#endif
  89  #include <stdlib.h>
  90  #include <string.h>
  91
  92 @@ -29,6 +32,7 @@
  93  #include "protocol/url.h"
  94  #include "util/base64.h"
  95  #include "util/blacklist.h"
  96 +#include "util/compress.h"
  97
  98  struct http_connection_info {
  99         enum blacklist_flags bl_flags;
 100 @@ -121,6 +125,7 @@
 101  #endif
 102                 }
 103         }
 104 +
 105         if (c->info && !((struct http_connection_info *)c->info)->close
 106  #ifdef HAVE_SSL
 107         && (!c->ssl) /* We won't keep alive ssl connections */
 108 @@ -401,7 +406,9 @@
 109         }
 110
 111         add_to_str(&hdr, &l, "Accept: */*\r\n");
 112 -
 113 +#ifdef HAVE_ZLIB_H
 114 +       add_to_str(&hdr, &l, "Accept-Encoding: gzip\r\n");
 115 +#endif
 116         if (!accept_charset) {
 117                 unsigned char *cs, *ac;
 118                 int aclen = 0;
 119 @@ -558,9 +565,27 @@
 120                 int l = rb->len;
 121                 if (info->length >= 0 && info->length < l) l = info->length;
 122                 c->received += l;
 123 -               if (add_fragment(c->cache, c->from, rb->data, l) == 1) c->tries = 0;
 124 +               if (l) {
 125 +#ifdef HAVE_ZLIB_H
 126 +                   if (c->gzip) {
 127 +                           int dl;
 128 +                           unsigned char *data = decompress_gzip(&c->z, rb->data, l, &dl);
 129 +                           if (!data) {
 130 +                                   setcstate(c, S_OUT_OF_MEM);
 131 +                                   abort_connection(c);
 132 +                                   return;
 133 +                           }
 134 +                           if (add_fragment(c->cache, c->from, data, dl) == 1) c->tries = 0;
 135 +                           mem_free(data);
 136 +                           c->from += dl;
 137 +                   } else
 138 +#endif
 139 +                   {
 140 +                       if (add_fragment(c->cache, c->from, rb->data, l) == 1) c->tries = 0;
 141 +                       c->from += l;
 142 +                   }
 143 +               }
 144                 if (info->length >= 0) info->length -= l;
 145 -               c->from += l;
 146                 kill_buffer_data(rb, l);
 147                 if (!info->length && !rb->close) {
 148                         setcstate(c, S_OK);
 149 @@ -604,9 +629,25 @@
 150                         int l = info->chunk_remaining;
 151                         if (l > rb->len) l = rb->len;
 152                         c->received += l;
 153 -                       if (add_fragment(c->cache, c->from, rb->data, l) == 1) c->tries = 0;
 154                         info->chunk_remaining -= l;
 155 -                       c->from += l;
 156 +#ifdef HAVE_ZLIB_H
 157 +                       if (c->gzip) {
 158 +                               int dl;
 159 +                               unsigned char *data = decompress_gzip(&c->z, rb->data, l, &dl);
 160 +                               if (!data) {
 161 +                                       setcstate(c, S_OUT_OF_MEM);
 162 +                                       abort_connection(c);
 163 +                                       return;
 164 +                               }
 165 +                               if (add_fragment(c->cache, c->from, data, dl) == 1) c->tries = 0;
 166 +                               mem_free(data);
 167 +                               c->from += dl;
 168 +                       } else
 169 +#endif
 170 +                       {
 171 +                               if (add_fragment(c->cache, c->from, rb->data, l) == 1) c->tries = 0;
 172 +                               c->from += l;
 173 +                       }
 174                         kill_buffer_data(rb, l);
 175                         if (!info->chunk_remaining && rb->len >= 1) {
 176                                 if (rb->data[0] == 10) kill_buffer_data(rb, 1);
 177 @@ -846,6 +887,14 @@
 178         if (!e->last_modified && (d = parse_http_header(e->head, "Date", NULL)))
 179                 e->last_modified = d;
 180         if (info->length == -1 || (version < 11 && info->close)) rb->close = 1;
 181 +#ifdef HAVE_ZLIB_H
 182 +       d = parse_http_header(e->head, "Content-Encoding", NULL);
 183 +       c->gzip = 0;
 184 +       if (d) {
 185 +               if (!strcasecmp(d, "gzip") || !strcasecmp(d, "x-gzip")) c->gzip = 1;
 186 +               mem_free(d);
 187 +       }
 188 +#endif
 189         read_http_data(c, rb);
 190  }
 191
 192 --- elinks/src/util/compress.c  Wed May 15 06:40:24 2002
 193 +++ elinks_with_gzip/src/util/compress.c        Wed May 15 06:37:29 2002
 194 @@ -23,7 +23,6 @@
 195  #include "util/compress.h"
 196
 197
 198 -#if 0
 199  static void *
 200  z_mem_alloc(void *opaque, int items, int size)
 201  {
 202 @@ -35,7 +34,6 @@
 203  {
 204         mem_free(address);
 205  }
 206 -#endif
 207
 208
 209  struct decoding_handlers {
 210 @@ -111,74 +109,162 @@
 211         gzclose((gzFile *) stream->data);
 212  }
 213
 214 -#if 0
 215 -static unsigned char *
 216 -decompress_gzip(unsigned char *stream, int cur_size, int *new_size)
 217 -{
 218 -       z_stream z;
 219 -       char *stream_pos = stream;
 220 -       char method, flags;
 221 -       char *output;
 222 -       int size;
 223 -       int ret;
 224 +struct decoding_handlers gzip_handlers = {
 225 +       gzip_open,
 226 +       gzip_read,
 227 +       gzip_close,
 228 +};
 229
 230 -       output = mem_alloc(cur_size * 4);
 231 -       if (!output) return stream;
 232 +#define WMAXBITS 15
 233 +#define ASCII_FLAG 0x01
 234 +#define HEAD_CRC 0x02
 235 +#define EXTRA_FIELD 0x04
 236 +#define ORIG_NAME 0x08
 237 +#define COMMENT 0x10
 238 +#define RESERVED 0xE0
 239
 240 -       z.opaque = NULL;
 241 -       z.zalloc = (alloc_func) z_mem_alloc;
 242 -       z.zfree = z_mem_free;
 243 -       z.next_in = stream_pos;
 244 -       z.next_out = output;
 245 -       z.avail_out = size = cur_size * 4;
 246 -       z.avail_in = cur_size + 1;
 247 +static z_streamp gzip_init(unsigned char *buf_old, int l) {
 248 +
 249 +/* check header */
 250 +/* gzip magic */
 251 +       unsigned char method;
 252 +       unsigned char flags;
 253 +       unsigned char *buf = buf_old;
 254 +       int len;
 255 +       int ret;
 256 +       z_streamp z;
 257 +
 258 +       if (buf[0] != 0x1f || buf[1] != 0x8b) return NULL;
 259 +
 260 +       method = buf[2];
 261 +       flags = buf[3];
 262 +
 263 +       if (method != Z_DEFLATED || (flags & RESERVED) != 0) return NULL;
 264
 265 -       /* XXX: Why -15? --pasky */
 266 -       ret = inflateInit2(&z, -15);
 267 +/* Comments are borrowed from gzio.c - zlib */
 268 +/* Discard time, xflags and OS code: */
 269 +       buf += 10;
 270 +       l -= 10;
 271 +
 272 +       if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */
 273 +               len  =  2 + buf[0] + (buf[1] << 8);
 274 +               buf += len;
 275 +               l -= len;
 276 +       }
 277 +       if (l <= 0) return NULL;
 278 +
 279 +       if ((flags & ORIG_NAME) != 0) {/* skip the original file name */
 280 +               len = strlen(buf) + 1;
 281 +               buf += len;
 282 +               l -= len;
 283 +       }
 284 +       if (l <= 0) return NULL;
 285 +
 286 +       if ((flags & COMMENT) != 0) {/* skip the .gz file comment */
 287 +               len = strlen(buf) + 1;
 288 +               buf += len;
 289 +               l -= len;
 290 +       }
 291 +       if (l <= 0) return NULL;
 292
 293 -       while (ret == Z_OK) {
 294 -               char *output_new;
 295 +       if ((flags & HEAD_CRC) != 0) {  /* skip the header crc */
 296 +               buf += 2;
 297 +               l -= 2;
 298 +       }
 299 +       if (l <= 0) return NULL;
 300
 301 -               ret = inflate(&z, Z_SYNC_FLUSH);
 302 +/* initialization of z_stream */
 303 +       z = (z_streamp)mem_alloc(sizeof(z_stream));
 304 +       if (!z) return NULL;
 305 +
 306 +       z->opaque = NULL;
 307 +       z->zalloc = (alloc_func)z_mem_alloc;
 308 +       z->zfree = (free_func)z_mem_free;
 309 +       z->avail_in = l;
 310 +       z->next_in = buf;
 311 +/* windowBits is passed < 0 to tell that there is no zlib header.
 312 + * Note that in this case inflate *requires* an extra "dummy" byte
 313 + * after the compressed stream in order to complete decompression and
 314 + * return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are
 315 + * present after the compressed stream.
 316 + */
 317 +       ret = inflateInit2(z, -WMAXBITS);
 318 +       if (ret == Z_OK) return z;
 319 +
 320 +       mem_free(z);
 321 +       return NULL;
 322 +}
 323
 324 -               if (ret == Z_STREAM_END) {
 325 -                       mem_free(stream);
 326 -                       *new_size = (int) z.total_out;
 327 -                       output = mem_realloc(output, z.total_out);
 328 -                       inflateEnd(&z);
 329 -                       return output;
 330 -               }
 331 +#define OUTPUT_BUFFER_SIZE 65536
 332
 333 -               if (ret != Z_OK) {
 334 -                       inflateEnd(&z);
 335 -                       break;
 336 +unsigned char *decompress_gzip(z_streamp *z, unsigned char *buf, int l, int *dl)
 337 +{
 338 +       unsigned char *output;
 339 +       int cur_size;
 340 +       int new_size;
 341 +       int ret;
 342 +
 343 +       if (!*z) {
 344 +               *z = gzip_init(buf, l);
 345 +               if (!*z) {
 346 +                       *dl = -1;
 347 +                       return NULL;
 348                 }
 349 +       }
 350 +       else {
 351 +               (*z)->next_in = buf;
 352 +               (*z)->avail_in = l;
 353 +       }
 354
 355 -               size += cur_size * 4;
 356 -
 357 -               output_new = mem_realloc(output, size);
 358 -               if (!output_new) {
 359 -                       mem_free(output);
 360 -                       inflateEnd(&z);
 361 -                       return stream;
 362 +       (*z)->total_out = 0L;
 363 +       cur_size = OUTPUT_BUFFER_SIZE;
 364 +       output = mem_alloc(cur_size); /* output will be freed in http_read_data */
 365 +       if (!output) {
 366 +               inflateEnd(*z);
 367 +               mem_free(*z);
 368 +               *z = NULL;
 369 +               *dl = -2;
 370 +               return NULL;
 371 +       }
 372 +
 373 +       (*z)->next_out = output;
 374 +       (*z)->avail_out = 65536;
 375 +
 376 +       ret = inflate(*z, Z_SYNC_FLUSH);
 377 +       while (ret == Z_OK) {
 378 +               if (!(*z)->avail_in) {
 379 +                       *dl = (int)(*z)->total_out;
 380 +                       return output;
 381                 }
 382 -
 383 -               output = output_new;
 384 -               z.avail_out += cur_size * 4;
 385 -               z.next_out = output + z.total_out;
 386 +
 387 +               new_size = cur_size + OUTPUT_BUFFER_SIZE;
 388 +               output = mem_realloc(output, new_size);
 389 +               if (!output) {
 390 +                       inflateEnd(*z);
 391 +                       mem_free(*z);
 392 +                       *z = NULL;
 393 +                       *dl = -3;
 394 +                       return NULL;
 395 +               }
 396 +
 397 +               (*z)->next_out = output + cur_size; /* assume that z->avail_out == 0 */
 398 +               (*z)->avail_out = OUTPUT_BUFFER_SIZE;
 399 +               cur_size = new_size;
 400 +               ret = inflate(*z, Z_SYNC_FLUSH);
 401         }
 402
 403 -       mem_free(output);
 404 +       if (ret == Z_STREAM_END) *dl = (int)(*z)->total_out;
 405 +       else { /* something went wrong */
 406 +               *dl = -4;
 407 +               mem_free(output);
 408 +               output = NULL;
 409 +       }
 410
 411 -       return stream;
 412 +       inflateEnd(*z);
 413 +       mem_free(*z);
 414 +       *z = NULL;
 415 +       return output;
 416  }
 417 -#endif
 418 -
 419 -struct decoding_handlers gzip_handlers = {
 420 -       gzip_open,
 421 -       gzip_read,
 422 -       gzip_close,
 423 -};
 424
 425  #endif
 426
 427 --- elinks/src/util/compress.h  Wed May 15 06:40:24 2002
 428 +++ elinks_with_gzip/src/util/compress.h        Wed May 15 01:35:11 2002
 429 @@ -3,6 +3,13 @@
 430  #ifndef EL__UTIL_COMPRESS_H
 431  #define EL__UTIL_COMPRESS_H
 432
 433 +#ifdef HAVE_CONFIG_H
 434 +#include "config.h"
 435 +#endif
 436 +#ifdef HAVE_ZLIB_H
 437 +#include <zlib.h>
 438 +#endif
 439 +
 440  enum stream_encoding {
 441         ENCODING_NONE,
 442         ENCODING_GZIP,
 443 @@ -17,5 +24,8 @@
 444  struct stream_encoded *open_encoded(int, enum stream_encoding);
 445  int read_encoded(struct stream_encoded *, unsigned char *, int);
 446  void close_encoded(struct stream_encoded *);
 447 +#ifdef HAVE_ZLIB_H
 448 +unsigned char *decompress_gzip(z_streamp *, unsigned char *, int, int *);
 449 +#endif
 450
 451  #endif
 452