3
Transforms content using gzip
5
@section license License
7
Licensed to the Apache Software Foundation (ASF) under one
8
or more contributor license agreements. See the NOTICE file
9
distributed with this work for additional information
10
regarding copyright ownership. The ASF licenses this file
11
to you under the Apache License, Version 2.0 (the
12
"License"); you may not use this file except in compliance
13
with the License. You may obtain a copy of the License at
15
http://www.apache.org/licenses/LICENSE-2.0
17
Unless required by applicable law or agreed to in writing, software
18
distributed under the License is distributed on an "AS IS" BASIS,
19
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20
See the License for the specific language governing permissions and
21
limitations under the License.
29
#include <sys/types.h>
36
#define DICT_PATH_MAX 512
37
#define DICT_ENTRY_MAX 2048
43
TSIOBuffer output_buffer;
44
TSIOBufferReader output_reader;
52
char preload_file[1024];
55
char dictionary[800000];
58
load_dictionary(char *dict, uLong * adler)
63
fp = fopen(preload_file, "r");
65
TSError("gzip-transform: ERROR: Unable to open dict file %s\n", preload_file);
69
/* dict = (char *) calloc(8000,sizeof(char)); */
73
if (fscanf(fp, "%s\n", dict + i) == 1) {
75
strcat(dict + i, " ");
81
/* TODO get the adler compute right */
82
*adler = adler32(*adler, (const Byte *) dict, sizeof(dict));
86
gzip_alloc(voidpf opaque, uInt items, uInt size)
88
return (voidpf) TSmalloc(items * size);
93
gzip_free(voidpf opaque, voidpf address)
105
data = (GzipData *) TSmalloc(sizeof(GzipData));
106
data->output_vio = NULL;
107
data->output_buffer = NULL;
108
data->output_reader = NULL;
109
data->output_length = 0;
111
data->crc = crc32(0L, Z_NULL, 0);
113
data->zstrm.next_in = Z_NULL;
114
data->zstrm.avail_in = 0;
115
data->zstrm.total_in = 0;
116
data->zstrm.next_out = Z_NULL;
117
data->zstrm.avail_out = 0;
118
data->zstrm.total_out = 0;
119
data->zstrm.zalloc = gzip_alloc;
120
data->zstrm.zfree = gzip_free;
121
data->zstrm.opaque = (voidpf) 0;
122
data->zstrm.data_type = Z_ASCII;
124
err = deflateInit(&data->zstrm, Z_BEST_COMPRESSION);
127
TSError("gzip-transform: ERROR: deflateInit (%d)!", err);
132
assert(&data->zstrm);
133
err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, strlen(dictionary));
135
TSError("gzip-transform: ERROR: deflateSetDictionary (%d)!", err);
144
gzip_data_destroy(GzipData * data)
149
err = deflateEnd(&data->zstrm);
151
TSError("gzip-transform: ERROR: deflateEnd (%d)!", err);
154
if (data->output_buffer)
155
TSIOBufferDestroy(data->output_buffer);
162
gzip_transform_init(TSCont contp, GzipData * data)
167
TSMLoc ce_loc; /* for the content encoding mime field */
172
* Mark the output data as having gzip content encoding
174
TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc);
175
TSMimeHdrFieldCreate(bufp, hdr_loc, &ce_loc); /* Probably should check for errors */
176
TSMimeHdrFieldNameSet(bufp, hdr_loc, ce_loc, "Content-Encoding", -1);
177
TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, "deflate", -1);
178
TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc);
179
TSHandleMLocRelease(bufp, hdr_loc, ce_loc);
180
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
183
/* Get the output connection where we'll write data to. */
184
output_conn = TSTransformOutputVConnGet(contp);
186
data->output_buffer = TSIOBufferCreate();
187
data->output_reader = TSIOBufferReaderAlloc(data->output_buffer);
188
data->output_vio = TSVConnWrite(output_conn, contp, data->output_reader, INT64_MAX);
193
gzip_transform_one(GzipData * data, TSIOBufferReader input_reader, int amount)
195
TSIOBufferBlock blkp;
198
int64_t ilength, olength;
202
blkp = TSIOBufferReaderStart(input_reader);
203
ibuf = TSIOBufferBlockReadStart(blkp, input_reader, &ilength);
205
if (ilength > amount) {
209
data->zstrm.next_in = (unsigned char *) ibuf;
210
data->zstrm.avail_in = ilength;
212
while (data->zstrm.avail_in > 0) {
213
blkp = TSIOBufferStart(data->output_buffer);
215
obuf = TSIOBufferBlockWriteStart(blkp, &olength);
217
data->zstrm.next_out = (unsigned char *) obuf;
218
data->zstrm.avail_out = olength;
221
err = deflate(&data->zstrm, Z_NO_FLUSH);
223
if (olength > data->zstrm.avail_out) {
224
TSIOBufferProduce(data->output_buffer, olength - data->zstrm.avail_out);
225
data->output_length += (olength - data->zstrm.avail_out);
228
if (data->zstrm.avail_out > 0) {
229
if (data->zstrm.avail_in != 0) {
230
TSError("gzip-transform: ERROR: avail_in is (%d): should be 0", data->zstrm.avail_in);
235
/* compute CRC for error checking at client */
236
data->crc = crc32(data->crc, (unsigned char *) ibuf, ilength);
238
TSIOBufferReaderConsume(input_reader, ilength);
245
gzip_transform_finish(GzipData * data)
247
if (data->state == 1) {
248
TSIOBufferBlock blkp;
256
blkp = TSIOBufferStart(data->output_buffer);
258
obuf = TSIOBufferBlockWriteStart(blkp, &olength);
259
data->zstrm.next_out = (unsigned char *) obuf;
260
data->zstrm.avail_out = olength;
262
/* Encode remaining data */
263
err = deflate(&data->zstrm, Z_FINISH);
265
if (olength > data->zstrm.avail_out) {
266
TSIOBufferProduce(data->output_buffer, olength - data->zstrm.avail_out);
267
data->output_length += (olength - data->zstrm.avail_out);
270
if (err == Z_OK) { /* some more data to encode */
274
if (err != Z_STREAM_END) {
275
TSDebug("gzip-transform", "deflate should report Z_STREAM_END\n");
280
if (data->output_length != (data->zstrm.total_out)) {
281
TSError("gzip-transform: ERROR: output lengths don't match (%d, %ld)", data->output_length,
282
data->zstrm.total_out);
285
/* compute/append crc to end of stream */
288
blkp = TSIOBufferStart (data->output_buffer);
291
buf[0] = tmp & 0xff; tmp >>= 8;
292
buf[1] = tmp & 0xff; tmp >>= 8;
293
buf[2] = tmp & 0xff; tmp >>= 8;
296
tmp = data->zstrm.total_in;
297
buf[4] = tmp & 0xff; tmp >>= 8;
298
buf[5] = tmp & 0xff; tmp >>= 8;
299
buf[6] = tmp & 0xff; tmp >>= 8;
306
obuf = TSIOBufferBlockWriteStart (blkp, &olength);
307
if (olength > length) {
311
memcpy (obuf, p, olength);
315
TSIOBufferProduce (data->output_buffer, olength);
318
data->output_length += 8;
325
gzip_transform_do(TSCont contp)
333
/* Get our data structure for this operation. The private data
334
structure contains the output vio and output buffer. If the
335
private data structure pointer is NULL, then we'll create it
336
and initialize its internals. */
337
data = TSContDataGet(contp);
338
if (data->state == 0) {
339
gzip_transform_init(contp, data);
342
/* Get the write vio for the write operation that was performed on
343
ourself. This vio contains the buffer that we are to read from
344
as well as the continuation we are to call when the buffer is
346
write_vio = TSVConnWriteVIOGet(contp);
348
length = data->output_length;
350
/* We also check to see if the write vio's buffer is non-NULL. A
351
NULL buffer indicates that the write operation has been
352
shutdown and that the continuation does not want us to send any
353
more WRITE_READY or WRITE_COMPLETE events. For this simplistic
354
transformation that means we're done. In a more complex
355
transformation we might have to finish writing the transformed
356
data to our output connection. */
357
if (!TSVIOBufferGet(write_vio)) {
358
gzip_transform_finish(data);
360
TSVIONBytesSet(data->output_vio, data->output_length);
361
TSDebug("gzip-transform", "Compressed size %d (bytes)", data->output_length);
363
if (data->output_length > length) {
364
TSVIOReenable(data->output_vio);
369
/* Determine how much data we have left to read. For this gzip
370
transform plugin this is also the amount of data we have left
371
to write to the output connection. */
372
towrite = TSVIONTodoGet(write_vio);
374
/* The amount of data left to read needs to be truncated by
375
the amount of data actually in the read buffer. */
376
avail = TSIOBufferReaderAvail(TSVIOReaderGet(write_vio));
377
if (towrite > avail) {
382
gzip_transform_one(data, TSVIOReaderGet(write_vio), towrite);
384
/* Modify the write vio to reflect how much data we've
386
TSVIONDoneSet(write_vio, TSVIONDoneGet(write_vio) + towrite);
390
/* Now we check the write vio to see if there is data left to
392
if (TSVIONTodoGet(write_vio) > 0) {
394
/* If we output some data then we reenable the output
395
connection by reenabling the output vio. This will wakeup
396
the output connection and allow it to consume data from the
398
if (data->output_length > length) {
399
TSVIOReenable(data->output_vio);
402
/* Call back the write vio continuation to let it know that we
403
are ready for more data. */
404
TSContCall(TSVIOContGet(write_vio), TS_EVENT_VCONN_WRITE_READY, write_vio);
407
/* If there is no data left to read, then we modify the output
408
vio to reflect how much data the output connection should
409
expect. This allows the output connection to know when it
410
is done reading. We then reenable the output connection so
411
that it can consume the data we just gave it. */
412
gzip_transform_finish(data);
414
TSVIONBytesSet(data->output_vio, data->output_length);
415
TSDebug("gzip-transform", "Compressed size %d (bytes)", data->output_length);
417
if (data->output_length > length) {
418
TSVIOReenable(data->output_vio);
421
/* Call back the write vio continuation to let it know that we
422
have completed the write operation. */
423
TSContCall(TSVIOContGet(write_vio), TS_EVENT_VCONN_WRITE_COMPLETE, write_vio);
429
gzip_transform(TSCont contp, TSEvent event, void *edata)
431
/* Check to see if the transformation has been closed by a call to
433
if (TSVConnClosedGet(contp)) {
434
gzip_data_destroy(TSContDataGet(contp));
435
TSContDestroy(contp);
443
/* Get the write vio for the write operation that was
444
performed on ourself. This vio contains the continuation of
445
our parent transformation. */
446
write_vio = TSVConnWriteVIOGet(contp);
448
/* Call back the write vio continuation to let it know that we
449
have completed the write operation. */
450
TSContCall(TSVIOContGet(write_vio), TS_EVENT_ERROR, write_vio);
453
case TS_EVENT_VCONN_WRITE_COMPLETE:
454
/* When our output connection says that it has finished
455
reading all the data we've written to it then we should
456
shutdown the write portion of its connection to
457
indicate that we don't want to hear about it anymore. */
458
TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
460
case TS_EVENT_VCONN_WRITE_READY:
462
/* If we get a WRITE_READY event or any other type of
463
event (sent, perhaps, because we were reenabled) then
464
we'll attempt to transform more data. */
465
gzip_transform_do(contp);
475
gzip_transformable(TSHttpTxn txnp, int server)
477
/* Server response header */
482
/* Client request header */
491
TSHttpTxnClientReqGet(txnp, &cbuf, &chdr);
493
/* check if client accepts "deflate" */
495
cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, -1);
496
if (TS_NULL_MLOC != cfield) {
497
nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield);
498
value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, 0, NULL);
501
while (nvalues > 0) {
502
if (value && (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0)) {
507
value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, NULL);
513
TSHandleMLocRelease(cbuf, chdr, cfield);
514
TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
516
TSHandleMLocRelease(cbuf, chdr, cfield);
517
TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
522
TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
524
TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
527
/* If there already exists a content encoding then we don't want
529
field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING, -1);
531
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
532
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
535
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
537
/* We only want to do gzip compression on documents that have a
538
content type of "text/" or "application/x-javascript". */
540
field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, -1);
542
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
546
value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, 0, NULL);
547
if (value && (strncasecmp(value, "text/", sizeof("text/") - 1) == 0)) {
548
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
549
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
551
} else if (value && (strncasecmp(value, "application/x-javascript", (sizeof("application/x-javascript") - 1)) == 0)) {
552
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
553
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
556
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
557
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
564
gzip_transform_add(TSHttpTxn txnp, int server)
569
connp = TSTransformCreate(gzip_transform, txnp);
571
data = gzip_data_alloc();
573
TSContDataSet(connp, data);
575
TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
580
transform_plugin(TSCont contp, TSEvent event, void *edata)
582
TSHttpTxn txnp = (TSHttpTxn) edata;
586
case TS_EVENT_HTTP_READ_RESPONSE_HDR:
587
reason = gzip_transformable(txnp, 1);
589
TSDebug("gzip-transform", "server content transformable");
590
gzip_transform_add(txnp, 1);
592
TSDebug("gzip-transform", "server content NOT transformable [%d]", reason);
595
TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
598
case TS_EVENT_HTTP_READ_CACHE_HDR:
600
reason = gzip_transformable(txnp, 0);
602
TSDebug("gzip-transform", "cached content transformable");
603
gzip_transform_add(txnp, 1);
605
TSDebug("gzip-transform", "cached data: forwarding unchanged (%d)", reason);
607
TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
619
TSPluginInit(int argc, const char *argv[])
621
dictId = adler32(0L, Z_NULL, 0);
623
strcpy(preload_file, argv[1]);
625
load_dictionary(dictionary, &dictId);
628
TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, TSContCreate(transform_plugin, NULL));
629
TSHttpHookAdd(TS_HTTP_READ_CACHE_HDR_HOOK, TSContCreate(transform_plugin, NULL));
added
removed
example/gzip-transform/gzip.c
