lorenz.stechauner/sesimos
1
1
Fork 0
Code Issues Pull Requests Projects Releases Activity

Unified compression interfaces

Browse Source
This commit is contained in:
Lorenz Stechauner
2021-05-05 18:07:12 +02:00
parent ff708230bd
commit c2f8f4c962
10 changed files with 138 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/client.c
View File

@ -330,21 +330,24 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
if (uri.meta->filename_comp_br[0] != 0 && strstr(accept_encoding, "br") != NULL) {
file = fopen(uri.meta->filename_comp_br, "rb");
if (file == NULL) {
printf("asdf\n");
cache_filename_comp_invalid(uri.filename);
} else {
http_add_header_field(&res.hdr, "Content-Encoding", "br");
}
http_add_header_field(&res.hdr, "Content-Encoding", "br");
} else if (uri.meta->filename_comp_gz[0] != 0 && strstr(accept_encoding, "gzip") != NULL) {
file = fopen(uri.meta->filename_comp_gz, "rb");
if (file == NULL) {
cache_filename_comp_invalid(uri.filename);
} else {
http_add_header_field(&res.hdr, "Content-Encoding", "gzip");
}
http_add_header_field(&res.hdr, "Content-Encoding", "gzip");
}
}
if (file == NULL) {
file = fopen(uri.filename, "rb");
}
fseek(file, 0, SEEK_END);
content_length = ftell(file);
fseek(file, 0, SEEK_SET);

24
src/lib/brotli.c
View File

@ -1,24 +0,0 @@
/**
*
*/
#include "brotli.h"
int brotli_init(BrotliEncoderState **state) {
*state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
if (*state == NULL) return -1;
BrotliEncoderSetParameter(*state, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC);
BrotliEncoderSetParameter(*state, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC);
return 0;
}
int brotli_compress(BrotliEncoderState *state, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
int ret = BrotliEncoderCompressStream(state, finish ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS,
in_len, (const unsigned char**) &in, out_len, (unsigned char **) &out, NULL);
return (ret == BROTLI_TRUE) ? 0 : -1;
}
int brotli_free(BrotliEncoderState *state) {
BrotliEncoderDestroyInstance(state);
return 0;
}

11
src/lib/brotli.h
View File

@ -1,11 +0,0 @@
/**
*
*/
#include <brotli/encode.h>
int brotli_init(BrotliEncoderState **state);
int brotli_compress(BrotliEncoderState *state, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish);
int brotli_free(BrotliEncoderState *state);

29
src/lib/cache.c
View File

@ -7,8 +7,7 @@
#include "cache.h"
#include "utils.h"
#include "gzip.h"
#include "brotli.h"
#include "compress.h"
#include <stdio.h>
#include <magic.h>
#include <sys/ipc.h>
@ -85,7 +84,7 @@ int cache_process() {
unsigned char hash[SHA_DIGEST_LENGTH];
int cache_changed = 0;
int p_len_gz, p_len_br;
int ret_1, ret_2;
int ret;
while (cache_continue) {
for (int i = 0; i < CACHE_ENTRIES; i++) {
if (cache[i].filename[0] != 0 && cache[i].meta.etag[0] == 0 && !cache[i].is_updating) {
@ -95,8 +94,7 @@ int cache_process() {
file = fopen(cache[i].filename, "rb");
compress = mime_is_compressible(cache[i].meta.type);
z_stream gz_state;
BrotliEncoderState *br_state = NULL;
compress_ctx comp_ctx;
FILE *comp_file_gz = NULL;
FILE *comp_file_br = NULL;
if (compress) {
@ -136,15 +134,9 @@ int cache_process() {
comp_err:
compress = 0;
} else {
ret_1 = gzip_init(&gz_state);
ret_2 = brotli_init(&br_state);
if (ret_1 != 0) {
fprintf(stderr, ERR_STR "Unable to init gzip: %s" CLR_STR "\n", strerror(errno));
compress = 0;
fclose(comp_file_gz);
fclose(comp_file_br);
} else if (ret_2 != 0) {
fprintf(stderr, ERR_STR "Unable to init brotli: %s" CLR_STR "\n", strerror(errno));
ret = compress_init(&comp_ctx, COMPRESS_GZ | COMPRESS_BR);
if (ret != 0) {
fprintf(stderr, ERR_STR "Unable to init compression: %s" CLR_STR "\n", strerror(errno));
compress = 0;
fclose(comp_file_gz);
fclose(comp_file_br);
@ -159,21 +151,22 @@ int cache_process() {
avail_in = read;
do {
avail_out = sizeof(comp_buf);
gzip_compress(&gz_state, buf + read - avail_in, &avail_in, comp_buf, &avail_out, feof(file));
compress_compress_mode(&comp_ctx, COMPRESS_GZ,buf + read - avail_in, &avail_in,
comp_buf, &avail_out, feof(file));
fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_gz);
} while (avail_in != 0);
avail_in = read;
do {
avail_out = sizeof(comp_buf);
brotli_compress(br_state, buf + read - avail_in, &avail_in, comp_buf, &avail_out, feof(file));
compress_compress_mode(&comp_ctx, COMPRESS_BR, buf + read - avail_in, &avail_in,
comp_buf, &avail_out, feof(file));
fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_br);
} while (avail_in != 0);
}
}
if (compress) {
gzip_free(&gz_state);
brotli_free(br_state);
compress_free(&comp_ctx);
fclose(comp_file_gz);
fclose(comp_file_br);
fprintf(stdout, "[cache] Finished compressing file %s\n", cache[i].filename);

76
src/lib/compress.c Normal file
View File

@ -0,0 +1,76 @@
/**
* Necronda Web Server
* Compression interface
* src/lib/compress.c
* Lorenz Stechauner, 2021-05-05
*/
#include "compress.h"
#include <malloc.h>
#include <errno.h>
int compress_init(compress_ctx *ctx, int mode) {
ctx->gzip = NULL;
ctx->brotli = NULL;
int ret;
if (mode & COMPRESS_GZ) {
ctx->gzip = malloc(sizeof(z_stream));
ctx->gzip->zalloc = Z_NULL;
ctx->gzip->zfree = Z_NULL;
ctx->gzip->opaque = Z_NULL;
ret = deflateInit2(ctx->gzip, COMPRESS_LEVEL_GZIP, Z_DEFLATED, 15 + 16, 9, Z_DEFAULT_STRATEGY);
if (ret != Z_OK) return -1;
}
if (mode & COMPRESS_BR) {
ctx->brotli = BrotliEncoderCreateInstance(NULL, NULL, NULL);
if (ctx->brotli == NULL) return -1;
BrotliEncoderSetParameter(ctx->brotli, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC);
BrotliEncoderSetParameter(ctx->brotli, BROTLI_PARAM_QUALITY, COMPRESS_LEVEL_BROTLI);
}
return 0;
}
int compress_compress(compress_ctx *ctx, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
if ((ctx->mode & COMPRESS_GZ) && (ctx->mode & COMPRESS_BR)) {
errno = EINVAL;
return -1;
}
return compress_compress_mode(ctx, ctx->mode, in, in_len, out, out_len, finish);
}
int compress_compress_mode(compress_ctx *ctx, int mode, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
if ((mode & COMPRESS_GZ) && (mode & COMPRESS_BR)) {
errno = EINVAL;
return -1;
} else if (mode & COMPRESS_GZ) {
ctx->gzip->next_in = (unsigned char*) in;
ctx->gzip->avail_in = *in_len;
ctx->gzip->next_out = (unsigned char*) out;
ctx->gzip->avail_out = *out_len;
int ret = deflate(ctx->gzip, finish ? Z_FINISH : Z_NO_FLUSH);
*in_len = ctx->gzip->avail_in;
*out_len = ctx->gzip->avail_out;
return ret;
} else if (mode & COMPRESS_BR) {
int ret = BrotliEncoderCompressStream(ctx->brotli, finish ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS,
in_len, (const unsigned char**) &in, out_len, (unsigned char **) &out, NULL);
return (ret == BROTLI_TRUE) ? 0 : -1;
} else {
errno = EINVAL;
return -2;
}
}
int compress_free(compress_ctx *ctx) {
if (ctx->gzip != NULL) {
deflateEnd(ctx->gzip);
free(ctx->gzip);
ctx->gzip = NULL;
}
if (ctx->brotli != NULL) {
BrotliEncoderDestroyInstance(ctx->brotli);
ctx->brotli = NULL;
}
ctx->mode = 0;
return 0;
}

36
src/lib/compress.h Normal file
View File

@ -0,0 +1,36 @@
/**
* Necronda Web Server
* Compression interface (header file)
* src/lib/compress.h
* Lorenz Stechauner, 2021-05-05
*/
#ifndef NECRONDA_SERVER_COMPRESS_H
#define NECRONDA_SERVER_COMPRESS_H
#include <zlib.h>
#include <brotli/encode.h>
#define COMPRESS_LEVEL_GZIP 9
#define COMPRESS_LEVEL_BROTLI BROTLI_MAX_QUALITY
#define COMPRESS_GZ 1
#define COMPRESS_BR 2
typedef struct {
int mode;
z_stream *gzip;
BrotliEncoderState *brotli;
} compress_ctx;
int compress_init(compress_ctx *ctx, int mode);
int compress_compress(compress_ctx *ctx, const char *in, unsigned long *in_len, char *out, unsigned long *out_len,
int finish);
int compress_compress_mode(compress_ctx *ctx, int mode, const char *in, unsigned long *in_len, char *out,
unsigned long *out_len, int finish);
int compress_free(compress_ctx *ctx);
#endif //NECRONDA_SERVER_COMPRESS_H

20
src/lib/fastcgi.c
View File

@ -7,8 +7,7 @@
#include "fastcgi.h"
#include "utils.h"
#include "gzip.h"
#include "brotli.h"
#include "compress.h"
#include "../necronda-server.h"
#include <sys/un.h>
#include <sys/socket.h>
@ -393,17 +392,16 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
char comp_out[4096];
int finish_comp = 0;
z_stream gz_state;
BrotliEncoderState *br_state = NULL;
compress_ctx comp_ctx;
if (flags & FASTCGI_COMPRESS_BR) {
flags &= ~FASTCGI_COMPRESS_GZ;
if (brotli_init(&br_state) != 0) {
if (compress_init(&comp_ctx, COMPRESS_BR) != 0) {
print(ERR_STR "Unable to init brotli: %s" CLR_STR, strerror(errno));
flags &= ~FASTCGI_COMPRESS_BR;
}
} else if (flags & FASTCGI_COMPRESS_GZ) {
flags &= ~FASTCGI_COMPRESS_BR;
if (gzip_init(&gz_state) != 0) {
if (compress_init(&comp_ctx, COMPRESS_BR) != 0) {
print(ERR_STR "Unable to init gzip: %s" CLR_STR, strerror(errno));
flags &= ~FASTCGI_COMPRESS_GZ;
}
@ -459,8 +457,7 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
content_len = 0;
goto out;
finish:
if (flags & FASTCGI_COMPRESS_GZ) gzip_free(&gz_state);
if (flags & FASTCGI_COMPRESS_BR) brotli_free(br_state);
if (flags & FASTCGI_COMPRESS) compress_free(&comp_ctx);
}
if (flags & FASTCGI_CHUNKED) {
@ -479,10 +476,9 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
int buf_len = content_len;
if (flags & FASTCGI_COMPRESS) {
avail_out = sizeof(comp_out);
if (flags & FASTCGI_COMPRESS_GZ) {
gzip_compress(&gz_state, next_in + content_len - avail_in, &avail_in, comp_out, &avail_out, finish_comp);
} else if (flags & FASTCGI_COMPRESS_BR) {
brotli_compress(br_state, next_in + content_len - avail_in, &avail_in, comp_out, &avail_out, finish_comp);
if (flags & FASTCGI_COMPRESS) {
compress_compress(&comp_ctx, next_in + content_len - avail_in, &avail_in,
comp_out, &avail_out, finish_comp);
}
ptr = comp_out;
buf_len = (int) (sizeof(comp_out) - avail_out);

28
src/lib/gzip.c
View File

@ -1,28 +0,0 @@
/**
*
*/
#include "gzip.h"
int gzip_init(z_stream *stream) {
stream->zalloc = Z_NULL;
stream->zfree = Z_NULL;
stream->opaque = Z_NULL;
int ret = deflateInit2(stream, GZIP_LEVEL, Z_DEFLATED, 15 + 16, 9, Z_DEFAULT_STRATEGY);
return (ret == Z_OK) ? 0 : -1;
}
int gzip_compress(z_stream *stream, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
stream->next_in = (unsigned char*) in;
stream->avail_in = *in_len;
stream->next_out = (unsigned char*) out;
stream->avail_out = *out_len;
int ret = deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH);
*in_len = stream->avail_in;
*out_len = stream->avail_out;
return ret;
}
int gzip_free(z_stream *stream) {
return deflateEnd(stream);
}

18
src/lib/gzip.h
View File

@ -1,18 +0,0 @@
/**
*
*/
#ifndef NECRONDA_SERVER_GZIP_H
#define NECRONDA_SERVER_GZIP_H
#include <zlib.h>
#define GZIP_LEVEL 9
int gzip_init(z_stream *stream);
int gzip_compress(z_stream *stream, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish);
int gzip_free(z_stream *stream);
#endif //NECRONDA_SERVER_GZIP_H

1
src/lib/rev_proxy.c
View File

@ -382,6 +382,7 @@ int rev_proxy_send(sock *client, int chunked, unsigned long len_to_send) {
char buffer[CHUNK_SIZE];
long len, snd_len;
// TODO handle websockets
// TODO compress -> Transfer-Encoding: br/gzip
do {
if (chunked) {
ret = sock_recv(&rev_proxy, buffer, 16, MSG_PEEK);