Unified compression interfaces

src/client.c

@@ -330,21 +330,24 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
                 if (uri.meta->filename_comp_br[0] != 0 && strstr(accept_encoding, "br") != NULL) {
                     file = fopen(uri.meta->filename_comp_br, "rb");
                     if (file == NULL) {
-                        printf("asdf\n");
                         cache_filename_comp_invalid(uri.filename);
-                    }
+                    } else {
                         http_add_header_field(&res.hdr, "Content-Encoding", "br");
+                    }
                 } else if (uri.meta->filename_comp_gz[0] != 0 && strstr(accept_encoding, "gzip") != NULL) {
                     file = fopen(uri.meta->filename_comp_gz, "rb");
                     if (file == NULL) {
                         cache_filename_comp_invalid(uri.filename);
-                    }
+                    } else {
                         http_add_header_field(&res.hdr, "Content-Encoding", "gzip");
                     }
                 }
+            }
+
             if (file == NULL) {
                 file = fopen(uri.filename, "rb");
             }
+
             fseek(file, 0, SEEK_END);
             content_length = ftell(file);
             fseek(file, 0, SEEK_SET);

src/lib/brotli.c

@@ -1,24 +0,0 @@
-/**
- *
- */
-
-#include "brotli.h"
-
-int brotli_init(BrotliEncoderState **state) {
-    *state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
-    if (*state == NULL) return -1;
-    BrotliEncoderSetParameter(*state, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC);
-    BrotliEncoderSetParameter(*state, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC);
-    return 0;
-}
-
-int brotli_compress(BrotliEncoderState *state, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
-    int ret = BrotliEncoderCompressStream(state, finish ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS,
-            in_len, (const unsigned char**) &in, out_len, (unsigned char **) &out, NULL);
-    return (ret == BROTLI_TRUE) ? 0 : -1;
-}
-
-int brotli_free(BrotliEncoderState *state) {
-    BrotliEncoderDestroyInstance(state);
-    return 0;
-}

src/lib/brotli.h

@@ -1,11 +0,0 @@
-/**
- *
- */
-
-#include <brotli/encode.h>
-
-int brotli_init(BrotliEncoderState **state);
-
-int brotli_compress(BrotliEncoderState *state, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish);
-
-int brotli_free(BrotliEncoderState *state);

src/lib/cache.c

@@ -7,8 +7,7 @@
 
 #include "cache.h"
 #include "utils.h"
-#include "gzip.h"
+#include "compress.h"
-#include "brotli.h"
 #include <stdio.h>
 #include <magic.h>
 #include <sys/ipc.h>
@@ -85,7 +84,7 @@ int cache_process() {
     unsigned char hash[SHA_DIGEST_LENGTH];
     int cache_changed = 0;
     int p_len_gz, p_len_br;
-    int ret_1, ret_2;
+    int ret;
     while (cache_continue) {
         for (int i = 0; i < CACHE_ENTRIES; i++) {
             if (cache[i].filename[0] != 0 && cache[i].meta.etag[0] == 0 && !cache[i].is_updating) {
@@ -95,8 +94,7 @@ int cache_process() {
                 file = fopen(cache[i].filename, "rb");
                 compress = mime_is_compressible(cache[i].meta.type);
 
-                z_stream gz_state;
+                compress_ctx comp_ctx;
-                BrotliEncoderState *br_state = NULL;
                 FILE *comp_file_gz = NULL;
                 FILE *comp_file_br = NULL;
                 if (compress) {
@@ -136,15 +134,9 @@ int cache_process() {
                         comp_err:
                         compress = 0;
                     } else {
-                        ret_1 = gzip_init(&gz_state);
+                        ret = compress_init(&comp_ctx, COMPRESS_GZ | COMPRESS_BR);
-                        ret_2 = brotli_init(&br_state);
+                        if (ret != 0) {
-                        if (ret_1 != 0) {
+                            fprintf(stderr, ERR_STR "Unable to init compression: %s" CLR_STR "\n", strerror(errno));
-                            fprintf(stderr, ERR_STR "Unable to init gzip: %s" CLR_STR "\n", strerror(errno));
-                            compress = 0;
-                            fclose(comp_file_gz);
-                            fclose(comp_file_br);
-                        } else if (ret_2 != 0) {
-                            fprintf(stderr, ERR_STR "Unable to init brotli: %s" CLR_STR "\n", strerror(errno));
                             compress = 0;
                             fclose(comp_file_gz);
                             fclose(comp_file_br);
@@ -159,21 +151,22 @@ int cache_process() {
                         avail_in = read;
                         do {
                             avail_out = sizeof(comp_buf);
-                            gzip_compress(&gz_state, buf + read - avail_in, &avail_in, comp_buf, &avail_out, feof(file));
+                            compress_compress_mode(&comp_ctx, COMPRESS_GZ,buf + read - avail_in, &avail_in,
+                                                   comp_buf, &avail_out, feof(file));
                             fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_gz);
                         } while (avail_in != 0);
                         avail_in = read;
                         do {
                             avail_out = sizeof(comp_buf);
-                            brotli_compress(br_state, buf + read - avail_in, &avail_in, comp_buf, &avail_out, feof(file));
+                            compress_compress_mode(&comp_ctx, COMPRESS_BR, buf + read - avail_in, &avail_in,
+                                                   comp_buf, &avail_out, feof(file));
                             fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_br);
                         } while (avail_in != 0);
                     }
                 }
 
                 if (compress) {
-                    gzip_free(&gz_state);
+                    compress_free(&comp_ctx);
-                    brotli_free(br_state);
                     fclose(comp_file_gz);
                     fclose(comp_file_br);
                     fprintf(stdout, "[cache] Finished compressing file %s\n", cache[i].filename);

src/lib/compress.c

@@ -0,0 +1,76 @@
+/**
+ * Necronda Web Server
+ * Compression interface
+ * src/lib/compress.c
+ * Lorenz Stechauner, 2021-05-05
+ */
+
+#include "compress.h"
+#include <malloc.h>
+#include <errno.h>
+
+int compress_init(compress_ctx *ctx, int mode) {
+    ctx->gzip = NULL;
+    ctx->brotli = NULL;
+    int ret;
+    if (mode & COMPRESS_GZ) {
+        ctx->gzip = malloc(sizeof(z_stream));
+        ctx->gzip->zalloc = Z_NULL;
+        ctx->gzip->zfree = Z_NULL;
+        ctx->gzip->opaque = Z_NULL;
+        ret = deflateInit2(ctx->gzip, COMPRESS_LEVEL_GZIP, Z_DEFLATED, 15 + 16, 9, Z_DEFAULT_STRATEGY);
+        if (ret != Z_OK) return -1;
+    }
+    if (mode & COMPRESS_BR) {
+        ctx->brotli = BrotliEncoderCreateInstance(NULL, NULL, NULL);
+        if (ctx->brotli == NULL) return -1;
+        BrotliEncoderSetParameter(ctx->brotli, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC);
+        BrotliEncoderSetParameter(ctx->brotli, BROTLI_PARAM_QUALITY, COMPRESS_LEVEL_BROTLI);
+    }
+    return 0;
+}
+
+int compress_compress(compress_ctx *ctx, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
+    if ((ctx->mode & COMPRESS_GZ) && (ctx->mode & COMPRESS_BR)) {
+        errno = EINVAL;
+        return -1;
+    }
+    return compress_compress_mode(ctx, ctx->mode, in, in_len, out, out_len, finish);
+}
+
+int compress_compress_mode(compress_ctx *ctx, int mode, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
+    if ((mode & COMPRESS_GZ) && (mode & COMPRESS_BR)) {
+        errno = EINVAL;
+        return -1;
+    } else if (mode & COMPRESS_GZ) {
+        ctx->gzip->next_in = (unsigned char*) in;
+        ctx->gzip->avail_in = *in_len;
+        ctx->gzip->next_out = (unsigned char*) out;
+        ctx->gzip->avail_out = *out_len;
+        int ret = deflate(ctx->gzip, finish ? Z_FINISH : Z_NO_FLUSH);
+        *in_len = ctx->gzip->avail_in;
+        *out_len = ctx->gzip->avail_out;
+        return ret;
+    } else if (mode & COMPRESS_BR) {
+        int ret = BrotliEncoderCompressStream(ctx->brotli, finish ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS,
+                                              in_len, (const unsigned char**) &in, out_len, (unsigned char **) &out, NULL);
+        return (ret == BROTLI_TRUE) ? 0 : -1;
+    } else {
+        errno = EINVAL;
+        return -2;
+    }
+}
+
+int compress_free(compress_ctx *ctx) {
+    if (ctx->gzip != NULL) {
+        deflateEnd(ctx->gzip);
+        free(ctx->gzip);
+        ctx->gzip = NULL;
+    }
+    if (ctx->brotli != NULL) {
+        BrotliEncoderDestroyInstance(ctx->brotli);
+        ctx->brotli = NULL;
+    }
+    ctx->mode = 0;
+    return 0;
+}

src/lib/compress.h

@@ -0,0 +1,36 @@
+/**
+ * Necronda Web Server
+ * Compression interface (header file)
+ * src/lib/compress.h
+ * Lorenz Stechauner, 2021-05-05
+ */
+
+#ifndef NECRONDA_SERVER_COMPRESS_H
+#define NECRONDA_SERVER_COMPRESS_H
+
+#include <zlib.h>
+#include <brotli/encode.h>
+
+#define COMPRESS_LEVEL_GZIP 9
+#define COMPRESS_LEVEL_BROTLI BROTLI_MAX_QUALITY
+
+#define COMPRESS_GZ 1
+#define COMPRESS_BR 2
+
+typedef struct {
+    int mode;
+    z_stream *gzip;
+    BrotliEncoderState *brotli;
+} compress_ctx;
+
+int compress_init(compress_ctx *ctx, int mode);
+
+int compress_compress(compress_ctx *ctx, const char *in, unsigned long *in_len, char *out, unsigned long *out_len,
+                      int finish);
+
+int compress_compress_mode(compress_ctx *ctx, int mode, const char *in, unsigned long *in_len, char *out,
+                           unsigned long *out_len, int finish);
+
+int compress_free(compress_ctx *ctx);
+
+#endif //NECRONDA_SERVER_COMPRESS_H

src/lib/fastcgi.c

@@ -7,8 +7,7 @@
 
 #include "fastcgi.h"
 #include "utils.h"
-#include "gzip.h"
+#include "compress.h"
-#include "brotli.h"
 #include "../necronda-server.h"
 #include <sys/un.h>
 #include <sys/socket.h>
@@ -393,17 +392,16 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
     char comp_out[4096];
     int finish_comp = 0;
 
-    z_stream gz_state;
+    compress_ctx comp_ctx;
-    BrotliEncoderState *br_state = NULL;
     if (flags & FASTCGI_COMPRESS_BR) {
         flags &= ~FASTCGI_COMPRESS_GZ;
-        if (brotli_init(&br_state) != 0) {
+        if (compress_init(&comp_ctx, COMPRESS_BR) != 0) {
             print(ERR_STR "Unable to init brotli: %s" CLR_STR, strerror(errno));
             flags &= ~FASTCGI_COMPRESS_BR;
         }
     } else if (flags & FASTCGI_COMPRESS_GZ) {
         flags &= ~FASTCGI_COMPRESS_BR;
-        if (gzip_init(&gz_state) != 0) {
+        if (compress_init(&comp_ctx, COMPRESS_BR) != 0) {
             print(ERR_STR "Unable to init gzip: %s" CLR_STR, strerror(errno));
             flags &= ~FASTCGI_COMPRESS_GZ;
         }
@@ -459,8 +457,7 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
                 content_len = 0;
                 goto out;
                 finish:
-                if (flags & FASTCGI_COMPRESS_GZ) gzip_free(&gz_state);
+                if (flags & FASTCGI_COMPRESS) compress_free(&comp_ctx);
-                if (flags & FASTCGI_COMPRESS_BR) brotli_free(br_state);
             }
 
             if (flags & FASTCGI_CHUNKED) {
@@ -479,10 +476,9 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
                 int buf_len = content_len;
                 if (flags & FASTCGI_COMPRESS) {
                     avail_out = sizeof(comp_out);
-                    if (flags & FASTCGI_COMPRESS_GZ) {
+                    if (flags & FASTCGI_COMPRESS) {
-                        gzip_compress(&gz_state, next_in + content_len - avail_in, &avail_in, comp_out, &avail_out, finish_comp);
+                        compress_compress(&comp_ctx, next_in + content_len - avail_in, &avail_in,
-                    } else if (flags & FASTCGI_COMPRESS_BR) {
+                                          comp_out, &avail_out, finish_comp);
-                        brotli_compress(br_state, next_in + content_len - avail_in, &avail_in, comp_out, &avail_out, finish_comp);
                     }
                     ptr = comp_out;
                     buf_len = (int) (sizeof(comp_out) - avail_out);

src/lib/gzip.c

@@ -1,28 +0,0 @@
-/**
- *
- */
-
-#include "gzip.h"
-
-int gzip_init(z_stream *stream) {
-    stream->zalloc = Z_NULL;
-    stream->zfree = Z_NULL;
-    stream->opaque = Z_NULL;
-    int ret = deflateInit2(stream, GZIP_LEVEL, Z_DEFLATED, 15 + 16, 9, Z_DEFAULT_STRATEGY);
-    return (ret == Z_OK) ? 0 : -1;
-}
-
-int gzip_compress(z_stream *stream, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
-    stream->next_in = (unsigned char*) in;
-    stream->avail_in = *in_len;
-    stream->next_out = (unsigned char*) out;
-    stream->avail_out = *out_len;
-    int ret = deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH);
-    *in_len = stream->avail_in;
-    *out_len = stream->avail_out;
-    return ret;
-}
-
-int gzip_free(z_stream *stream) {
-    return deflateEnd(stream);
-}

src/lib/gzip.h

@@ -1,18 +0,0 @@
-/**
- *
- */
-
-#ifndef NECRONDA_SERVER_GZIP_H
-#define NECRONDA_SERVER_GZIP_H
-
-#include <zlib.h>
-
-#define GZIP_LEVEL 9
-
-int gzip_init(z_stream *stream);
-
-int gzip_compress(z_stream *stream, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish);
-
-int gzip_free(z_stream *stream);
-
-#endif //NECRONDA_SERVER_GZIP_H

src/lib/rev_proxy.c

@@ -382,6 +382,7 @@ int rev_proxy_send(sock *client, int chunked, unsigned long len_to_send) {
     char buffer[CHUNK_SIZE];
     long len, snd_len;
     // TODO handle websockets
+    // TODO compress -> Transfer-Encoding: br/gzip
     do {
         if (chunked) {
             ret = sock_recv(&rev_proxy, buffer, 16, MSG_PEEK);