ETag Content-Encoding bugfix

Makefile

@@ -1,9 +1,9 @@
 
 CFLAGS=-std=c11 -Wall
-INCLUDE=-lssl -lcrypto -lmagic -lz -lmaxminddb
+INCLUDE=-lssl -lcrypto -lmagic -lz -lmaxminddb -lbrotlienc
 LIBS=src/lib/*.c
 
-DEBIAN_OPTS=-D MAGIC_FILE="\"/usr/share/file/magic.mgc\"" -D PHP_FPM_SOCKET="\"/var/run/php/php7.3-fpm.sock\""
+DEBIAN_OPTS=-D CACHE_MAGIC_FILE="\"/usr/share/file/magic.mgc\"" -D PHP_FPM_SOCKET="\"/var/run/php/php7.3-fpm.sock\""
 
 packages:
 	@echo "Installing packages..."

src/client.c

@@ -5,7 +5,6 @@
  * Lorenz Stechauner, 2020-12-03
  */
 
-#include "client.h"
 #include "lib/utils.h"
 #include "lib/config.h"
 #include "lib/sock.h"
@@ -13,6 +12,8 @@
 #include "lib/rev_proxy.h"
 #include "lib/fastcgi.h"
 #include "lib/cache.h"
+#include "lib/geoip.h"
+#include "lib/compress.h"
 
 #include <string.h>
 #include <sys/select.h>
@@ -32,7 +33,7 @@ char *client_addr_str, *client_addr_str_ptr, *server_addr_str, *server_addr_str_
 struct timeval client_timeout;
 
 host_config *get_host_config(const char *host) {
-    for (int i = 0; i < MAX_HOST_CONFIG; i++) {
+    for (int i = 0; i < CONFIG_MAX_HOST_CONFIG; i++) {
         host_config *hc = &config[i];
         if (hc->type == CONFIG_TYPE_UNSET) break;
         if (strcmp(hc->name, host) == 0) return hc;
@@ -263,9 +264,40 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
             http_add_header_field(&res.hdr, "Last-Modified", last_modified);
             sprintf(buf1, "%s; charset=%s", uri.meta->type, uri.meta->charset);
             http_add_header_field(&res.hdr, "Content-Type", buf1);
-            if (uri.meta->etag[0] != 0) {
-                http_add_header_field(&res.hdr, "ETag", uri.meta->etag);
+
+
+            char *accept_encoding = http_get_header_field(&req.hdr, "Accept-Encoding");
+            int enc = 0;
+            if (accept_encoding != NULL) {
+                if (uri.meta->filename_comp_br[0] != 0 && strstr(accept_encoding, "br") != NULL) {
+                    file = fopen(uri.meta->filename_comp_br, "rb");
+                    if (file == NULL) {
+                        cache_filename_comp_invalid(uri.filename);
+                    } else {
+                        http_add_header_field(&res.hdr, "Content-Encoding", "br");
+                        enc = COMPRESS_BR;
+                    }
+                } else if (uri.meta->filename_comp_gz[0] != 0 && strstr(accept_encoding, "gzip") != NULL) {
+                    file = fopen(uri.meta->filename_comp_gz, "rb");
+                    if (file == NULL) {
+                        cache_filename_comp_invalid(uri.filename);
+                    } else {
+                        http_add_header_field(&res.hdr, "Content-Encoding", "gzip");
+                        enc = COMPRESS_GZ;
+                    }
+                }
             }
+
+            if (uri.meta->etag[0] != 0) {
+                if (enc) {
+                    sprintf(buf0, "%s-%s", uri.meta->etag,
+                            (enc & COMPRESS_BR) ? "br" : (enc & COMPRESS_GZ) ? "gzip" : "");
+                    http_add_header_field(&res.hdr, "ETag", buf0);
+                } else {
+                    http_add_header_field(&res.hdr, "ETag", uri.meta->etag);
+                }
+            }
+
             if (strncmp(uri.meta->type, "text/", 5) == 0) {
                 http_add_header_field(&res.hdr, "Cache-Control", "public, max-age=3600");
             } else {
@@ -325,19 +357,10 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
                 goto respond;
             }
 
-            char *accept_encoding = http_get_header_field(&req.hdr, "Accept-Encoding");
-            if (uri.meta->filename_comp[0] != 0 && accept_encoding != NULL &&
-                    strstr(accept_encoding, "deflate") != NULL) {
-                file = fopen(uri.meta->filename_comp, "rb");
-                if (file == NULL) {
-                    cache_filename_comp_invalid(uri.filename);
-                    goto not_compressed;
-                }
-                http_add_header_field(&res.hdr, "Content-Encoding", "deflate");
-            } else {
-                not_compressed:
+            if (file == NULL) {
                 file = fopen(uri.filename, "rb");
             }
+
             fseek(file, 0, SEEK_END);
             content_length = ftell(file);
             fseek(file, 0, SEEK_SET);
@@ -397,13 +420,11 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
             content_length = -1;
             use_fastcgi = 1;
 
-            char *accept_encoding = http_get_header_field(&req.hdr, "Accept-Encoding");
-            char *content_type = http_get_header_field(&res.hdr, "Content-Type");
-            char *content_encoding = http_get_header_field(&res.hdr, "Content-Encoding");
-            if (mime_is_compressible(content_type) && content_encoding == NULL &&
-                    accept_encoding != NULL && strstr(accept_encoding, "deflate") != NULL) {
-                http_add_header_field(&res.hdr, "Content-Encoding", "deflate");
-                use_fastcgi = 2;
+            int http_comp = http_get_compression(&req, &res);
+            if (http_comp & COMPRESS_BR) {
+                use_fastcgi |= FASTCGI_COMPRESS_BR;
+            } else if (http_comp & COMPRESS_GZ) {
+                use_fastcgi |= FASTCGI_COMPRESS_GZ;
             }
 
             if (http_get_header_field(&res.hdr, "Content-Length") == NULL) {
@@ -414,8 +435,32 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
         print("Reverse proxy for " BLD_STR "%s:%i" CLR_STR, conf->rev_proxy.hostname, conf->rev_proxy.port);
         http_remove_header_field(&res.hdr, "Date", HTTP_REMOVE_ALL);
         http_remove_header_field(&res.hdr, "Server", HTTP_REMOVE_ALL);
+
         ret = rev_proxy_init(&req, &res, conf, client, &custom_status, err_msg);
-        use_rev_proxy = ret == 0;
+        use_rev_proxy = (ret == 0);
+
+        char *content_encoding = http_get_header_field(&res.hdr, "Content-Encoding");
+        if (use_rev_proxy && content_encoding == NULL) {
+            int http_comp = http_get_compression(&req, &res);
+            if (http_comp & COMPRESS_BR) {
+                use_rev_proxy |= REV_PROXY_COMPRESS_BR;
+            } else if (http_comp & COMPRESS_GZ) {
+                use_rev_proxy |= REV_PROXY_COMPRESS_GZ;
+            }
+            use_rev_proxy &= ~REV_PROXY_COMPRESS;
+        }
+
+        char *transfer_encoding = http_get_header_field(&res.hdr, "Transfer-Encoding");
+        int chunked = transfer_encoding != NULL && strcmp(transfer_encoding, "chunked") == 0;
+        http_remove_header_field(&res.hdr, "Transfer-Encoding", HTTP_REMOVE_ALL);
+        ret = sprintf(buf0, "%s%s%s",
+                      (use_rev_proxy & REV_PROXY_COMPRESS_BR) ? "br" :
+                      ((use_rev_proxy & REV_PROXY_COMPRESS_GZ) ? "gzip" : ""),
+                      ((use_rev_proxy & REV_PROXY_COMPRESS) && chunked) ? ", " : "",
+                      chunked ? "chunked" : "");
+        if (ret > 0) {
+            http_add_header_field(&res.hdr, "Transfer-Encoding", buf0);
+        }
     } else {
         print(ERR_STR "Unknown host type: %i" CLR_STR, conf->type);
         res.status = http_get_status(501);
@@ -501,17 +546,21 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
         } else if (use_fastcgi) {
             char *transfer_encoding = http_get_header_field(&res.hdr, "Transfer-Encoding");
             int chunked = transfer_encoding != NULL && strcmp(transfer_encoding, "chunked") == 0;
-            int flags = (chunked ? FASTCGI_CHUNKED : 0) | ((use_fastcgi == 2) ? FASTCGI_COMPRESS : 0);
+
+            int flags = (chunked ? FASTCGI_CHUNKED : 0) | (use_fastcgi & FASTCGI_COMPRESS);
             fastcgi_send(&php_fpm, client, flags);
         } else if (use_rev_proxy) {
             char *transfer_encoding = http_get_header_field(&res.hdr, "Transfer-Encoding");
-            int chunked = transfer_encoding != NULL && strcmp(transfer_encoding, "chunked") == 0;
+            int chunked = transfer_encoding != NULL && strstr(transfer_encoding, "chunked") != NULL;
+
             char *content_len = http_get_header_field(&res.hdr, "Content-Length");
             unsigned long len_to_send = 0;
             if (content_len != NULL) {
                 len_to_send = strtol(content_len, NULL, 10);
             }
-            rev_proxy_send(client, chunked, len_to_send);
+
+            int flags = (chunked ? REV_PROXY_CHUNKED : 0) | (use_rev_proxy & REV_PROXY_COMPRESS);
+            rev_proxy_send(client, len_to_send, flags);
         }
     }
 

src/client.h

@@ -1,18 +0,0 @@
-/**
- * Necronda Web Server
- * Client connection and request handlers (header file)
- * src/client.h
- * Lorenz Stechauner, 2021-01-17
- */
-
-#ifndef NECRONDA_SERVER_CLIENT_H
-#define NECRONDA_SERVER_CLIENT_H
-
-#include <sys/time.h>
-
-extern int server_keep_alive;
-extern char *log_client_prefix, *log_conn_prefix, *log_req_prefix, *client_geoip;
-extern char *client_addr_str, *client_addr_str_ptr, *server_addr_str, *server_addr_str_ptr, *client_host_str;
-extern struct timeval client_timeout;
-
-#endif //NECRONDA_SERVER_CLIENT_H

src/lib/cache.c

@@ -7,9 +7,8 @@
 
 #include "cache.h"
 #include "utils.h"
-#include "../necronda-server.h"
+#include "compress.h"
 #include <stdio.h>
-#include <zlib.h>
 #include <magic.h>
 #include <sys/ipc.h>
 #include <sys/shm.h>
@@ -17,6 +16,7 @@
 #include <errno.h>
 #include <signal.h>
 #include <openssl/sha.h>
+#include <malloc.h>
 
 int cache_continue = 1;
 magic_t magic;
@@ -28,7 +28,7 @@ int magic_init() {
         fprintf(stderr, ERR_STR "Unable to open magic cookie: %s" CLR_STR "\n", strerror(errno));
         return -1;
     }
-    if (magic_load(magic, MAGIC_FILE) != 0) {
+    if (magic_load(magic, CACHE_MAGIC_FILE) != 0) {
         fprintf(stderr, ERR_STR "Unable to load magic cookie: %s" CLR_STR "\n", magic_error(magic));
         return -2;
     }
@@ -43,7 +43,7 @@ int cache_process() {
     signal(SIGINT, cache_process_term);
     signal(SIGTERM, cache_process_term);
 
-    int shm_id = shmget(SHM_KEY_CACHE, FILE_CACHE_SIZE * sizeof(cache_entry), 0);
+    int shm_id = shmget(CACHE_SHM_KEY, CACHE_ENTRIES * sizeof(cache_entry), 0);
     if (shm_id < 0) {
         fprintf(stderr, ERR_STR "Unable to create shared memory: %s" CLR_STR "\n", strerror(errno));
         return -1;
@@ -66,26 +66,28 @@ int cache_process() {
 
     FILE *cache_file = fopen("/var/necronda-server/cache", "rb");
     if (cache_file != NULL) {
-        fread(cache, sizeof(cache_entry), FILE_CACHE_SIZE, cache_file);
+        fread(cache, sizeof(cache_entry), CACHE_ENTRIES, cache_file);
         fclose(cache_file);
     }
 
-    for (int i = 0; i < FILE_CACHE_SIZE; i++) {
+    for (int i = 0; i < CACHE_ENTRIES; i++) {
         cache[i].is_updating = 0;
     }
 
     FILE *file;
-    char buf[16384];
-    char comp_buf[16384];
-    char filename_comp[256];
+    char *buf = malloc(CACHE_BUF_SIZE);
+    char *comp_buf = malloc(CACHE_BUF_SIZE);
+    char filename_comp_gz[256];
+    char filename_comp_br[256];
     unsigned long read;
     int compress;
     SHA_CTX ctx;
     unsigned char hash[SHA_DIGEST_LENGTH];
     int cache_changed = 0;
-    int p_len;
+    int p_len_gz, p_len_br;
+    int ret;
     while (cache_continue) {
-        for (int i = 0; i < FILE_CACHE_SIZE; i++) {
+        for (int i = 0; i < CACHE_ENTRIES; i++) {
             if (cache[i].filename[0] != 0 && cache[i].meta.etag[0] == 0 && !cache[i].is_updating) {
                 cache[i].is_updating = 1;
                 fprintf(stdout, "[cache] Hashing file %s\n", cache[i].filename);
@@ -93,9 +95,9 @@ int cache_process() {
                 file = fopen(cache[i].filename, "rb");
                 compress = mime_is_compressible(cache[i].meta.type);
 
-                int level = NECRONDA_ZLIB_LEVEL;
-                z_stream strm;
-                FILE *comp_file = NULL;
+                compress_ctx comp_ctx;
+                FILE *comp_file_gz = NULL;
+                FILE *comp_file_br = NULL;
                 if (compress) {
                     sprintf(buf, "%.*s/.necronda-server", cache[i].webroot_len, cache[i].filename);
                     mkdir(buf, 0755);
@@ -110,53 +112,70 @@ int cache_process() {
                         buf[j] = ch;
                     }
                     buf[strlen(rel_path)] = 0;
-                    p_len = snprintf(filename_comp, sizeof(filename_comp), "%.*s/.necronda-server/cache/%s.z",
-                                     cache[i].webroot_len, cache[i].filename, buf);
-                    if (p_len < 0 || p_len >= sizeof(filename_comp)) {
+
+                    p_len_gz = snprintf(filename_comp_gz, sizeof(filename_comp_gz),
+                                        "%.*s/.necronda-server/cache/%s.gz",
+                                        cache[i].webroot_len, cache[i].filename, buf);
+                    p_len_br = snprintf(filename_comp_br, sizeof(filename_comp_br),
+                                        "%.*s/.necronda-server/cache/%s.br",
+                                        cache[i].webroot_len, cache[i].filename, buf);
+                    if (p_len_gz < 0 || p_len_gz >= sizeof(filename_comp_gz) ||
+                            p_len_br < 0 || p_len_br >= sizeof(filename_comp_br)) {
                         fprintf(stderr, ERR_STR "Unable to open cached file: "
                                                 "File name for compressed file too long" CLR_STR "\n");
                         goto comp_err;
                     }
+
                     fprintf(stdout, "[cache] Compressing file %s\n", cache[i].filename);
-                    comp_file = fopen(filename_comp, "wb");
-                    if (comp_file == NULL) {
+
+                    comp_file_gz = fopen(filename_comp_gz, "wb");
+                    comp_file_br = fopen(filename_comp_br, "wb");
+                    if (comp_file_gz == NULL || comp_file_br == NULL) {
                         fprintf(stderr, ERR_STR "Unable to open cached file: %s" CLR_STR "\n", strerror(errno));
                         comp_err:
                         compress = 0;
                     } else {
-                        strm.zalloc = Z_NULL;
-                        strm.zfree = Z_NULL;
-                        strm.opaque = Z_NULL;
-                        if (deflateInit(&strm, level) != Z_OK) {
-                            fprintf(stderr, ERR_STR "Unable to init deflate: %s" CLR_STR "\n", strerror(errno));
+                        ret = compress_init(&comp_ctx, COMPRESS_GZ | COMPRESS_BR);
+                        if (ret != 0) {
+                            fprintf(stderr, ERR_STR "Unable to init compression: %s" CLR_STR "\n", strerror(errno));
                             compress = 0;
-                            fclose(comp_file);
+                            fclose(comp_file_gz);
+                            fclose(comp_file_br);
                         }
                     }
                 }
 
-                while ((read = fread(buf, 1, sizeof(buf), file)) > 0) {
+                while ((read = fread(buf, 1, CACHE_BUF_SIZE, file)) > 0) {
                     SHA1_Update(&ctx, buf, read);
                     if (compress) {
-                        strm.avail_in = read;
-                        strm.next_in = (unsigned char *) buf;
+                        unsigned long avail_in, avail_out;
+                        avail_in = read;
                         do {
-                            strm.avail_out = sizeof(comp_buf);
-                            strm.next_out = (unsigned char *) comp_buf;
-                            deflate(&strm, feof(file) ? Z_FINISH : Z_NO_FLUSH);
-                            fwrite(comp_buf, 1, sizeof(comp_buf) - strm.avail_out, comp_file);
-                            strm.avail_in = 0;
-                        } while (strm.avail_out == 0);
+                            avail_out = CACHE_BUF_SIZE;
+                            compress_compress_mode(&comp_ctx, COMPRESS_GZ,buf + read - avail_in, &avail_in,
+                                                   comp_buf, &avail_out, feof(file));
+                            fwrite(comp_buf, 1, CACHE_BUF_SIZE - avail_out, comp_file_gz);
+                        } while (avail_in != 0 || avail_out != CACHE_BUF_SIZE);
+                        avail_in = read;
+                        do {
+                            avail_out = CACHE_BUF_SIZE;
+                            compress_compress_mode(&comp_ctx, COMPRESS_BR, buf + read - avail_in, &avail_in,
+                                                   comp_buf, &avail_out, feof(file));
+                            fwrite(comp_buf, 1, CACHE_BUF_SIZE - avail_out, comp_file_br);
+                        } while (avail_in != 0 || avail_out != CACHE_BUF_SIZE);
                     }
                 }
 
                 if (compress) {
-                    deflateEnd(&strm);
-                    fclose(comp_file);
+                    compress_free(&comp_ctx);
+                    fclose(comp_file_gz);
+                    fclose(comp_file_br);
                     fprintf(stdout, "[cache] Finished compressing file %s\n", cache[i].filename);
-                    strcpy(cache[i].meta.filename_comp, filename_comp);
+                    strcpy(cache[i].meta.filename_comp_gz, filename_comp_gz);
+                    strcpy(cache[i].meta.filename_comp_br, filename_comp_br);
                 } else {
-                    memset(cache[i].meta.filename_comp, 0, sizeof(cache[i].meta.filename_comp));
+                    memset(cache[i].meta.filename_comp_gz, 0, sizeof(cache[i].meta.filename_comp_gz));
+                    memset(cache[i].meta.filename_comp_br, 0, sizeof(cache[i].meta.filename_comp_br));
                 }
                 SHA1_Final(hash, &ctx);
                 memset(cache[i].meta.etag, 0, sizeof(cache[i].meta.etag));
@@ -173,12 +192,20 @@ int cache_process() {
         if (cache_changed) {
             cache_changed = 0;
             cache_file = fopen("/var/necronda-server/cache", "wb");
-            fwrite(cache, sizeof(cache_entry), FILE_CACHE_SIZE, cache_file);
+            if (cache_file == NULL) {
+                fprintf(stderr, ERR_STR "Unable to open cache file: %s" CLR_STR "\n", strerror(errno));
+                free(buf);
+                free(comp_buf);
+                return -1;
+            }
+            fwrite(cache, sizeof(cache_entry), CACHE_ENTRIES, cache_file);
             fclose(cache_file);
         } else {
             sleep(1);
         }
     }
+    free(buf);
+    free(comp_buf);
     return 0;
 }
 
@@ -187,7 +214,7 @@ int cache_init() {
         return -1;
     }
 
-    int shm_id = shmget(SHM_KEY_CACHE, FILE_CACHE_SIZE * sizeof(cache_entry), IPC_CREAT | IPC_EXCL | 0600);
+    int shm_id = shmget(CACHE_SHM_KEY, CACHE_ENTRIES * sizeof(cache_entry), IPC_CREAT | IPC_EXCL | 0600);
     if (shm_id < 0) {
         fprintf(stderr, ERR_STR "Unable to create shared memory: %s" CLR_STR "\n", strerror(errno));
         return -2;
@@ -206,7 +233,7 @@ int cache_init() {
         return -4;
     }
     cache = shm_rw;
-    memset(cache, 0, FILE_CACHE_SIZE * sizeof(cache_entry));
+    memset(cache, 0, CACHE_ENTRIES * sizeof(cache_entry));
     shmdt(shm_rw);
     cache = shm;
 
@@ -229,7 +256,7 @@ int cache_init() {
 }
 
 int cache_unload() {
-    int shm_id = shmget(SHM_KEY_CACHE, 0, 0);
+    int shm_id = shmget(CACHE_SHM_KEY, 0, 0);
     if (shm_id < 0) {
         fprintf(stderr, ERR_STR "Unable to get shared memory id: %s" CLR_STR "\n", strerror(errno));
         shmdt(cache);
@@ -245,7 +272,7 @@ int cache_unload() {
 
 int cache_update_entry(int entry_num, const char *filename, const char *webroot) {
     void *cache_ro = cache;
-    int shm_id = shmget(SHM_KEY_CACHE, 0, 0);
+    int shm_id = shmget(CACHE_SHM_KEY, 0, 0);
     void *shm_rw = shmat(shm_id, NULL, 0);
     if (shm_rw == (void *) -1) {
         print(ERR_STR "Unable to attach shared memory (rw): %s" CLR_STR, strerror(errno));
@@ -277,7 +304,8 @@ int cache_update_entry(int entry_num, const char *filename, const char *webroot)
     strcpy(cache[entry_num].meta.charset, magic_file(magic, filename));
 
     memset(cache[entry_num].meta.etag, 0, sizeof(cache[entry_num].meta.etag));
-    memset(cache[entry_num].meta.filename_comp, 0, sizeof(cache[entry_num].meta.filename_comp));
+    memset(cache[entry_num].meta.filename_comp_gz, 0, sizeof(cache[entry_num].meta.filename_comp_gz));
+    memset(cache[entry_num].meta.filename_comp_br, 0, sizeof(cache[entry_num].meta.filename_comp_br));
     cache[entry_num].is_updating = 0;
 
     shmdt(shm_rw);
@@ -287,7 +315,7 @@ int cache_update_entry(int entry_num, const char *filename, const char *webroot)
 
 int cache_filename_comp_invalid(const char *filename) {
     void *cache_ro = cache;
-    int shm_id = shmget(SHM_KEY_CACHE, 0, 0);
+    int shm_id = shmget(CACHE_SHM_KEY, 0, 0);
     void *shm_rw = shmat(shm_id, NULL, 0);
     if (shm_rw == (void *) -1) {
         print(ERR_STR "Unable to attach shared memory (rw): %s" CLR_STR, strerror(errno));
@@ -296,7 +324,7 @@ int cache_filename_comp_invalid(const char *filename) {
     cache = shm_rw;
 
     int i;
-    for (i = 0; i < FILE_CACHE_SIZE; i++) {
+    for (i = 0; i < CACHE_ENTRIES; i++) {
         if (cache[i].filename[0] != 0 && strlen(cache[i].filename) == strlen(filename) &&
                 strcmp(cache[i].filename, filename) == 0) {
             if (cache[i].is_updating) {
@@ -308,7 +336,8 @@ int cache_filename_comp_invalid(const char *filename) {
     }
 
     memset(cache[i].meta.etag, 0, sizeof(cache[i].meta.etag));
-    memset(cache[i].meta.filename_comp, 0, sizeof(cache[i].meta.filename_comp));
+    memset(cache[i].meta.filename_comp_gz, 0, sizeof(cache[i].meta.filename_comp_gz));
+    memset(cache[i].meta.filename_comp_br, 0, sizeof(cache[i].meta.filename_comp_br));
     cache[i].is_updating = 0;
 
     shmdt(shm_rw);
@@ -322,7 +351,7 @@ int uri_cache_init(http_uri *uri) {
     }
 
     int i;
-    for (i = 0; i < FILE_CACHE_SIZE; i++) {
+    for (i = 0; i < CACHE_ENTRIES; i++) {
         if (cache[i].filename[0] != 0 && strlen(cache[i].filename) == strlen(uri->filename) &&
                 strcmp(cache[i].filename, uri->filename) == 0) {
             uri->meta = &cache[i].meta;
@@ -335,7 +364,7 @@ int uri_cache_init(http_uri *uri) {
     }
 
     if (uri->meta == NULL) {
-        for (i = 0; i < FILE_CACHE_SIZE; i++) {
+        for (i = 0; i < CACHE_ENTRIES; i++) {
             if (cache[i].filename[0] == 0) {
                 if (cache_update_entry(i, uri->filename, uri->webroot) != 0) {
                     return -1;

src/lib/cache.h

@@ -10,6 +10,19 @@
 
 #include "uri.h"
 
+#define CACHE_SHM_KEY 255641
+#define CACHE_ENTRIES 1024
+#define CACHE_BUF_SIZE 16384
+
+#ifndef CACHE_MAGIC_FILE
+#   define CACHE_MAGIC_FILE "/usr/share/file/misc/magic.mgc"
+#endif
+
+#ifndef DEFAULT_CONFIG_FILE
+#   define DEFAULT_CONFIG_FILE "/etc/necronda-server/necronda-server.conf"
+#endif
+
+
 typedef struct {
     char filename[256];
     unsigned char webroot_len;

src/lib/compress.c

@@ -0,0 +1,79 @@
+/**
+ * Necronda Web Server
+ * Compression interface
+ * src/lib/compress.c
+ * Lorenz Stechauner, 2021-05-05
+ */
+
+#include "compress.h"
+#include <malloc.h>
+#include <errno.h>
+
+int compress_init(compress_ctx *ctx, int mode) {
+    ctx->gzip = NULL;
+    ctx->brotli = NULL;
+    ctx->mode = 0;
+    int ret;
+    if (mode & COMPRESS_GZ) {
+        ctx->mode |= COMPRESS_GZ;
+        ctx->gzip = malloc(sizeof(z_stream));
+        ctx->gzip->zalloc = Z_NULL;
+        ctx->gzip->zfree = Z_NULL;
+        ctx->gzip->opaque = Z_NULL;
+        ret = deflateInit2(ctx->gzip, COMPRESS_LEVEL_GZIP, Z_DEFLATED, 15 + 16, 9, Z_DEFAULT_STRATEGY);
+        if (ret != Z_OK) return -1;
+    }
+    if (mode & COMPRESS_BR) {
+        ctx->mode |= COMPRESS_BR;
+        ctx->brotli = BrotliEncoderCreateInstance(NULL, NULL, NULL);
+        if (ctx->brotli == NULL) return -1;
+        BrotliEncoderSetParameter(ctx->brotli, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC);
+        BrotliEncoderSetParameter(ctx->brotli, BROTLI_PARAM_QUALITY, COMPRESS_LEVEL_BROTLI);
+    }
+    return 0;
+}
+
+int compress_compress(compress_ctx *ctx, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
+    if ((ctx->mode & COMPRESS_GZ) && (ctx->mode & COMPRESS_BR)) {
+        errno = EINVAL;
+        return -1;
+    }
+    return compress_compress_mode(ctx, ctx->mode, in, in_len, out, out_len, finish);
+}
+
+int compress_compress_mode(compress_ctx *ctx, int mode, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) {
+    if ((mode & COMPRESS_GZ) && (mode & COMPRESS_BR)) {
+        errno = EINVAL;
+        return -1;
+    } else if (mode & COMPRESS_GZ) {
+        ctx->gzip->next_in = (unsigned char*) in;
+        ctx->gzip->avail_in = *in_len;
+        ctx->gzip->next_out = (unsigned char*) out;
+        ctx->gzip->avail_out = *out_len;
+        int ret = deflate(ctx->gzip, finish ? Z_FINISH : Z_NO_FLUSH);
+        *in_len = ctx->gzip->avail_in;
+        *out_len = ctx->gzip->avail_out;
+        return ret;
+    } else if (mode & COMPRESS_BR) {
+        int ret = BrotliEncoderCompressStream(ctx->brotli, finish ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS,
+                                              in_len, (const unsigned char**) &in, out_len, (unsigned char **) &out, NULL);
+        return (ret == BROTLI_TRUE) ? 0 : -1;
+    } else {
+        errno = EINVAL;
+        return -2;
+    }
+}
+
+int compress_free(compress_ctx *ctx) {
+    if (ctx->gzip != NULL) {
+        deflateEnd(ctx->gzip);
+        free(ctx->gzip);
+        ctx->gzip = NULL;
+    }
+    if (ctx->brotli != NULL) {
+        BrotliEncoderDestroyInstance(ctx->brotli);
+        ctx->brotli = NULL;
+    }
+    ctx->mode = 0;
+    return 0;
+}

src/lib/compress.h

@@ -0,0 +1,36 @@
+/**
+ * Necronda Web Server
+ * Compression interface (header file)
+ * src/lib/compress.h
+ * Lorenz Stechauner, 2021-05-05
+ */
+
+#ifndef NECRONDA_SERVER_COMPRESS_H
+#define NECRONDA_SERVER_COMPRESS_H
+
+#include <zlib.h>
+#include <brotli/encode.h>
+
+#define COMPRESS_LEVEL_GZIP 9
+#define COMPRESS_LEVEL_BROTLI BROTLI_MAX_QUALITY
+
+#define COMPRESS_GZ 1
+#define COMPRESS_BR 2
+
+typedef struct {
+    int mode;
+    z_stream *gzip;
+    BrotliEncoderState *brotli;
+} compress_ctx;
+
+int compress_init(compress_ctx *ctx, int mode);
+
+int compress_compress(compress_ctx *ctx, const char *in, unsigned long *in_len, char *out, unsigned long *out_len,
+                      int finish);
+
+int compress_compress_mode(compress_ctx *ctx, int mode, const char *in, unsigned long *in_len, char *out,
+                           unsigned long *out_len, int finish);
+
+int compress_free(compress_ctx *ctx);
+
+#endif //NECRONDA_SERVER_COMPRESS_H

src/lib/config.c

@@ -6,20 +6,19 @@
  */
 
 #include "config.h"
-#include "../necronda-server.h"
+#include "utils.h"
 #include <stdio.h>
 #include <sys/ipc.h>
 #include <sys/shm.h>
 #include <string.h>
 #include <errno.h>
-#include <malloc.h>
 #include <stdlib.h>
 
 host_config *config;
 char cert_file[256], key_file[256], geoip_dir[256], dns_server[256];
 
 int config_init() {
-    int shm_id = shmget(SHM_KEY_CONFIG, MAX_HOST_CONFIG * sizeof(host_config), IPC_CREAT | IPC_EXCL | 0640);
+    int shm_id = shmget(CONFIG_SHM_KEY, CONFIG_MAX_HOST_CONFIG * sizeof(host_config), IPC_CREAT | IPC_EXCL | 0640);
     if (shm_id < 0) {
         fprintf(stderr, ERR_STR "Unable to create shared memory: %s" CLR_STR "\n", strerror(errno));
         return -1;
@@ -38,14 +37,14 @@ int config_init() {
         return -3;
     }
     config = shm_rw;
-    memset(config, 0, MAX_HOST_CONFIG * sizeof(host_config));
+    memset(config, 0, CONFIG_MAX_HOST_CONFIG * sizeof(host_config));
     shmdt(shm_rw);
     config = shm;
     return 0;
 }
 
 int config_unload() {
-    int shm_id = shmget(SHM_KEY_CONFIG, 0, 0);
+    int shm_id = shmget(CONFIG_SHM_KEY, 0, 0);
     if (shm_id < 0) {
         fprintf(stderr, ERR_STR "Unable to get shared memory id: %s" CLR_STR "\n", strerror(errno));
         shmdt(config);
@@ -73,8 +72,8 @@ int config_load(const char *filename) {
     fread(conf, 1, len, file);
     fclose(file);
 
-    host_config *tmp_config = malloc(MAX_HOST_CONFIG * sizeof(host_config));
-    memset(tmp_config, 0, MAX_HOST_CONFIG * sizeof(host_config));
+    host_config *tmp_config = malloc(CONFIG_MAX_HOST_CONFIG * sizeof(host_config));
+    memset(tmp_config, 0, CONFIG_MAX_HOST_CONFIG * sizeof(host_config));
 
     int i = 0;
     int mode = 0;
@@ -195,7 +194,7 @@ int config_load(const char *filename) {
         }
     }
 
-    int shm_id = shmget(SHM_KEY_CONFIG, 0, 0);
+    int shm_id = shmget(CONFIG_SHM_KEY, 0, 0);
     if (shm_id < 0) {
         fprintf(stderr, ERR_STR "Unable to get shared memory id: %s" CLR_STR "\n", strerror(errno));
         shmdt(config);
@@ -208,7 +207,7 @@ int config_load(const char *filename) {
         fprintf(stderr, ERR_STR "Unable to attach shared memory (rw): %s" CLR_STR "\n", strerror(errno));
         return -4;
     }
-    memcpy(shm_rw, tmp_config, MAX_HOST_CONFIG * sizeof(host_config));
+    memcpy(shm_rw, tmp_config, CONFIG_MAX_HOST_CONFIG * sizeof(host_config));
     free(tmp_config);
     shmdt(shm_rw);
     return 0;

src/lib/config.h

@@ -10,6 +10,9 @@
 
 #include "uri.h"
 
+#define CONFIG_SHM_KEY 255642
+#define CONFIG_MAX_HOST_CONFIG 64
+
 #define CONFIG_TYPE_UNSET 0
 #define CONFIG_TYPE_LOCAL 1
 #define CONFIG_TYPE_REVERSE_PROXY 2

src/lib/fastcgi.c

@@ -7,10 +7,9 @@
 
 #include "fastcgi.h"
 #include "utils.h"
-#include "../client.h"
+#include "compress.h"
 #include "../necronda-server.h"
 #include <sys/un.h>
-#include <zlib.h>
 #include <sys/socket.h>
 #include <errno.h>
 #include <string.h>
@@ -385,7 +384,7 @@ int fastcgi_header(fastcgi_conn *conn, http_res *res, char *err_msg) {
 
 int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
     FCGI_Header header;
-    int ret;
+    long ret;
     char buf0[256];
     int len;
     char *content, *ptr;
@@ -393,15 +392,18 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
     char comp_out[4096];
     int finish_comp = 0;
 
-    z_stream strm;
-    if (flags & FASTCGI_COMPRESS) {
-        int level = NECRONDA_ZLIB_LEVEL;
-        strm.zalloc = Z_NULL;
-        strm.zfree = Z_NULL;
-        strm.opaque = Z_NULL;
-        if (deflateInit(&strm, level) != Z_OK) {
-            print(ERR_STR "Unable to init deflate: %s" CLR_STR, strerror(errno));
-            flags &= !FASTCGI_COMPRESS;
+    compress_ctx comp_ctx;
+    if (flags & FASTCGI_COMPRESS_BR) {
+        flags &= ~FASTCGI_COMPRESS_GZ;
+        if (compress_init(&comp_ctx, COMPRESS_BR) != 0) {
+            print(ERR_STR "Unable to init brotli: %s" CLR_STR, strerror(errno));
+            flags &= ~FASTCGI_COMPRESS_BR;
+        }
+    } else if (flags & FASTCGI_COMPRESS_GZ) {
+        flags &= ~FASTCGI_COMPRESS_BR;
+        if (compress_init(&comp_ctx, COMPRESS_GZ) != 0) {
+            print(ERR_STR "Unable to init gzip: %s" CLR_STR, strerror(errno));
+            flags &= ~FASTCGI_COMPRESS_GZ;
         }
     }
 
@@ -455,7 +457,7 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
                 content_len = 0;
                 goto out;
                 finish:
-                deflateEnd(&strm);
+                compress_free(&comp_ctx);
             }
 
             if (flags & FASTCGI_CHUNKED) {
@@ -466,20 +468,18 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
         } else if (header.type == FCGI_STDERR) {
             fastcgi_php_error(content, content_len, buf0);
         } else if (header.type == FCGI_STDOUT) {
+            unsigned long avail_in, avail_out;
             out:
-            if (flags & FASTCGI_COMPRESS) {
-                strm.avail_in = content_len;
-                strm.next_in = (unsigned char *) ptr;
-            }
+            avail_in = content_len;
+            void *next_in = ptr;
             do {
                 int buf_len = content_len;
                 if (flags & FASTCGI_COMPRESS) {
-                    strm.avail_out = sizeof(comp_out);
-                    strm.next_out = (unsigned char *) comp_out;
-                    deflate(&strm, finish_comp ? Z_FINISH : Z_NO_FLUSH);
-                    strm.avail_in = 0;
+                    avail_out = sizeof(comp_out);
+                    compress_compress(&comp_ctx, next_in + content_len - avail_in, &avail_in, comp_out, &avail_out,
+                                      finish_comp);
                     ptr = comp_out;
-                    buf_len = (int) (sizeof(comp_out) - strm.avail_out);
+                    buf_len = (int) (sizeof(comp_out) - avail_out);
                 }
                 if (buf_len != 0) {
                     len = sprintf(buf0, "%X\r\n", buf_len);
@@ -487,7 +487,7 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
                     sock_send(client, ptr, buf_len, 0);
                     if (flags & FASTCGI_CHUNKED) sock_send(client, "\r\n", 2, 0);
                 }
-            } while ((flags & FASTCGI_COMPRESS) && strm.avail_out == 0);
+            } while ((flags & FASTCGI_COMPRESS) && (avail_in != 0 || avail_out != sizeof(comp_out)));
             if (finish_comp) goto finish;
         } else {
             print(ERR_STR "Unknown FastCGI type: %i" CLR_STR, header.type);

src/lib/fastcgi.h

@@ -13,7 +13,13 @@
 #include "uri.h"
 
 #define FASTCGI_CHUNKED 1
-#define FASTCGI_COMPRESS 2
+#define FASTCGI_COMPRESS_GZ 2
+#define FASTCGI_COMPRESS_BR 4
+#define FASTCGI_COMPRESS 6
+
+#ifndef PHP_FPM_SOCKET
+#   define PHP_FPM_SOCKET "/var/run/php-fpm/php-fpm.sock"
+#endif
 
 typedef struct {
     int socket;

src/lib/geoip.c

@@ -0,0 +1,69 @@
+/**
+ * Necronda Web Server
+ * MaxMind GeoIP Database interface
+ * src/lib/geoip.c
+ * Lorenz Stechauner, 2021-05-04
+ */
+
+#include "geoip.h"
+
+MMDB_entry_data_list_s *mmdb_json(MMDB_entry_data_list_s *list, char *str, long *str_off, long str_len) {
+    switch (list->entry_data.type) {
+        case MMDB_DATA_TYPE_MAP:
+            *str_off += sprintf(str + *str_off, "{");
+            break;
+        case MMDB_DATA_TYPE_ARRAY:
+            *str_off += sprintf(str + *str_off, "[");
+            break;
+        case MMDB_DATA_TYPE_UTF8_STRING:
+            *str_off += sprintf(str + *str_off, "\"%.*s\"", list->entry_data.data_size, list->entry_data.utf8_string);
+            break;
+        case MMDB_DATA_TYPE_UINT16:
+            *str_off += sprintf(str + *str_off, "%u", list->entry_data.uint16);
+            break;
+        case MMDB_DATA_TYPE_UINT32:
+            *str_off += sprintf(str + *str_off, "%u", list->entry_data.uint32);
+            break;
+        case MMDB_DATA_TYPE_UINT64:
+            *str_off += sprintf(str + *str_off, "%lu", list->entry_data.uint64);
+            break;
+        case MMDB_DATA_TYPE_UINT128:
+            *str_off += sprintf(str + *str_off, "%llu", (unsigned long long) list->entry_data.uint128);
+            break;
+        case MMDB_DATA_TYPE_INT32:
+            *str_off += sprintf(str + *str_off, "%i", list->entry_data.uint32);
+            break;
+        case MMDB_DATA_TYPE_BOOLEAN:
+            *str_off += sprintf(str + *str_off, "%s", list->entry_data.boolean ? "true" : "false");
+            break;
+        case MMDB_DATA_TYPE_FLOAT:
+            *str_off += sprintf(str + *str_off, "%f", list->entry_data.float_value);
+            break;
+        case MMDB_DATA_TYPE_DOUBLE:
+            *str_off += sprintf(str + *str_off, "%f", list->entry_data.double_value);
+            break;
+    }
+    if (list->entry_data.type != MMDB_DATA_TYPE_MAP && list->entry_data.type != MMDB_DATA_TYPE_ARRAY) {
+        return list->next;
+    }
+    MMDB_entry_data_list_s *next = list->next;
+    int stat = 0;
+    for (int i = 0; i < list->entry_data.data_size; i++) {
+        next = mmdb_json(next, str, str_off, str_len);
+        if (list->entry_data.type == MMDB_DATA_TYPE_MAP) {
+            stat = !stat;
+            if (stat) {
+                i--;
+                *str_off += sprintf(str + *str_off, ":");
+                continue;
+            }
+        }
+        if (i != list->entry_data.data_size - 1) *str_off += sprintf(str + *str_off, ",");
+    }
+    if (list->entry_data.type == MMDB_DATA_TYPE_MAP) {
+        *str_off += sprintf(str + *str_off, "}");
+    } else {
+        *str_off += sprintf(str + *str_off, "]");
+    }
+    return next;
+}

src/lib/geoip.h

@@ -0,0 +1,18 @@
+/**
+ * Necronda Web Server
+ * MaxMind GeoIP Database interface (header file)
+ * src/lib/geoip.h
+ * Lorenz Stechauner, 2021-05-04
+ */
+
+
+#ifndef NECRONDA_SERVER_GEOIP_H
+#define NECRONDA_SERVER_GEOIP_H
+
+#include <maxminddb.h>
+
+#define GEOIP_MAX_SIZE 8192
+
+MMDB_entry_data_list_s *mmdb_json(MMDB_entry_data_list_s *list, char *str, long *str_off, long str_len);
+
+#endif //NECRONDA_SERVER_GEOIP_H

src/lib/http.c

@@ -7,6 +7,7 @@
 
 #include "http.h"
 #include "utils.h"
+#include "compress.h"
 #include "../necronda-server.h"
 #include <string.h>
 
@@ -312,3 +313,17 @@ const http_doc_info *http_get_status_info(const http_status *status) {
     }
     return NULL;
 }
+
+int http_get_compression(const http_req *req, const http_res *res) {
+    char *accept_encoding = http_get_header_field(&req->hdr, "Accept-Encoding");
+    char *content_type = http_get_header_field(&res->hdr, "Content-Type");
+    char *content_encoding = http_get_header_field(&res->hdr, "Content-Encoding");
+    if (mime_is_compressible(content_type) && content_encoding == NULL && accept_encoding != NULL) {
+        if (strstr(accept_encoding, "br") != NULL) {
+            return COMPRESS_BR;
+        } else if (strstr(accept_encoding, "gzip") != NULL) {
+            return COMPRESS_GZ;
+        }
+    }
+    return 0;
+}

src/lib/http.h

@@ -18,11 +18,25 @@
 #define HTTP_REMOVE_ALL 1
 #define HTTP_REMOVE_LAST 2
 
+#define HTTP_1XX_STR "\x1B[1;32m"
+#define HTTP_2XX_STR "\x1B[1;32m"
+#define HTTP_3XX_STR "\x1B[1;33m"
+#define HTTP_4XX_STR "\x1B[1;31m"
+#define HTTP_5XX_STR "\x1B[1;31m"
+
 #define HTTP_COLOR_SUCCESS "#008000"
 #define HTTP_COLOR_INFO "#606060"
 #define HTTP_COLOR_WARNING "#E0C000"
 #define HTTP_COLOR_ERROR "#C00000"
 
+#ifndef SERVER_STR
+#   define SERVER_STR "Necronda"
+#endif
+
+#ifndef SERVER_STR_HTML
+#   define SERVER_STR_HTML "Necronda web server"
+#endif
+
 typedef struct {
     unsigned short code;
     char type[16];
@@ -108,4 +122,6 @@ char *http_get_date(char *buf, size_t size);
 
 const http_doc_info *http_get_status_info(const http_status *status);
 
+int http_get_compression(const http_req *req, const http_res *res);
+
 #endif //NECRONDA_SERVER_HTTP_H

src/lib/http_static.c

@@ -6,7 +6,7 @@
  */
 
 #include "http.h"
-#include "../necronda-server.h"
+#include "utils.h"
 
 const http_status http_statuses[] = {
         {100, "Informational", "Continue"},
@@ -137,7 +137,7 @@ const char http_default_document[] =
         "\t<main>\n"
         "\t\t<section>\n"
         "%3$s"
-        "\t\t\t<div class=\"footer\"><a href=\"https://%7$s/\">%7$s</a> - Necronda&nbsp;web&nbsp;server&nbsp;" NECRONDA_VERSION "</div>\n"
+        "\t\t\t<div class=\"footer\"><a href=\"https://%7$s/\">%7$s</a> - " SERVER_STR_HTML "</div>\n"
         "\t\t</section>\n"
         "\t</main>\n"
         "</body>\n"

src/lib/rev_proxy.c

@@ -7,7 +7,7 @@
 
 #include "rev_proxy.h"
 #include "utils.h"
-#include "../client.h"
+#include "compress.h"
 #include "../necronda-server.h"
 #include <openssl/ssl.h>
 #include <string.h>
@@ -378,13 +378,33 @@ int rev_proxy_init(http_req *req, http_res *res, host_config *conf, sock *client
     return -1;
 }
 
-int rev_proxy_send(sock *client, int chunked, unsigned long len_to_send) {
+int rev_proxy_send(sock *client, unsigned long len_to_send, int flags) {
+    // TODO handle websockets
     long ret;
     char buffer[CHUNK_SIZE];
+    char comp_out[CHUNK_SIZE];
+    char buf[256];
     long len, snd_len;
-    // TODO handle websockets
+    int finish_comp = 0;
+    char *ptr;
+
+    compress_ctx comp_ctx;
+    if (flags & REV_PROXY_COMPRESS_BR) {
+        flags &= ~REV_PROXY_COMPRESS_GZ;
+        if (compress_init(&comp_ctx, COMPRESS_BR) != 0) {
+            print(ERR_STR "Unable to init brotli: %s" CLR_STR, strerror(errno));
+            flags &= ~REV_PROXY_COMPRESS_BR;
+        }
+    } else if (flags & REV_PROXY_COMPRESS_GZ) {
+        flags &= ~REV_PROXY_COMPRESS_BR;
+        if (compress_init(&comp_ctx, COMPRESS_GZ) != 0) {
+            print(ERR_STR "Unable to init gzip: %s" CLR_STR, strerror(errno));
+            flags &= ~REV_PROXY_COMPRESS_GZ;
+        }
+    }
+
     do {
-        if (chunked) {
+        if (flags & REV_PROXY_CHUNKED) {
             ret = sock_recv(&rev_proxy, buffer, 16, MSG_PEEK);
             if (ret <= 0) {
                 print("Unable to receive: %s", sock_strerror(&rev_proxy));
@@ -394,30 +414,61 @@ int rev_proxy_send(sock *client, int chunked, unsigned long len_to_send) {
             len_to_send = strtol(buffer, NULL, 16);
             char *pos = strstr(buffer, "\r\n");
             len = pos - buffer + 2;
-            ret = sock_send(client, buffer, len, 0);
-
             sock_recv(&rev_proxy, buffer, len, 0);
             if (ret <= 0) break;
+
+            if (len_to_send == 0 && (flags & REV_PROXY_COMPRESS)) {
+                finish_comp = 1;
+                len = 0;
+                goto out;
+                finish:
+                compress_free(&comp_ctx);
+            }
         }
         snd_len = 0;
         while (snd_len < len_to_send) {
+            unsigned long avail_in, avail_out;
             len = sock_recv(&rev_proxy, buffer, CHUNK_SIZE < (len_to_send - snd_len) ? CHUNK_SIZE : len_to_send - snd_len, 0);
-            ret = sock_send(client, buffer, len, 0);
-            if (ret <= 0) {
-                print(ERR_STR "Unable to send: %s" CLR_STR, sock_strerror(client));
-                break;
-            }
-            snd_len += ret;
+            ptr = buffer;
+            out:
+            avail_in = len;
+            void *next_in = ptr;
+            do {
+                long buf_len = len;
+                if (flags & REV_PROXY_COMPRESS) {
+                    avail_out = sizeof(comp_out);
+                    compress_compress(&comp_ctx, next_in + len - avail_in, &avail_in, comp_out, &avail_out,
+                                      finish_comp);
+                    ptr = comp_out;
+                    buf_len = (int) (sizeof(comp_out) - avail_out);
+                    snd_len += (long) (len - avail_in);
+                }
+                if (buf_len != 0) {
+                    len = sprintf(buf, "%lX\r\n", buf_len);
+                    ret = 1;
+                    if (flags & REV_PROXY_CHUNKED) ret = sock_send(client, buf, len, 0);
+                    if (ret <= 0) goto err;
+                    ret = sock_send(client, ptr, buf_len, 0);
+                    if (ret <= 0) goto err;
+                    if (!(flags & REV_PROXY_COMPRESS)) snd_len += ret;
+                    if (flags & REV_PROXY_CHUNKED) ret = sock_send(client, "\r\n", 2, 0);
+                    if (ret <= 0) {
+                        err:
+                        print(ERR_STR "Unable to send: %s" CLR_STR, sock_strerror(client));
+                        break;
+                    }
+                }
+            } while ((flags & REV_PROXY_COMPRESS) && (avail_in != 0 || avail_out != sizeof(comp_out)));
+            if (ret <= 0) break;
+            if (finish_comp) goto finish;
         }
         if (ret <= 0) break;
-        if (chunked) {
-            sock_recv(&rev_proxy, buffer, 2, 0);
-            ret = sock_send(client, "\r\n", 2, 0);
-            if (ret <= 0) {
-                print(ERR_STR "Unable to send: %s" CLR_STR, sock_strerror(client));
-                break;
-            }
-        }
-    } while (chunked && len_to_send > 0);
+        if (flags & REV_PROXY_CHUNKED) sock_recv(&rev_proxy, buffer, 2, 0);
+    } while ((flags & REV_PROXY_CHUNKED) && len_to_send > 0);
+
+    if (flags & REV_PROXY_CHUNKED) {
+        sock_send(client, "0\r\n\r\n", 5, 0);
+    }
+
     return 0;
 }

src/lib/rev_proxy.h

@@ -8,6 +8,11 @@
 #ifndef NECRONDA_SERVER_REV_PROXY_H
 #define NECRONDA_SERVER_REV_PROXY_H
 
+#define REV_PROXY_CHUNKED 1
+#define REV_PROXY_COMPRESS_GZ 2
+#define REV_PROXY_COMPRESS_BR 4
+#define REV_PROXY_COMPRESS 6
+
 #include "http.h"
 #include "config.h"
 
@@ -22,6 +27,6 @@ int rev_proxy_response_header(http_req *req, http_res *res);
 int rev_proxy_init(http_req *req, http_res *res, host_config *conf, sock *client, http_status *custom_status,
                    char *err_msg);
 
-int rev_proxy_send(sock *client, int chunked, unsigned long len_to_send);
+int rev_proxy_send(sock *client, unsigned long len_to_send, int flags);
 
 #endif //NECRONDA_SERVER_REV_PROXY_H

src/lib/uri.h

@@ -19,7 +19,8 @@ typedef struct {
     char etag[64];
     char type[24];
     char charset[16];
-    char filename_comp[256];
+    char filename_comp_gz[256];
+    char filename_comp_br[256];
     struct stat stat;
 } meta_data;
 

src/lib/utils.c

@@ -6,6 +6,7 @@
  */
 
 #include "utils.h"
+#include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 
@@ -26,7 +27,7 @@ char *format_duration(unsigned long micros, char *buf) {
     return buf;
 }
 
-int url_encode_component(const char *str, char *enc, ssize_t *size) {
+int url_encode_component(const char *str, char *enc, long *size) {
     char *ptr = enc;
     char ch;
     memset(enc, 0, *size);
@@ -53,7 +54,7 @@ int url_encode_component(const char *str, char *enc, ssize_t *size) {
     return 0;
 }
 
-int url_encode(const char *str, char *enc, ssize_t *size) {
+int url_encode(const char *str, char *enc, long *size) {
     char *ptr = enc;
     unsigned char ch;
     memset(enc, 0, *size);
@@ -76,7 +77,7 @@ int url_encode(const char *str, char *enc, ssize_t *size) {
     return 0;
 }
 
-int url_decode(const char *str, char *dec, ssize_t *size) {
+int url_decode(const char *str, char *dec, long *size) {
     char *ptr = dec;
     char ch, buf[3];
     memset(dec, 0, *size);
@@ -103,76 +104,30 @@ int url_decode(const char *str, char *dec, ssize_t *size) {
 }
 
 int mime_is_compressible(const char *type) {
+    if (type == NULL) return 0;
+    char type_parsed[64];
+    strncpy(type_parsed, type, sizeof(type_parsed));
+    char *pos = strchr(type_parsed, ';');
+    if (pos != NULL) pos[0] = 0;
     return
-        strncmp(type, "text/", 5) == 0 ||
-        strncmp(type, "message/", 7) == 0 ||
-        strstr(type, "+xml") != NULL ||
-        strcmp(type, "application/javascript") == 0 ||
-        strcmp(type, "application/json") == 0 ||
-        strcmp(type, "application/xml") == 0 ||
-        strcmp(type, "application/x-www-form-urlencoded") == 0 ||
-        strcmp(type, "application/x-tex") == 0 ||
-        strcmp(type, "application/x-httpd-php") == 0 ||
-        strcmp(type, "application/x-latex") == 0;
-}
-
-MMDB_entry_data_list_s *mmdb_json(MMDB_entry_data_list_s *list, char *str, long *str_off, long str_len) {
-    switch (list->entry_data.type) {
-        case MMDB_DATA_TYPE_MAP:
-            *str_off += sprintf(str + *str_off, "{");
-            break;
-        case MMDB_DATA_TYPE_ARRAY:
-            *str_off += sprintf(str + *str_off, "[");
-            break;
-        case MMDB_DATA_TYPE_UTF8_STRING:
-            *str_off += sprintf(str + *str_off, "\"%.*s\"", list->entry_data.data_size, list->entry_data.utf8_string);
-            break;
-        case MMDB_DATA_TYPE_UINT16:
-            *str_off += sprintf(str + *str_off, "%u", list->entry_data.uint16);
-            break;
-        case MMDB_DATA_TYPE_UINT32:
-            *str_off += sprintf(str + *str_off, "%u", list->entry_data.uint32);
-            break;
-        case MMDB_DATA_TYPE_UINT64:
-            *str_off += sprintf(str + *str_off, "%lu", list->entry_data.uint64);
-            break;
-        case MMDB_DATA_TYPE_UINT128:
-            *str_off += sprintf(str + *str_off, "%llu", (unsigned long long) list->entry_data.uint128);
-            break;
-        case MMDB_DATA_TYPE_INT32:
-            *str_off += sprintf(str + *str_off, "%i", list->entry_data.uint32);
-            break;
-        case MMDB_DATA_TYPE_BOOLEAN:
-            *str_off += sprintf(str + *str_off, "%s", list->entry_data.boolean ? "true" : "false");
-            break;
-        case MMDB_DATA_TYPE_FLOAT:
-            *str_off += sprintf(str + *str_off, "%f", list->entry_data.float_value);
-            break;
-        case MMDB_DATA_TYPE_DOUBLE:
-            *str_off += sprintf(str + *str_off, "%f", list->entry_data.double_value);
-            break;
-    }
-    if (list->entry_data.type != MMDB_DATA_TYPE_MAP && list->entry_data.type != MMDB_DATA_TYPE_ARRAY) {
-        return list->next;
-    }
-    MMDB_entry_data_list_s *next = list->next;
-    int stat = 0;
-    for (int i = 0; i < list->entry_data.data_size; i++) {
-        next = mmdb_json(next, str, str_off, str_len);
-        if (list->entry_data.type == MMDB_DATA_TYPE_MAP) {
-            stat = !stat;
-            if (stat) {
-                i--;
-                *str_off += sprintf(str + *str_off, ":");
-                continue;
-            }
-        }
-        if (i != list->entry_data.data_size - 1) *str_off += sprintf(str + *str_off, ",");
-    }
-    if (list->entry_data.type == MMDB_DATA_TYPE_MAP) {
-        *str_off += sprintf(str + *str_off, "}");
-    } else {
-        *str_off += sprintf(str + *str_off, "]");
-    }
-    return next;
+        strncmp(type_parsed, "text/", 5) == 0 ||
+        strncmp(type_parsed, "message/", 7) == 0 ||
+        strstr(type_parsed, "+xml") != NULL ||
+        strstr(type_parsed, "+json") != NULL ||
+        strcmp(type_parsed, "application/javascript") == 0 ||
+        strcmp(type_parsed, "application/json") == 0 ||
+        strcmp(type_parsed, "application/xml") == 0 ||
+        strcmp(type_parsed, "application/x-www-form-urlencoded") == 0 ||
+        strcmp(type_parsed, "application/x-tex") == 0 ||
+        strcmp(type_parsed, "application/x-httpd-php") == 0 ||
+        strcmp(type_parsed, "application/x-latex") == 0 ||
+        strcmp(type_parsed, "application/vnd.ms-fontobject") == 0 ||
+        strcmp(type_parsed, "application/x-font-ttf") == 0 ||
+        strcmp(type_parsed, "application/x-javascript") == 0 ||
+        strcmp(type_parsed, "application/x-web-app-manifest+json") == 0 ||
+        strcmp(type_parsed, "font/eot") == 0 ||
+        strcmp(type_parsed, "font/opentype") == 0 ||
+        strcmp(type_parsed, "image/bmp") == 0 ||
+        strcmp(type_parsed, "image/vnd.microsoft.icon") == 0 ||
+        strcmp(type_parsed, "image/x-icon") == 0;
 }

src/lib/utils.h

@@ -8,7 +8,15 @@
 #ifndef NECRONDA_SERVER_UTILS_H
 #define NECRONDA_SERVER_UTILS_H
 
-#include <maxminddb.h>
+#include <stdio.h>
+
+#define ERR_STR "\x1B[1;31m"
+#define CLR_STR "\x1B[0m"
+#define BLD_STR "\x1B[1m"
+#define WRN_STR "\x1B[1;33m"
+#define BLUE_STR "\x1B[34m"
+#define HTTP_STR "\x1B[1;31m"
+#define HTTPS_STR "\x1B[1;32m"
 
 extern char *log_prefix;
 
@@ -24,14 +32,12 @@ extern char *log_prefix;
 
 char *format_duration(unsigned long micros, char *buf);
 
-int url_encode_component(const char *str, char *enc, ssize_t *size);
+int url_encode_component(const char *str, char *enc, long *size);
 
-int url_encode(const char *str, char *enc, ssize_t *size);
+int url_encode(const char *str, char *enc, long *size);
 
-int url_decode(const char *str, char *dec, ssize_t *size);
+int url_decode(const char *str, char *dec, long *size);
 
 int mime_is_compressible(const char *type);
 
-MMDB_entry_data_list_s *mmdb_json(MMDB_entry_data_list_s *list, char *str, long *str_off, long str_len);
-
 #endif //NECRONDA_SERVER_UTILS_H

src/necronda-server.c

@@ -7,6 +7,7 @@
 
 #define _POSIX_C_SOURCE 199309L
 
+#include "necronda.h"
 #include "necronda-server.h"
 #include "client.c"
 
@@ -14,6 +15,7 @@
 #include "lib/config.h"
 #include "lib/sock.h"
 #include "lib/rev_proxy.h"
+#include "lib/geoip.h"
 
 #include <stdio.h>
 #include <sys/socket.h>
@@ -29,7 +31,6 @@
 #include <openssl/pem.h>
 #include <openssl/ssl.h>
 #include <openssl/conf.h>
-#include <maxminddb.h>
 #include <dirent.h>
 
 int active = 1;
@@ -274,6 +275,16 @@ int main(int argc, const char *argv[]) {
         closedir(geoip);
     }
 
+    ret = cache_init();
+    if (ret < 0) {
+        config_unload();
+        return 1;
+    } else if (ret != 0) {
+        children[0] = ret;  // pid
+    } else {
+        return 0;
+    }
+
     openssl_init();
 
     client.buf = NULL;
@@ -282,7 +293,7 @@ int main(int argc, const char *argv[]) {
     client.ctx = SSL_CTX_new(TLS_server_method());
     SSL_CTX_set_options(client.ctx, SSL_OP_SINGLE_DH_USE);
     SSL_CTX_set_verify(client.ctx, SSL_VERIFY_NONE, NULL);
-    SSL_CTX_set_min_proto_version(client.ctx, TLS1_VERSION);
+    SSL_CTX_set_min_proto_version(client.ctx, TLS1_2_VERSION);
     SSL_CTX_set_mode(client.ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
     SSL_CTX_set_cipher_list(client.ctx, "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4");
     SSL_CTX_set_ecdh_auto(client.ctx, 1);
@@ -318,16 +329,6 @@ int main(int argc, const char *argv[]) {
         }
     }
 
-    ret = cache_init();
-    if (ret < 0) {
-        config_unload();
-        return 1;
-    } else if (ret != 0) {
-        children[0] = ret;  // pid
-    } else {
-        return 0;
-    }
-
     fprintf(stderr, "Ready to accept connections\n");
 
     while (active) {
@@ -374,6 +375,7 @@ int main(int argc, const char *argv[]) {
             }
         }
 
+        // TODO outsource in thread
         int status = 0;
         for (int i = 0; i < MAX_CHILDREN; i++) {
             if (children[i] != 0) {

src/necronda-server.h

@@ -1,20 +1,19 @@
 /**
  * Necronda Web Server
  * Main executable (header file)
- * src/necronda-server.c
+ * src/necronda-server.h
  * Lorenz Stechauner, 2020-12-03
  */
 
 #ifndef NECRONDA_SERVER_NECRONDA_SERVER_H
 #define NECRONDA_SERVER_NECRONDA_SERVER_H
 
-#include <sys/types.h>
+#include <sys/time.h>
 #include <maxminddb.h>
 
 #define NUM_SOCKETS 2
 #define MAX_CHILDREN 1024
 #define MAX_MMDB 3
-#define MAX_HOST_CONFIG 64
 #define LISTEN_BACKLOG 16
 #define REQ_PER_CONNECTION 100
 #define CLIENT_TIMEOUT 3600
@@ -22,45 +21,18 @@
 
 #define CHUNK_SIZE 8192
 #define CLIENT_MAX_HEADER_SIZE 8192
-#define FILE_CACHE_SIZE 1024
-#define GEOIP_MAX_SIZE 8192
-
-#define SHM_KEY_CACHE 255641
-#define SHM_KEY_CONFIG 255642
-
-#define ERR_STR "\x1B[1;31m"
-#define CLR_STR "\x1B[0m"
-#define BLD_STR "\x1B[1m"
-#define WRN_STR "\x1B[1;33m"
-#define BLUE_STR "\x1B[34m"
-#define HTTP_STR "\x1B[1;31m"
-#define HTTPS_STR "\x1B[1;32m"
-
-#define HTTP_1XX_STR "\x1B[1;32m"
-#define HTTP_2XX_STR "\x1B[1;32m"
-#define HTTP_3XX_STR "\x1B[1;33m"
-#define HTTP_4XX_STR "\x1B[1;31m"
-#define HTTP_5XX_STR "\x1B[1;31m"
-
-#define NECRONDA_VERSION "4.3"
-#define SERVER_STR "Necronda/" NECRONDA_VERSION
-#define NECRONDA_ZLIB_LEVEL 9
 
 #ifndef DEFAULT_HOST
-#define DEFAULT_HOST "www.necronda.net"
-#endif
-#ifndef MAGIC_FILE
-#define MAGIC_FILE "/usr/share/file/misc/magic.mgc"
-#endif
-#ifndef PHP_FPM_SOCKET
-#define PHP_FPM_SOCKET "/var/run/php-fpm/php-fpm.sock"
-#endif
-#ifndef DEFAULT_CONFIG_FILE
-#define DEFAULT_CONFIG_FILE "/etc/necronda-server/necronda-server.conf"
+#   define DEFAULT_HOST "www.necronda.net"
 #endif
 
 extern int sockets[NUM_SOCKETS];
 extern pid_t children[MAX_CHILDREN];
 extern MMDB_s mmdbs[MAX_MMDB];
 
+extern int server_keep_alive;
+extern char *log_client_prefix, *log_conn_prefix, *log_req_prefix, *client_geoip;
+extern char *client_addr_str, *client_addr_str_ptr, *server_addr_str, *server_addr_str_ptr, *client_host_str;
+extern struct timeval client_timeout;
+
 #endif //NECRONDA_SERVER_NECRONDA_SERVER_H

src/necronda.h

@@ -0,0 +1,15 @@
+/**
+ * Necronda Web Server
+ * Definitions
+ * src/necronda.h
+ * Lorenz Stechauner, 2021-05-04
+ */
+
+#ifndef NECRONDA_SERVER_NECRONDA_H
+#define NECRONDA_SERVER_NECRONDA_H
+
+#define NECRONDA_VERSION "4.3"
+#define SERVER_STR "Necronda/" NECRONDA_VERSION
+#define SERVER_STR_HTML "Necronda web server " NECRONDA_VERSION
+
+#endif //NECRONDA_SERVER_NECRONDA_H