Add gcc opimization

Makefile

@@ -22,7 +22,7 @@ compile:
 compile-debian:
 	@mkdir -p bin
 	gcc $(LIBS) -o bin/libnecronda-server.so --shared -fPIC $(CFLAGS) $(INCLUDE) \
-		$(DEBIAN_OPTS)
+		$(DEBIAN_OPTS) -O3
 	gcc src/necronda-server.c -o bin/necronda-server $(CFLAGS) $(INCLUDE) \
 		-Lbin -lnecronda-server -Wl,-rpath=$(shell pwd)/bin \
-		$(DEBIAN_OPTS)
+		$(DEBIAN_OPTS) -O3

README.md

@@ -6,11 +6,11 @@ Necronda web server
 
 * Full IPv4 and IPv6 support
 * Serving local files via HTTP and HTTPS
-  * File compression and disk cache for compressed files
+  * File compression ([gzip](https://www.gzip.org/), [Brotli](https://www.brotli.org/)) and disk cache for compressed files
 * Reverse proxy for other HTTP and HTTPS servers
 * FastCGI support (e.g. [PHP-FPM](https://php-fpm.org/))
 * Support for [MaxMind's GeoIP Database](https://www.maxmind.com/en/geoip2-services-and-databases)
-* DNS reverse lookup for connecting hosts
+* Optional DNS reverse lookup for connecting hosts
 * Automatic URL rewrite (e.g. `/index.html` -> `/`, `/test.php` -> `/test`)
 * Modern looking and responsive error documents
 

src/client.c

@@ -13,6 +13,7 @@
 #include "lib/fastcgi.h"
 #include "lib/cache.h"
 #include "lib/geoip.h"
+#include "lib/compress.h"
 
 #include <string.h>
 #include <sys/select.h>
@@ -101,15 +102,15 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
         if (ret < 0) {
             goto abort;
         } else if (ret == 1) {
-            sprintf(err_msg, "Unable to parse header: Invalid header format.");
+            sprintf(err_msg, "Unable to parse http header: Invalid header format.");
         } else if (ret == 2) {
-            sprintf(err_msg, "Unable to parse header: Invalid method.");
+            sprintf(err_msg, "Unable to parse http header: Invalid method.");
         } else if (ret == 3) {
-            sprintf(err_msg, "Unable to parse header: Invalid version.");
+            sprintf(err_msg, "Unable to parse http header: Invalid version.");
         } else if (ret == 4) {
-            sprintf(err_msg, "Unable to parse header: Header contains illegal characters.");
+            sprintf(err_msg, "Unable to parse http header: Header contains illegal characters.");
         } else if (ret == 5) {
-            sprintf(err_msg, "Unable to parse header: End of header not found.");
+            sprintf(err_msg, "Unable to parse http header: End of header not found.");
         }
         res.status = http_get_status(400);
         goto respond;
@@ -137,7 +138,7 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
         strcpy(host, host_ptr);
     }
 
-    sprintf(log_req_prefix, "[%s%24s%s]%s ", BLD_STR, host, CLR_STR, log_client_prefix);
+    sprintf(log_req_prefix, "[%6i][%s%24s%s]%s ", getpid(), BLD_STR, host, CLR_STR, log_client_prefix);
     log_prefix = log_req_prefix;
     print(BLD_STR "%s %s" CLR_STR, req.method, req.uri);
 
@@ -218,6 +219,10 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
             goto respond;
         }
 
+        if (strncmp(uri.req_path, "/.well-known/", 13) == 0) {
+            http_add_header_field(&res.hdr, "Access-Control-Allow-Origin", "*");
+        }
+
         if (strncmp(uri.req_path, "/.well-known/", 13) != 0 && strstr(uri.path, "/.") != NULL) {
             res.status = http_get_status(403);
             sprintf(err_msg, "Parts of this URI are hidden.");
@@ -263,9 +268,43 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
             http_add_header_field(&res.hdr, "Last-Modified", last_modified);
             sprintf(buf1, "%s; charset=%s", uri.meta->type, uri.meta->charset);
             http_add_header_field(&res.hdr, "Content-Type", buf1);
+
+
+            char *accept_encoding = http_get_header_field(&req.hdr, "Accept-Encoding");
+            int enc = 0;
+            if (accept_encoding != NULL) {
+                if (uri.meta->filename_comp_br[0] != 0 && strstr(accept_encoding, "br") != NULL) {
+                    file = fopen(uri.meta->filename_comp_br, "rb");
+                    if (file == NULL) {
+                        cache_filename_comp_invalid(uri.filename);
+                    } else {
+                        http_add_header_field(&res.hdr, "Content-Encoding", "br");
+                        enc = COMPRESS_BR;
+                    }
+                } else if (uri.meta->filename_comp_gz[0] != 0 && strstr(accept_encoding, "gzip") != NULL) {
+                    file = fopen(uri.meta->filename_comp_gz, "rb");
+                    if (file == NULL) {
+                        cache_filename_comp_invalid(uri.filename);
+                    } else {
+                        http_add_header_field(&res.hdr, "Content-Encoding", "gzip");
+                        enc = COMPRESS_GZ;
+                    }
+                }
+                if (enc != 0) {
+                    http_add_header_field(&res.hdr, "Vary", "Accept-Encoding");
+                }
+            }
+
             if (uri.meta->etag[0] != 0) {
+                if (enc) {
+                    sprintf(buf0, "%s-%s", uri.meta->etag,
+                            (enc & COMPRESS_BR) ? "br" : (enc & COMPRESS_GZ) ? "gzip" : "");
+                    http_add_header_field(&res.hdr, "ETag", buf0);
+                } else {
                     http_add_header_field(&res.hdr, "ETag", uri.meta->etag);
                 }
+            }
+
             if (strncmp(uri.meta->type, "text/", 5) == 0) {
                 http_add_header_field(&res.hdr, "Cache-Control", "public, max-age=3600");
             } else {
@@ -325,25 +364,6 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
                 goto respond;
             }
 
-            char *accept_encoding = http_get_header_field(&req.hdr, "Accept-Encoding");
-            if (accept_encoding != NULL) {
-                if (uri.meta->filename_comp_br[0] != 0 && strstr(accept_encoding, "br") != NULL) {
-                    file = fopen(uri.meta->filename_comp_br, "rb");
-                    if (file == NULL) {
-                        cache_filename_comp_invalid(uri.filename);
-                    } else {
-                        http_add_header_field(&res.hdr, "Content-Encoding", "br");
-                    }
-                } else if (uri.meta->filename_comp_gz[0] != 0 && strstr(accept_encoding, "gzip") != NULL) {
-                    file = fopen(uri.meta->filename_comp_gz, "rb");
-                    if (file == NULL) {
-                        cache_filename_comp_invalid(uri.filename);
-                    } else {
-                        http_add_header_field(&res.hdr, "Content-Encoding", "gzip");
-                    }
-                }
-            }
-
             if (file == NULL) {
                 file = fopen(uri.filename, "rb");
             }
@@ -407,17 +427,17 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
             content_length = -1;
             use_fastcgi = 1;
 
-            char *accept_encoding = http_get_header_field(&req.hdr, "Accept-Encoding");
-            char *content_type = http_get_header_field(&res.hdr, "Content-Type");
-            char *content_encoding = http_get_header_field(&res.hdr, "Content-Encoding");
-            if (mime_is_compressible(content_type) && content_encoding == NULL && accept_encoding != NULL) {
-                if (strstr(accept_encoding, "br") != NULL) {
-                    http_add_header_field(&res.hdr, "Content-Encoding", "br");
+            int http_comp = http_get_compression(&req, &res);
+            if (http_comp & COMPRESS) {
+                if (http_comp & COMPRESS_BR) {
                     use_fastcgi |= FASTCGI_COMPRESS_BR;
-                } else if (strstr(accept_encoding, "gzip") != NULL) {
-                    http_add_header_field(&res.hdr, "Content-Encoding", "gzip");
+                    sprintf(buf0, "br");
+                } else if (http_comp & COMPRESS_GZ) {
                     use_fastcgi |= FASTCGI_COMPRESS_GZ;
+                    sprintf(buf0, "gzip");
                 }
+                http_add_header_field(&res.hdr, "Vary", "Accept-Encoding");
+                http_add_header_field(&res.hdr, "Content-Encoding", buf0);
             }
 
             if (http_get_header_field(&res.hdr, "Content-Length") == NULL) {
@@ -428,8 +448,33 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
         print("Reverse proxy for " BLD_STR "%s:%i" CLR_STR, conf->rev_proxy.hostname, conf->rev_proxy.port);
         http_remove_header_field(&res.hdr, "Date", HTTP_REMOVE_ALL);
         http_remove_header_field(&res.hdr, "Server", HTTP_REMOVE_ALL);
+
         ret = rev_proxy_init(&req, &res, conf, client, &custom_status, err_msg);
-        use_rev_proxy = ret == 0;
+        use_rev_proxy = (ret == 0);
+
+        /*
+        char *content_encoding = http_get_header_field(&res.hdr, "Content-Encoding");
+        if (use_rev_proxy && content_encoding == NULL) {
+            int http_comp = http_get_compression(&req, &res);
+            if (http_comp & COMPRESS_BR) {
+                use_rev_proxy |= REV_PROXY_COMPRESS_BR;
+            } else if (http_comp & COMPRESS_GZ) {
+                use_rev_proxy |= REV_PROXY_COMPRESS_GZ;
+            }
+        }
+
+        char *transfer_encoding = http_get_header_field(&res.hdr, "Transfer-Encoding");
+        int chunked = transfer_encoding != NULL && strcmp(transfer_encoding, "chunked") == 0;
+        http_remove_header_field(&res.hdr, "Transfer-Encoding", HTTP_REMOVE_ALL);
+        ret = sprintf(buf0, "%s%s%s",
+                      (use_rev_proxy & REV_PROXY_COMPRESS_BR) ? "br" :
+                      ((use_rev_proxy & REV_PROXY_COMPRESS_GZ) ? "gzip" : ""),
+                      ((use_rev_proxy & REV_PROXY_COMPRESS) && chunked) ? ", " : "",
+                      chunked ? "chunked" : "");
+        if (ret > 0) {
+            http_add_header_field(&res.hdr, "Transfer-Encoding", buf0);
+        }
+        */
     } else {
         print(ERR_STR "Unknown host type: %i" CLR_STR, conf->type);
         res.status = http_get_status(501);
@@ -515,17 +560,21 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int
         } else if (use_fastcgi) {
             char *transfer_encoding = http_get_header_field(&res.hdr, "Transfer-Encoding");
             int chunked = transfer_encoding != NULL && strcmp(transfer_encoding, "chunked") == 0;
+
             int flags = (chunked ? FASTCGI_CHUNKED : 0) | (use_fastcgi & FASTCGI_COMPRESS);
             fastcgi_send(&php_fpm, client, flags);
         } else if (use_rev_proxy) {
             char *transfer_encoding = http_get_header_field(&res.hdr, "Transfer-Encoding");
-            int chunked = transfer_encoding != NULL && strcmp(transfer_encoding, "chunked") == 0;
+            int chunked = transfer_encoding != NULL && strstr(transfer_encoding, "chunked") != NULL;
+
             char *content_len = http_get_header_field(&res.hdr, "Content-Length");
             unsigned long len_to_send = 0;
             if (content_len != NULL) {
                 len_to_send = strtol(content_len, NULL, 10);
             }
-            rev_proxy_send(client, chunked, len_to_send);
+
+            int flags = (chunked ? REV_PROXY_CHUNKED : 0) | (use_rev_proxy & REV_PROXY_COMPRESS);
+            rev_proxy_send(client, len_to_send, flags);
         }
     }
 
@@ -662,6 +711,7 @@ int client_connection_handler(sock *client, unsigned long client_num) {
         client->_ssl_error = ERR_get_error();
         if (ret <= 0) {
             print(ERR_STR "Unable to perform handshake: %s" CLR_STR, sock_strerror(client));
+            ret = -1;
             goto close;
         }
     }
@@ -724,12 +774,13 @@ int client_handler(sock *client, unsigned long client_num, struct sockaddr_in6 *
             ntohs(client_addr->sin6_port), CLR_STR);
 
     log_conn_prefix = malloc(256);
-    sprintf(log_conn_prefix, "[%24s]%s ", server_addr_str, log_client_prefix);
+    sprintf(log_conn_prefix, "[%6i][%24s]%s ", getpid(), server_addr_str, log_client_prefix);
     log_prefix = log_conn_prefix;
 
     print("Started child process with PID %i", getpid());
 
     ret = client_connection_handler(client, client_num);
+
     free(client_addr_str_ptr);
     client_addr_str_ptr = NULL;
     free(server_addr_str_ptr);
@@ -744,5 +795,6 @@ int client_handler(sock *client, unsigned long client_num, struct sockaddr_in6 *
     log_req_prefix = NULL;
     free(log_client_prefix);
     log_client_prefix = NULL;
+
     return ret;
 }

src/lib/cache.c

@@ -16,6 +16,7 @@
 #include <errno.h>
 #include <signal.h>
 #include <openssl/sha.h>
+#include <malloc.h>
 
 int cache_continue = 1;
 magic_t magic;
@@ -74,8 +75,8 @@ int cache_process() {
     }
 
     FILE *file;
-    char buf[16384];
-    char comp_buf[16384];
+    char *buf = malloc(CACHE_BUF_SIZE);
+    char *comp_buf = malloc(CACHE_BUF_SIZE);
     char filename_comp_gz[256];
     char filename_comp_br[256];
     unsigned long read;
@@ -144,24 +145,24 @@ int cache_process() {
                     }
                 }
 
-                while ((read = fread(buf, 1, sizeof(buf), file)) > 0) {
+                while ((read = fread(buf, 1, CACHE_BUF_SIZE, file)) > 0) {
                     SHA1_Update(&ctx, buf, read);
                     if (compress) {
                         unsigned long avail_in, avail_out;
                         avail_in = read;
                         do {
-                            avail_out = sizeof(comp_buf);
+                            avail_out = CACHE_BUF_SIZE;
                             compress_compress_mode(&comp_ctx, COMPRESS_GZ,buf + read - avail_in, &avail_in,
                                                    comp_buf, &avail_out, feof(file));
-                            fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_gz);
-                        } while (avail_in != 0);
+                            fwrite(comp_buf, 1, CACHE_BUF_SIZE - avail_out, comp_file_gz);
+                        } while (avail_in != 0 || avail_out != CACHE_BUF_SIZE);
                         avail_in = read;
                         do {
-                            avail_out = sizeof(comp_buf);
+                            avail_out = CACHE_BUF_SIZE;
                             compress_compress_mode(&comp_ctx, COMPRESS_BR, buf + read - avail_in, &avail_in,
                                                    comp_buf, &avail_out, feof(file));
-                            fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_br);
-                        } while (avail_in != 0);
+                            fwrite(comp_buf, 1, CACHE_BUF_SIZE - avail_out, comp_file_br);
+                        } while (avail_in != 0 || avail_out != CACHE_BUF_SIZE);
                     }
                 }
 
@@ -193,6 +194,8 @@ int cache_process() {
             cache_file = fopen("/var/necronda-server/cache", "wb");
             if (cache_file == NULL) {
                 fprintf(stderr, ERR_STR "Unable to open cache file: %s" CLR_STR "\n", strerror(errno));
+                free(buf);
+                free(comp_buf);
                 return -1;
             }
             fwrite(cache, sizeof(cache_entry), CACHE_ENTRIES, cache_file);
@@ -201,6 +204,8 @@ int cache_process() {
             sleep(1);
         }
     }
+    free(buf);
+    free(comp_buf);
     return 0;
 }
 
@@ -286,11 +291,11 @@ int cache_update_entry(int entry_num, const char *filename, const char *webroot)
     const char *type = magic_file(magic, filename);
     char type_new[24];
     sprintf(type_new, "%s", type);
-    if (strcmp(type, "text/plain") == 0) {
+    if (strncmp(type, "text/", 5) == 0) {
         if (strcmp(filename + strlen(filename) - 4, ".css") == 0) {
             sprintf(type_new, "text/css");
         } else if (strcmp(filename + strlen(filename) - 3, ".js") == 0) {
-            sprintf(type_new, "text/javascript");
+            sprintf(type_new, "application/javascript");
         }
     }
     strcpy(cache[entry_num].meta.type, type_new);

src/lib/cache.h

@@ -12,6 +12,7 @@
 
 #define CACHE_SHM_KEY 255641
 #define CACHE_ENTRIES 1024
+#define CACHE_BUF_SIZE 16384
 
 #ifndef CACHE_MAGIC_FILE
 #   define CACHE_MAGIC_FILE "/usr/share/file/misc/magic.mgc"

src/lib/compress.c

@@ -12,8 +12,10 @@
 int compress_init(compress_ctx *ctx, int mode) {
     ctx->gzip = NULL;
     ctx->brotli = NULL;
+    ctx->mode = 0;
     int ret;
     if (mode & COMPRESS_GZ) {
+        ctx->mode |= COMPRESS_GZ;
         ctx->gzip = malloc(sizeof(z_stream));
         ctx->gzip->zalloc = Z_NULL;
         ctx->gzip->zfree = Z_NULL;
@@ -22,6 +24,7 @@ int compress_init(compress_ctx *ctx, int mode) {
         if (ret != Z_OK) return -1;
     }
     if (mode & COMPRESS_BR) {
+        ctx->mode |= COMPRESS_BR;
         ctx->brotli = BrotliEncoderCreateInstance(NULL, NULL, NULL);
         if (ctx->brotli == NULL) return -1;
         BrotliEncoderSetParameter(ctx->brotli, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC);

src/lib/compress.h

@@ -16,6 +16,7 @@
 
 #define COMPRESS_GZ 1
 #define COMPRESS_BR 2
+#define COMPRESS 3
 
 typedef struct {
     int mode;

src/lib/fastcgi.c

@@ -278,7 +278,7 @@ int fastcgi_header(fastcgi_conn *conn, http_res *res, char *err_msg) {
     FCGI_Header header;
     char *content;
     unsigned short content_len, req_id;
-    int ret;
+    long ret;
     int err = 0;
 
     while (1) {
@@ -372,7 +372,7 @@ int fastcgi_header(fastcgi_conn *conn, http_res *res, char *err_msg) {
         }
 
         ret = http_parse_header_field(&res->hdr, ptr, pos0);
-        if (ret != 0) return ret;
+        if (ret != 0) return (int) ret;
         if (pos0[2] == '\r' && pos0[3] == '\n') {
             return 0;
         }
@@ -401,7 +401,7 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
         }
     } else if (flags & FASTCGI_COMPRESS_GZ) {
         flags &= ~FASTCGI_COMPRESS_BR;
-        if (compress_init(&comp_ctx, COMPRESS_BR) != 0) {
+        if (compress_init(&comp_ctx, COMPRESS_GZ) != 0) {
             print(ERR_STR "Unable to init gzip: %s" CLR_STR, strerror(errno));
             flags &= ~FASTCGI_COMPRESS_GZ;
         }
@@ -423,6 +423,7 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
             print(ERR_STR "Unable to receive from PHP-FPM" CLR_STR);
             return -1;
         }
+
         req_id = (header.requestIdB1 << 8) | header.requestIdB0;
         content_len = (header.contentLengthB1 << 8) | header.contentLengthB0;
         content = malloc(content_len + header.paddingLength);
@@ -457,7 +458,7 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
                 content_len = 0;
                 goto out;
                 finish:
-                if (flags & FASTCGI_COMPRESS) compress_free(&comp_ctx);
+                compress_free(&comp_ctx);
             }
 
             if (flags & FASTCGI_CHUNKED) {
@@ -476,10 +477,8 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
                 int buf_len = content_len;
                 if (flags & FASTCGI_COMPRESS) {
                     avail_out = sizeof(comp_out);
-                    if (flags & FASTCGI_COMPRESS) {
-                        compress_compress(&comp_ctx, next_in + content_len - avail_in, &avail_in,
-                                          comp_out, &avail_out, finish_comp);
-                    }
+                    compress_compress(&comp_ctx, next_in + content_len - avail_in, &avail_in, comp_out, &avail_out,
+                                      finish_comp);
                     ptr = comp_out;
                     buf_len = (int) (sizeof(comp_out) - avail_out);
                 }
@@ -489,7 +488,7 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) {
                     sock_send(client, ptr, buf_len, 0);
                     if (flags & FASTCGI_CHUNKED) sock_send(client, "\r\n", 2, 0);
                 }
-            } while ((flags & FASTCGI_COMPRESS) && avail_in != 0);
+            } while ((flags & FASTCGI_COMPRESS) && (avail_in != 0 || avail_out != sizeof(comp_out)));
             if (finish_comp) goto finish;
         } else {
             print(ERR_STR "Unknown FastCGI type: %i" CLR_STR, header.type);

src/lib/http.c

@@ -7,7 +7,7 @@
 
 #include "http.h"
 #include "utils.h"
-#include "../necronda-server.h"
+#include "compress.h"
 #include <string.h>
 
 void http_to_camel_case(char *str, int mode) {
@@ -33,7 +33,7 @@ void http_free_hdr(http_hdr *hdr) {
 }
 
 void http_free_req(http_req *req) {
-    if (req->uri == NULL) free(req->uri);
+    if (req->uri != NULL) free(req->uri);
     req->uri = NULL;
     http_free_hdr(&req->hdr);
 }
@@ -85,19 +85,19 @@ int http_receive_request(sock *client, http_req *req) {
     while (1) {
         rcv_len  = sock_recv(client, buf, CLIENT_MAX_HEADER_SIZE, 0);
         if (rcv_len <= 0) {
-            print("Unable to receive: %s", sock_strerror(client));
+            print("Unable to receive http header: %s", sock_strerror(client));
             return -1;
         }
 
         unsigned long header_len = strstr(buf, "\r\n\r\n") - buf + 4;
         if (header_len <= 0) {
-            print(ERR_STR "Unable to parse header: End of header not found" CLR_STR);
+            print(ERR_STR "Unable to parse http header: End of header not found" CLR_STR);
             return 5;
         }
 
         for (int i = 0; i < header_len; i++) {
             if ((buf[i] >= 0x00 && buf[i] <= 0x1F && buf[i] != '\r' && buf[i] != '\n') || buf[i] == 0x7F) {
-                print(ERR_STR "Unable to parse header: Header contains illegal characters" CLR_STR);
+                print(ERR_STR "Unable to parse http header: Header contains illegal characters" CLR_STR);
                 return 4;
             }
         }
@@ -106,7 +106,7 @@ int http_receive_request(sock *client, http_req *req) {
         while (header_len > (ptr - buf + 2)) {
             pos0 = strstr(ptr, "\r\n");
             if (pos0 == NULL) {
-                print(ERR_STR "Unable to parse header: Invalid header format" CLR_STR);
+                print(ERR_STR "Unable to parse http header: Invalid header format" CLR_STR);
                 return 1;
             }
 
@@ -115,13 +115,13 @@ int http_receive_request(sock *client, http_req *req) {
                 if (pos1 == NULL) goto err_hdr_fmt;
 
                 if (pos1 - ptr - 1 >= sizeof(req->method)) {
-                    print(ERR_STR "Unable to parse header: Method name too long" CLR_STR);
+                    print(ERR_STR "Unable to parse http header: Method name too long" CLR_STR);
                     return 2;
                 }
 
                 for (int i = 0; i < (pos1 - ptr - 1); i++) {
                     if (ptr[i] < 'A' || ptr[i] > 'Z') {
-                        print(ERR_STR "Unable to parse header: Invalid method" CLR_STR);
+                        print(ERR_STR "Unable to parse http header: Invalid method" CLR_STR);
                         return 2;
                     }
                 }
@@ -130,12 +130,12 @@ int http_receive_request(sock *client, http_req *req) {
                 pos2 = memchr(pos1, ' ', rcv_len - (pos1 - buf)) + 1;
                 if (pos2 == NULL) {
                     err_hdr_fmt:
-                    print(ERR_STR "Unable to parse header: Invalid header format" CLR_STR);
+                    print(ERR_STR "Unable to parse http header: Invalid header format" CLR_STR);
                     return 1;
                 }
 
                 if (memcmp(pos2, "HTTP/", 5) != 0 || memcmp(pos2 + 8, "\r\n", 2) != 0) {
-                    print(ERR_STR "Unable to parse header: Invalid version" CLR_STR);
+                    print(ERR_STR "Unable to parse http header: Invalid version" CLR_STR);
                     return 3;
                 }
 
@@ -312,3 +312,17 @@ const http_doc_info *http_get_status_info(const http_status *status) {
     }
     return NULL;
 }
+
+int http_get_compression(const http_req *req, const http_res *res) {
+    char *accept_encoding = http_get_header_field(&req->hdr, "Accept-Encoding");
+    char *content_type = http_get_header_field(&res->hdr, "Content-Type");
+    char *content_encoding = http_get_header_field(&res->hdr, "Content-Encoding");
+    if (mime_is_compressible(content_type) && content_encoding == NULL && accept_encoding != NULL) {
+        if (strstr(accept_encoding, "br") != NULL) {
+            return COMPRESS_BR;
+        } else if (strstr(accept_encoding, "gzip") != NULL) {
+            return COMPRESS_GZ;
+        }
+    }
+    return 0;
+}

src/lib/http.h

@@ -29,6 +29,8 @@
 #define HTTP_COLOR_WARNING "#E0C000"
 #define HTTP_COLOR_ERROR "#C00000"
 
+#define CLIENT_MAX_HEADER_SIZE 8192
+
 #ifndef SERVER_STR
 #   define SERVER_STR "Necronda"
 #endif
@@ -63,13 +65,13 @@ typedef struct {
 typedef struct {
     char method[16];
     char *uri;
-    char version[3];
+    char version[4];
     http_hdr hdr;
 } http_req;
 
 typedef struct {
     const http_status *status;
-    char version[3];
+    char version[4];
     http_hdr hdr;
 } http_res;
 
@@ -122,4 +124,6 @@ char *http_get_date(char *buf, size_t size);
 
 const http_doc_info *http_get_status_info(const http_status *status);
 
+int http_get_compression(const http_req *req, const http_res *res);
+
 #endif //NECRONDA_SERVER_HTTP_H

src/lib/http_static.c

@@ -5,8 +5,8 @@
  * Lorenz Stechauner, 2021-05-03
  */
 
+#include "../necronda.h"
 #include "http.h"
-#include "utils.h"
 
 const http_status http_statuses[] = {
         {100, "Informational", "Continue"},

src/lib/rev_proxy.c

@@ -7,6 +7,7 @@
 
 #include "rev_proxy.h"
 #include "utils.h"
+#include "compress.h"
 #include "../necronda-server.h"
 #include <openssl/ssl.h>
 #include <string.h>
@@ -377,47 +378,110 @@ int rev_proxy_init(http_req *req, http_res *res, host_config *conf, sock *client
     return -1;
 }
 
-int rev_proxy_send(sock *client, int chunked, unsigned long len_to_send) {
+int rev_proxy_send(sock *client, unsigned long len_to_send, int flags) {
+    // TODO handle websockets
     long ret;
     char buffer[CHUNK_SIZE];
+    char comp_out[CHUNK_SIZE];
+    char buf[256];
     long len, snd_len;
-    // TODO handle websockets
-    // TODO compress -> Transfer-Encoding: br/gzip
-    do {
-        if (chunked) {
-            ret = sock_recv(&rev_proxy, buffer, 16, MSG_PEEK);
-            if (ret <= 0) {
-                print("Unable to receive: %s", sock_strerror(&rev_proxy));
-                break;
+    int finish_comp = 0;
+    char *ptr;
+
+    compress_ctx comp_ctx;
+    if (flags & REV_PROXY_COMPRESS_BR) {
+        flags &= ~REV_PROXY_COMPRESS_GZ;
+        if (compress_init(&comp_ctx, COMPRESS_BR) != 0) {
+            print(ERR_STR "Unable to init brotli: %s" CLR_STR, strerror(errno));
+            flags &= ~REV_PROXY_COMPRESS_BR;
         }
+    } else if (flags & REV_PROXY_COMPRESS_GZ) {
+        flags &= ~REV_PROXY_COMPRESS_BR;
+        if (compress_init(&comp_ctx, COMPRESS_GZ) != 0) {
+            print(ERR_STR "Unable to init gzip: %s" CLR_STR, strerror(errno));
+            flags &= ~REV_PROXY_COMPRESS_GZ;
+        }
+    }
+
+    do {
+        snd_len = 0;
+        if (flags & REV_PROXY_CHUNKED) {
+            char *pos;
+            ret = sock_recv(&rev_proxy, buffer, 16, MSG_PEEK);
+            if (ret <= 0) goto err0;
 
             len_to_send = strtol(buffer, NULL, 16);
-            char *pos = strstr(buffer, "\r\n");
+            pos = strstr(buffer, "\r\n");
             len = pos - buffer + 2;
-            ret = sock_send(client, buffer, len, 0);
-
             sock_recv(&rev_proxy, buffer, len, 0);
-            if (ret <= 0) break;
+            if (ret <= 0) {
+                err0:
+                print("Unable to receive from server: %s", sock_strerror(&rev_proxy));
+                break;
+            }
+
+            if (len_to_send == 0 && (flags & REV_PROXY_COMPRESS)) {
+                finish_comp = 1;
+                len = 0;
+                goto out;
+                finish:
+                compress_free(&comp_ctx);
+            }
         }
-        snd_len = 0;
         while (snd_len < len_to_send) {
-            len = sock_recv(&rev_proxy, buffer, CHUNK_SIZE < (len_to_send - snd_len) ? CHUNK_SIZE : len_to_send - snd_len, 0);
-            ret = sock_send(client, buffer, len, 0);
+            unsigned long avail_in, avail_out;
+            ret = sock_recv(&rev_proxy, buffer, CHUNK_SIZE < (len_to_send - snd_len) ? CHUNK_SIZE : len_to_send - snd_len, 0);
             if (ret <= 0) {
+                print("Unable to receive from server: %s", sock_strerror(&rev_proxy));
+                break;
+            }
+            len = ret;
+            ptr = buffer;
+            out:
+            avail_in = len;
+            void *next_in = ptr;
+            do {
+                long buf_len = len;
+                if (flags & REV_PROXY_COMPRESS) {
+                    avail_out = sizeof(comp_out);
+                    compress_compress(&comp_ctx, next_in + len - avail_in, &avail_in, comp_out, &avail_out,
+                                      finish_comp);
+                    ptr = comp_out;
+                    buf_len = (int) (sizeof(comp_out) - avail_out);
+                    snd_len += (long) (len - avail_in);
+                }
+                if (buf_len != 0) {
+                    len = sprintf(buf, "%lX\r\n", buf_len);
+                    ret = 1;
+                    if (flags & REV_PROXY_CHUNKED) ret = sock_send(client, buf, len, 0);
+                    if (ret <= 0) goto err;
+                    ret = sock_send(client, ptr, buf_len, 0);
+                    if (ret <= 0) goto err;
+                    if (!(flags & REV_PROXY_COMPRESS)) snd_len += ret;
+                    if (flags & REV_PROXY_CHUNKED) ret = sock_send(client, "\r\n", 2, 0);
+                    if (ret <= 0) {
+                        err:
                         print(ERR_STR "Unable to send: %s" CLR_STR, sock_strerror(client));
                         break;
                     }
-            snd_len += ret;
+                }
+            } while ((flags & REV_PROXY_COMPRESS) && (avail_in != 0 || avail_out != sizeof(comp_out)));
+            if (ret <= 0) break;
+            if (finish_comp) goto finish;
         }
         if (ret <= 0) break;
-        if (chunked) {
-            sock_recv(&rev_proxy, buffer, 2, 0);
-            ret = sock_send(client, "\r\n", 2, 0);
+        if (flags & REV_PROXY_CHUNKED) sock_recv(&rev_proxy, buffer, 2, 0);
+    } while ((flags & REV_PROXY_CHUNKED) && len_to_send > 0);
+
+    if (ret <= 0) return (int) -1;
+
+    if (flags & REV_PROXY_CHUNKED) {
+        ret = sock_send(client, "0\r\n\r\n", 5, 0);
         if (ret <= 0) {
             print(ERR_STR "Unable to send: %s" CLR_STR, sock_strerror(client));
-                break;
+            return -1;
         }
     }
-    } while (chunked && len_to_send > 0);
+
     return 0;
 }

src/lib/rev_proxy.h

@@ -8,6 +8,11 @@
 #ifndef NECRONDA_SERVER_REV_PROXY_H
 #define NECRONDA_SERVER_REV_PROXY_H
 
+#define REV_PROXY_CHUNKED 1
+#define REV_PROXY_COMPRESS_GZ 2
+#define REV_PROXY_COMPRESS_BR 4
+#define REV_PROXY_COMPRESS 6
+
 #include "http.h"
 #include "config.h"
 
@@ -22,6 +27,6 @@ int rev_proxy_response_header(http_req *req, http_res *res);
 int rev_proxy_init(http_req *req, http_res *res, host_config *conf, sock *client, http_status *custom_status,
                    char *err_msg);
 
-int rev_proxy_send(sock *client, int chunked, unsigned long len_to_send);
+int rev_proxy_send(sock *client, unsigned long len_to_send, int flags);
 
 #endif //NECRONDA_SERVER_REV_PROXY_H

src/lib/sock.c

@@ -52,12 +52,12 @@ long sock_send(sock *s, void *buf, unsigned long len, int flags) {
     long ret;
     if (s->enc) {
         ret = SSL_write(s->ssl, buf, (int) len);
+        s->_ssl_error = ERR_get_error();
     } else {
         ret = send(s->socket, buf, len, flags);
     }
     s->_last_ret = ret;
     s->_errno = errno;
-    s->_ssl_error = ERR_get_error();
     return ret >= 0 ? ret : -1;
 }
 
@@ -69,12 +69,12 @@ long sock_recv(sock *s, void *buf, unsigned long len, int flags) {
         } else {
             ret = SSL_read(s->ssl, buf, (int) len);
         }
+        s->_ssl_error = ERR_get_error();
     } else {
         ret = recv(s->socket, buf, len, flags);
     }
     s->_last_ret = ret;
     s->_errno = errno;
-    s->_ssl_error = ERR_get_error();
     return ret >= 0 ? ret : -1;
 }
 
@@ -97,14 +97,14 @@ long sock_splice(sock *dst, sock *src, void *buf, unsigned long buf_len, unsigne
 
 int sock_close(sock *s) {
     if ((int) s->enc && s->ssl != NULL) {
-        SSL_shutdown(s->ssl);
+        if (s->_last_ret >= 0) SSL_shutdown(s->ssl);
         SSL_free(s->ssl);
+        s->ssl = NULL;
     }
     shutdown(s->socket, SHUT_RDWR);
     close(s->socket);
     s->socket = 0;
     s->enc = 0;
-    s->ssl = NULL;
     return 0;
 }
 

src/lib/utils.c

@@ -104,25 +104,30 @@ int url_decode(const char *str, char *dec, long *size) {
 }
 
 int mime_is_compressible(const char *type) {
+    if (type == NULL) return 0;
+    char type_parsed[64];
+    strncpy(type_parsed, type, sizeof(type_parsed) - 1);
+    char *pos = strchr(type_parsed, ';');
+    if (pos != NULL) pos[0] = 0;
     return
-        strncmp(type, "text/", 5) == 0 ||
-        strncmp(type, "message/", 7) == 0 ||
-        strstr(type, "+xml") != NULL ||
-        strstr(type, "+json") != NULL ||
-        strcmp(type, "application/javascript") == 0 ||
-        strcmp(type, "application/json") == 0 ||
-        strcmp(type, "application/xml") == 0 ||
-        strcmp(type, "application/x-www-form-urlencoded") == 0 ||
-        strcmp(type, "application/x-tex") == 0 ||
-        strcmp(type, "application/x-httpd-php") == 0 ||
-        strcmp(type, "application/x-latex") == 0 ||
-        strcmp(type, "application/vnd.ms-fontobject") == 0 ||
-        strcmp(type, "application/x-font-ttf") == 0 ||
-        strcmp(type, "application/x-javascript") == 0 ||
-        strcmp(type, "application/x-web-app-manifest+json") == 0 ||
-        strcmp(type, "font/eot") == 0 ||
-        strcmp(type, "font/opentype") == 0 ||
-        strcmp(type, "image/bmp") == 0 ||
-        strcmp(type, "image/vnd.microsoft.icon") == 0 ||
-        strcmp(type, "image/x-icon") == 0;
+        strncmp(type_parsed, "text/", 5) == 0 ||
+        strncmp(type_parsed, "message/", 7) == 0 ||
+        strstr(type_parsed, "+xml") != NULL ||
+        strstr(type_parsed, "+json") != NULL ||
+        strcmp(type_parsed, "application/javascript") == 0 ||
+        strcmp(type_parsed, "application/json") == 0 ||
+        strcmp(type_parsed, "application/xml") == 0 ||
+        strcmp(type_parsed, "application/x-www-form-urlencoded") == 0 ||
+        strcmp(type_parsed, "application/x-tex") == 0 ||
+        strcmp(type_parsed, "application/x-httpd-php") == 0 ||
+        strcmp(type_parsed, "application/x-latex") == 0 ||
+        strcmp(type_parsed, "application/vnd.ms-fontobject") == 0 ||
+        strcmp(type_parsed, "application/x-font-ttf") == 0 ||
+        strcmp(type_parsed, "application/x-javascript") == 0 ||
+        strcmp(type_parsed, "application/x-web-app-manifest+json") == 0 ||
+        strcmp(type_parsed, "font/eot") == 0 ||
+        strcmp(type_parsed, "font/opentype") == 0 ||
+        strcmp(type_parsed, "image/bmp") == 0 ||
+        strcmp(type_parsed, "image/vnd.microsoft.icon") == 0 ||
+        strcmp(type_parsed, "image/x-icon") == 0;
 }

src/necronda-server.c

@@ -275,6 +275,16 @@ int main(int argc, const char *argv[]) {
         closedir(geoip);
     }
 
+    ret = cache_init();
+    if (ret < 0) {
+        config_unload();
+        return 1;
+    } else if (ret != 0) {
+        children[0] = ret;  // pid
+    } else {
+        return 0;
+    }
+
     openssl_init();
 
     client.buf = NULL;
@@ -283,7 +293,7 @@ int main(int argc, const char *argv[]) {
     client.ctx = SSL_CTX_new(TLS_server_method());
     SSL_CTX_set_options(client.ctx, SSL_OP_SINGLE_DH_USE);
     SSL_CTX_set_verify(client.ctx, SSL_VERIFY_NONE, NULL);
-    SSL_CTX_set_min_proto_version(client.ctx, TLS1_VERSION);
+    SSL_CTX_set_min_proto_version(client.ctx, TLS1_2_VERSION);
     SSL_CTX_set_mode(client.ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
     SSL_CTX_set_cipher_list(client.ctx, "HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4");
     SSL_CTX_set_ecdh_auto(client.ctx, 1);
@@ -319,16 +329,6 @@ int main(int argc, const char *argv[]) {
         }
     }
 
-    ret = cache_init();
-    if (ret < 0) {
-        config_unload();
-        return 1;
-    } else if (ret != 0) {
-        children[0] = ret;  // pid
-    } else {
-        return 0;
-    }
-
     fprintf(stderr, "Ready to accept connections\n");
 
     while (active) {
@@ -375,6 +375,7 @@ int main(int argc, const char *argv[]) {
             }
         }
 
+        // TODO outsource in thread
         int status = 0;
         for (int i = 0; i < MAX_CHILDREN; i++) {
             if (children[i] != 0) {

src/necronda-server.h

@@ -15,12 +15,11 @@
 #define MAX_CHILDREN 1024
 #define MAX_MMDB 3
 #define LISTEN_BACKLOG 16
-#define REQ_PER_CONNECTION 100
+#define REQ_PER_CONNECTION 200
 #define CLIENT_TIMEOUT 3600
 #define SERVER_TIMEOUT 4
 
 #define CHUNK_SIZE 8192
-#define CLIENT_MAX_HEADER_SIZE 8192
 
 #ifndef DEFAULT_HOST
 #   define DEFAULT_HOST "www.necronda.net"

src/necronda.h

@@ -8,7 +8,7 @@
 #ifndef NECRONDA_SERVER_NECRONDA_H
 #define NECRONDA_SERVER_NECRONDA_H
 
-#define NECRONDA_VERSION "4.3"
+#define NECRONDA_VERSION "4.4"
 #define SERVER_STR "Necronda/" NECRONDA_VERSION
 #define SERVER_STR_HTML "Necronda web server " NECRONDA_VERSION