From 80986325cea699a48d7a8041d8e0060b8cba2da5 Mon Sep 17 00:00:00 2001 From: Lorenz Stechauner Date: Tue, 4 May 2021 22:32:21 +0200 Subject: [PATCH] Added brotli compression --- Makefile | 2 +- src/client.c | 29 ++++++---- src/client.h | 18 ------- src/lib/brotli.c | 24 +++++++++ src/lib/brotli.h | 11 ++++ src/lib/cache.c | 123 ++++++++++++++++++++++++++---------------- src/lib/cache.h | 12 +++++ src/lib/config.c | 17 +++--- src/lib/config.h | 3 ++ src/lib/fastcgi.c | 51 ++++++++++-------- src/lib/fastcgi.h | 8 ++- src/lib/geoip.c | 69 ++++++++++++++++++++++++ src/lib/geoip.h | 18 +++++++ src/lib/gzip.c | 28 ++++++++++ src/lib/gzip.h | 18 +++++++ src/lib/http.h | 14 +++++ src/lib/http_static.c | 4 +- src/lib/rev_proxy.c | 1 - src/lib/uri.h | 3 +- src/lib/utils.c | 80 ++++++--------------------- src/lib/utils.h | 18 ++++--- src/necronda-server.c | 3 +- src/necronda-server.h | 44 +++------------ src/necronda.h | 15 ++++++ 24 files changed, 392 insertions(+), 221 deletions(-) delete mode 100644 src/client.h create mode 100644 src/lib/brotli.c create mode 100644 src/lib/brotli.h create mode 100644 src/lib/geoip.c create mode 100644 src/lib/geoip.h create mode 100644 src/lib/gzip.c create mode 100644 src/lib/gzip.h create mode 100644 src/necronda.h diff --git a/Makefile b/Makefile index f6e8654..f9cd357 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ CFLAGS=-std=c11 -Wall -INCLUDE=-lssl -lcrypto -lmagic -lz -lmaxminddb +INCLUDE=-lssl -lcrypto -lmagic -lz -lmaxminddb -lbrotlienc LIBS=src/lib/*.c DEBIAN_OPTS=-D MAGIC_FILE="\"/usr/share/file/magic.mgc\"" -D PHP_FPM_SOCKET="\"/var/run/php/php7.3-fpm.sock\"" diff --git a/src/client.c b/src/client.c index dafdd6d..cd4142a 100644 --- a/src/client.c +++ b/src/client.c @@ -5,7 +5,6 @@ * Lorenz Stechauner, 2020-12-03 */ -#include "client.h" #include "lib/utils.h" #include "lib/config.h" #include "lib/sock.h" @@ -13,6 +12,7 @@ #include "lib/rev_proxy.h" #include "lib/fastcgi.h" #include "lib/cache.h" +#include "lib/geoip.h" #include #include @@ -32,7 +32,7 @@ char *client_addr_str, *client_addr_str_ptr, *server_addr_str, *server_addr_str_ struct timeval client_timeout; host_config *get_host_config(const char *host) { - for (int i = 0; i < MAX_HOST_CONFIG; i++) { + for (int i = 0; i < CONFIG_MAX_HOST_CONFIG; i++) { host_config *hc = &config[i]; if (hc->type == CONFIG_TYPE_UNSET) break; if (strcmp(hc->name, host) == 0) return hc; @@ -326,16 +326,23 @@ int client_request_handler(sock *client, unsigned long client_num, unsigned int } char *accept_encoding = http_get_header_field(&req.hdr, "Accept-Encoding"); - if (uri.meta->filename_comp[0] != 0 && accept_encoding != NULL && - strstr(accept_encoding, "deflate") != NULL) { - file = fopen(uri.meta->filename_comp, "rb"); - if (file == NULL) { - cache_filename_comp_invalid(uri.filename); - goto not_compressed; + if (accept_encoding != NULL) { + if (uri.meta->filename_comp_br[0] != 0 && strstr(accept_encoding, "br") != NULL) { + file = fopen(uri.meta->filename_comp_br, "rb"); + if (file == NULL) { + printf("asdf\n"); + cache_filename_comp_invalid(uri.filename); + } + http_add_header_field(&res.hdr, "Content-Encoding", "br"); + } else if (uri.meta->filename_comp_gz[0] != 0 && strstr(accept_encoding, "gzip") != NULL) { + file = fopen(uri.meta->filename_comp_gz, "rb"); + if (file == NULL) { + cache_filename_comp_invalid(uri.filename); + } + http_add_header_field(&res.hdr, "Content-Encoding", "gzip"); } - http_add_header_field(&res.hdr, "Content-Encoding", "deflate"); - } else { - not_compressed: + } + if (file == NULL) { file = fopen(uri.filename, "rb"); } fseek(file, 0, SEEK_END); diff --git a/src/client.h b/src/client.h deleted file mode 100644 index 461dd18..0000000 --- a/src/client.h +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Necronda Web Server - * Client connection and request handlers (header file) - * src/client.h - * Lorenz Stechauner, 2021-01-17 - */ - -#ifndef NECRONDA_SERVER_CLIENT_H -#define NECRONDA_SERVER_CLIENT_H - -#include - -extern int server_keep_alive; -extern char *log_client_prefix, *log_conn_prefix, *log_req_prefix, *client_geoip; -extern char *client_addr_str, *client_addr_str_ptr, *server_addr_str, *server_addr_str_ptr, *client_host_str; -extern struct timeval client_timeout; - -#endif //NECRONDA_SERVER_CLIENT_H diff --git a/src/lib/brotli.c b/src/lib/brotli.c new file mode 100644 index 0000000..ae10f6e --- /dev/null +++ b/src/lib/brotli.c @@ -0,0 +1,24 @@ +/** + * + */ + +#include "brotli.h" + +int brotli_init(BrotliEncoderState **state) { + *state = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if (*state == NULL) return -1; + BrotliEncoderSetParameter(*state, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC); + BrotliEncoderSetParameter(*state, BROTLI_PARAM_MODE, BROTLI_MODE_GENERIC); + return 0; +} + +int brotli_compress(BrotliEncoderState *state, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) { + int ret = BrotliEncoderCompressStream(state, finish ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS, + in_len, (const unsigned char**) &in, out_len, (unsigned char **) &out, NULL); + return (ret == BROTLI_TRUE) ? 0 : -1; +} + +int brotli_free(BrotliEncoderState *state) { + BrotliEncoderDestroyInstance(state); + return 0; +} diff --git a/src/lib/brotli.h b/src/lib/brotli.h new file mode 100644 index 0000000..f5e12ef --- /dev/null +++ b/src/lib/brotli.h @@ -0,0 +1,11 @@ +/** + * + */ + +#include + +int brotli_init(BrotliEncoderState **state); + +int brotli_compress(BrotliEncoderState *state, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish); + +int brotli_free(BrotliEncoderState *state); diff --git a/src/lib/cache.c b/src/lib/cache.c index a0a6188..4058282 100644 --- a/src/lib/cache.c +++ b/src/lib/cache.c @@ -7,9 +7,9 @@ #include "cache.h" #include "utils.h" -#include "../necronda-server.h" +#include "gzip.h" +#include "brotli.h" #include -#include #include #include #include @@ -28,7 +28,7 @@ int magic_init() { fprintf(stderr, ERR_STR "Unable to open magic cookie: %s" CLR_STR "\n", strerror(errno)); return -1; } - if (magic_load(magic, MAGIC_FILE) != 0) { + if (magic_load(magic, CACHE_MAGIC_FILE) != 0) { fprintf(stderr, ERR_STR "Unable to load magic cookie: %s" CLR_STR "\n", magic_error(magic)); return -2; } @@ -43,7 +43,7 @@ int cache_process() { signal(SIGINT, cache_process_term); signal(SIGTERM, cache_process_term); - int shm_id = shmget(SHM_KEY_CACHE, FILE_CACHE_SIZE * sizeof(cache_entry), 0); + int shm_id = shmget(CACHE_SHM_KEY, CACHE_ENTRIES * sizeof(cache_entry), 0); if (shm_id < 0) { fprintf(stderr, ERR_STR "Unable to create shared memory: %s" CLR_STR "\n", strerror(errno)); return -1; @@ -66,26 +66,28 @@ int cache_process() { FILE *cache_file = fopen("/var/necronda-server/cache", "rb"); if (cache_file != NULL) { - fread(cache, sizeof(cache_entry), FILE_CACHE_SIZE, cache_file); + fread(cache, sizeof(cache_entry), CACHE_ENTRIES, cache_file); fclose(cache_file); } - for (int i = 0; i < FILE_CACHE_SIZE; i++) { + for (int i = 0; i < CACHE_ENTRIES; i++) { cache[i].is_updating = 0; } FILE *file; char buf[16384]; char comp_buf[16384]; - char filename_comp[256]; + char filename_comp_gz[256]; + char filename_comp_br[256]; unsigned long read; int compress; SHA_CTX ctx; unsigned char hash[SHA_DIGEST_LENGTH]; int cache_changed = 0; - int p_len; + int p_len_gz, p_len_br; + int ret_1, ret_2; while (cache_continue) { - for (int i = 0; i < FILE_CACHE_SIZE; i++) { + for (int i = 0; i < CACHE_ENTRIES; i++) { if (cache[i].filename[0] != 0 && cache[i].meta.etag[0] == 0 && !cache[i].is_updating) { cache[i].is_updating = 1; fprintf(stdout, "[cache] Hashing file %s\n", cache[i].filename); @@ -93,9 +95,10 @@ int cache_process() { file = fopen(cache[i].filename, "rb"); compress = mime_is_compressible(cache[i].meta.type); - int level = NECRONDA_ZLIB_LEVEL; - z_stream strm; - FILE *comp_file = NULL; + z_stream gz_state; + BrotliEncoderState *br_state = NULL; + FILE *comp_file_gz = NULL; + FILE *comp_file_br = NULL; if (compress) { sprintf(buf, "%.*s/.necronda-server", cache[i].webroot_len, cache[i].filename); mkdir(buf, 0755); @@ -110,27 +113,41 @@ int cache_process() { buf[j] = ch; } buf[strlen(rel_path)] = 0; - p_len = snprintf(filename_comp, sizeof(filename_comp), "%.*s/.necronda-server/cache/%s.z", - cache[i].webroot_len, cache[i].filename, buf); - if (p_len < 0 || p_len >= sizeof(filename_comp)) { + + p_len_gz = snprintf(filename_comp_gz, sizeof(filename_comp_gz), + "%.*s/.necronda-server/cache/%s.gz", + cache[i].webroot_len, cache[i].filename, buf); + p_len_br = snprintf(filename_comp_br, sizeof(filename_comp_br), + "%.*s/.necronda-server/cache/%s.br", + cache[i].webroot_len, cache[i].filename, buf); + if (p_len_gz < 0 || p_len_gz >= sizeof(filename_comp_gz) || + p_len_br < 0 || p_len_br >= sizeof(filename_comp_br)) { fprintf(stderr, ERR_STR "Unable to open cached file: " "File name for compressed file too long" CLR_STR "\n"); goto comp_err; } + fprintf(stdout, "[cache] Compressing file %s\n", cache[i].filename); - comp_file = fopen(filename_comp, "wb"); - if (comp_file == NULL) { + + comp_file_gz = fopen(filename_comp_gz, "wb"); + comp_file_br = fopen(filename_comp_br, "wb"); + if (comp_file_gz == NULL || comp_file_br == NULL) { fprintf(stderr, ERR_STR "Unable to open cached file: %s" CLR_STR "\n", strerror(errno)); comp_err: compress = 0; } else { - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - if (deflateInit(&strm, level) != Z_OK) { - fprintf(stderr, ERR_STR "Unable to init deflate: %s" CLR_STR "\n", strerror(errno)); + ret_1 = gzip_init(&gz_state); + ret_2 = brotli_init(&br_state); + if (ret_1 != 0) { + fprintf(stderr, ERR_STR "Unable to init gzip: %s" CLR_STR "\n", strerror(errno)); compress = 0; - fclose(comp_file); + fclose(comp_file_gz); + fclose(comp_file_br); + } else if (ret_2 != 0) { + fprintf(stderr, ERR_STR "Unable to init brotli: %s" CLR_STR "\n", strerror(errno)); + compress = 0; + fclose(comp_file_gz); + fclose(comp_file_br); } } } @@ -138,25 +155,33 @@ int cache_process() { while ((read = fread(buf, 1, sizeof(buf), file)) > 0) { SHA1_Update(&ctx, buf, read); if (compress) { - strm.avail_in = read; - strm.next_in = (unsigned char *) buf; + unsigned long avail_in, avail_out; + avail_in = read; do { - strm.avail_out = sizeof(comp_buf); - strm.next_out = (unsigned char *) comp_buf; - deflate(&strm, feof(file) ? Z_FINISH : Z_NO_FLUSH); - fwrite(comp_buf, 1, sizeof(comp_buf) - strm.avail_out, comp_file); - strm.avail_in = 0; - } while (strm.avail_out == 0); + avail_out = sizeof(comp_buf); + gzip_compress(&gz_state, buf + read - avail_in, &avail_in, comp_buf, &avail_out, feof(file)); + fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_gz); + } while (avail_in != 0); + avail_in = read; + do { + avail_out = sizeof(comp_buf); + brotli_compress(br_state, buf + read - avail_in, &avail_in, comp_buf, &avail_out, feof(file)); + fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_br); + } while (avail_in != 0); } } if (compress) { - deflateEnd(&strm); - fclose(comp_file); + gzip_free(&gz_state); + brotli_free(br_state); + fclose(comp_file_gz); + fclose(comp_file_br); fprintf(stdout, "[cache] Finished compressing file %s\n", cache[i].filename); - strcpy(cache[i].meta.filename_comp, filename_comp); + strcpy(cache[i].meta.filename_comp_gz, filename_comp_gz); + strcpy(cache[i].meta.filename_comp_br, filename_comp_br); } else { - memset(cache[i].meta.filename_comp, 0, sizeof(cache[i].meta.filename_comp)); + memset(cache[i].meta.filename_comp_gz, 0, sizeof(cache[i].meta.filename_comp_gz)); + memset(cache[i].meta.filename_comp_br, 0, sizeof(cache[i].meta.filename_comp_br)); } SHA1_Final(hash, &ctx); memset(cache[i].meta.etag, 0, sizeof(cache[i].meta.etag)); @@ -173,7 +198,11 @@ int cache_process() { if (cache_changed) { cache_changed = 0; cache_file = fopen("/var/necronda-server/cache", "wb"); - fwrite(cache, sizeof(cache_entry), FILE_CACHE_SIZE, cache_file); + if (cache_file == NULL) { + fprintf(stderr, ERR_STR "Unable to open cache file: %s" CLR_STR "\n", strerror(errno)); + return -1; + } + fwrite(cache, sizeof(cache_entry), CACHE_ENTRIES, cache_file); fclose(cache_file); } else { sleep(1); @@ -187,7 +216,7 @@ int cache_init() { return -1; } - int shm_id = shmget(SHM_KEY_CACHE, FILE_CACHE_SIZE * sizeof(cache_entry), IPC_CREAT | IPC_EXCL | 0600); + int shm_id = shmget(CACHE_SHM_KEY, CACHE_ENTRIES * sizeof(cache_entry), IPC_CREAT | IPC_EXCL | 0600); if (shm_id < 0) { fprintf(stderr, ERR_STR "Unable to create shared memory: %s" CLR_STR "\n", strerror(errno)); return -2; @@ -206,7 +235,7 @@ int cache_init() { return -4; } cache = shm_rw; - memset(cache, 0, FILE_CACHE_SIZE * sizeof(cache_entry)); + memset(cache, 0, CACHE_ENTRIES * sizeof(cache_entry)); shmdt(shm_rw); cache = shm; @@ -229,7 +258,7 @@ int cache_init() { } int cache_unload() { - int shm_id = shmget(SHM_KEY_CACHE, 0, 0); + int shm_id = shmget(CACHE_SHM_KEY, 0, 0); if (shm_id < 0) { fprintf(stderr, ERR_STR "Unable to get shared memory id: %s" CLR_STR "\n", strerror(errno)); shmdt(cache); @@ -245,7 +274,7 @@ int cache_unload() { int cache_update_entry(int entry_num, const char *filename, const char *webroot) { void *cache_ro = cache; - int shm_id = shmget(SHM_KEY_CACHE, 0, 0); + int shm_id = shmget(CACHE_SHM_KEY, 0, 0); void *shm_rw = shmat(shm_id, NULL, 0); if (shm_rw == (void *) -1) { print(ERR_STR "Unable to attach shared memory (rw): %s" CLR_STR, strerror(errno)); @@ -277,7 +306,8 @@ int cache_update_entry(int entry_num, const char *filename, const char *webroot) strcpy(cache[entry_num].meta.charset, magic_file(magic, filename)); memset(cache[entry_num].meta.etag, 0, sizeof(cache[entry_num].meta.etag)); - memset(cache[entry_num].meta.filename_comp, 0, sizeof(cache[entry_num].meta.filename_comp)); + memset(cache[entry_num].meta.filename_comp_gz, 0, sizeof(cache[entry_num].meta.filename_comp_gz)); + memset(cache[entry_num].meta.filename_comp_br, 0, sizeof(cache[entry_num].meta.filename_comp_br)); cache[entry_num].is_updating = 0; shmdt(shm_rw); @@ -287,7 +317,7 @@ int cache_update_entry(int entry_num, const char *filename, const char *webroot) int cache_filename_comp_invalid(const char *filename) { void *cache_ro = cache; - int shm_id = shmget(SHM_KEY_CACHE, 0, 0); + int shm_id = shmget(CACHE_SHM_KEY, 0, 0); void *shm_rw = shmat(shm_id, NULL, 0); if (shm_rw == (void *) -1) { print(ERR_STR "Unable to attach shared memory (rw): %s" CLR_STR, strerror(errno)); @@ -296,7 +326,7 @@ int cache_filename_comp_invalid(const char *filename) { cache = shm_rw; int i; - for (i = 0; i < FILE_CACHE_SIZE; i++) { + for (i = 0; i < CACHE_ENTRIES; i++) { if (cache[i].filename[0] != 0 && strlen(cache[i].filename) == strlen(filename) && strcmp(cache[i].filename, filename) == 0) { if (cache[i].is_updating) { @@ -308,7 +338,8 @@ int cache_filename_comp_invalid(const char *filename) { } memset(cache[i].meta.etag, 0, sizeof(cache[i].meta.etag)); - memset(cache[i].meta.filename_comp, 0, sizeof(cache[i].meta.filename_comp)); + memset(cache[i].meta.filename_comp_gz, 0, sizeof(cache[i].meta.filename_comp_gz)); + memset(cache[i].meta.filename_comp_br, 0, sizeof(cache[i].meta.filename_comp_br)); cache[i].is_updating = 0; shmdt(shm_rw); @@ -322,7 +353,7 @@ int uri_cache_init(http_uri *uri) { } int i; - for (i = 0; i < FILE_CACHE_SIZE; i++) { + for (i = 0; i < CACHE_ENTRIES; i++) { if (cache[i].filename[0] != 0 && strlen(cache[i].filename) == strlen(uri->filename) && strcmp(cache[i].filename, uri->filename) == 0) { uri->meta = &cache[i].meta; @@ -335,7 +366,7 @@ int uri_cache_init(http_uri *uri) { } if (uri->meta == NULL) { - for (i = 0; i < FILE_CACHE_SIZE; i++) { + for (i = 0; i < CACHE_ENTRIES; i++) { if (cache[i].filename[0] == 0) { if (cache_update_entry(i, uri->filename, uri->webroot) != 0) { return -1; diff --git a/src/lib/cache.h b/src/lib/cache.h index 00fb647..6d5ddde 100644 --- a/src/lib/cache.h +++ b/src/lib/cache.h @@ -10,6 +10,18 @@ #include "uri.h" +#define CACHE_SHM_KEY 255641 +#define CACHE_ENTRIES 1024 + +#ifndef CACHE_MAGIC_FILE +# define CACHE_MAGIC_FILE "/usr/share/file/misc/magic.mgc" +#endif + +#ifndef DEFAULT_CONFIG_FILE +# define DEFAULT_CONFIG_FILE "/etc/necronda-server/necronda-server.conf" +#endif + + typedef struct { char filename[256]; unsigned char webroot_len; diff --git a/src/lib/config.c b/src/lib/config.c index 75de0cc..e642df2 100644 --- a/src/lib/config.c +++ b/src/lib/config.c @@ -6,20 +6,19 @@ */ #include "config.h" -#include "../necronda-server.h" +#include "utils.h" #include #include #include #include #include -#include #include host_config *config; char cert_file[256], key_file[256], geoip_dir[256], dns_server[256]; int config_init() { - int shm_id = shmget(SHM_KEY_CONFIG, MAX_HOST_CONFIG * sizeof(host_config), IPC_CREAT | IPC_EXCL | 0640); + int shm_id = shmget(CONFIG_SHM_KEY, CONFIG_MAX_HOST_CONFIG * sizeof(host_config), IPC_CREAT | IPC_EXCL | 0640); if (shm_id < 0) { fprintf(stderr, ERR_STR "Unable to create shared memory: %s" CLR_STR "\n", strerror(errno)); return -1; @@ -38,14 +37,14 @@ int config_init() { return -3; } config = shm_rw; - memset(config, 0, MAX_HOST_CONFIG * sizeof(host_config)); + memset(config, 0, CONFIG_MAX_HOST_CONFIG * sizeof(host_config)); shmdt(shm_rw); config = shm; return 0; } int config_unload() { - int shm_id = shmget(SHM_KEY_CONFIG, 0, 0); + int shm_id = shmget(CONFIG_SHM_KEY, 0, 0); if (shm_id < 0) { fprintf(stderr, ERR_STR "Unable to get shared memory id: %s" CLR_STR "\n", strerror(errno)); shmdt(config); @@ -73,8 +72,8 @@ int config_load(const char *filename) { fread(conf, 1, len, file); fclose(file); - host_config *tmp_config = malloc(MAX_HOST_CONFIG * sizeof(host_config)); - memset(tmp_config, 0, MAX_HOST_CONFIG * sizeof(host_config)); + host_config *tmp_config = malloc(CONFIG_MAX_HOST_CONFIG * sizeof(host_config)); + memset(tmp_config, 0, CONFIG_MAX_HOST_CONFIG * sizeof(host_config)); int i = 0; int mode = 0; @@ -195,7 +194,7 @@ int config_load(const char *filename) { } } - int shm_id = shmget(SHM_KEY_CONFIG, 0, 0); + int shm_id = shmget(CONFIG_SHM_KEY, 0, 0); if (shm_id < 0) { fprintf(stderr, ERR_STR "Unable to get shared memory id: %s" CLR_STR "\n", strerror(errno)); shmdt(config); @@ -208,7 +207,7 @@ int config_load(const char *filename) { fprintf(stderr, ERR_STR "Unable to attach shared memory (rw): %s" CLR_STR "\n", strerror(errno)); return -4; } - memcpy(shm_rw, tmp_config, MAX_HOST_CONFIG * sizeof(host_config)); + memcpy(shm_rw, tmp_config, CONFIG_MAX_HOST_CONFIG * sizeof(host_config)); free(tmp_config); shmdt(shm_rw); return 0; diff --git a/src/lib/config.h b/src/lib/config.h index 158a923..a75ac62 100644 --- a/src/lib/config.h +++ b/src/lib/config.h @@ -10,6 +10,9 @@ #include "uri.h" +#define CONFIG_SHM_KEY 255642 +#define CONFIG_MAX_HOST_CONFIG 64 + #define CONFIG_TYPE_UNSET 0 #define CONFIG_TYPE_LOCAL 1 #define CONFIG_TYPE_REVERSE_PROXY 2 diff --git a/src/lib/fastcgi.c b/src/lib/fastcgi.c index 0ab8a90..3351a63 100644 --- a/src/lib/fastcgi.c +++ b/src/lib/fastcgi.c @@ -7,10 +7,10 @@ #include "fastcgi.h" #include "utils.h" -#include "../client.h" +#include "gzip.h" +#include "brotli.h" #include "../necronda-server.h" #include -#include #include #include #include @@ -385,7 +385,7 @@ int fastcgi_header(fastcgi_conn *conn, http_res *res, char *err_msg) { int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) { FCGI_Header header; - int ret; + long ret; char buf0[256]; int len; char *content, *ptr; @@ -393,15 +393,19 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) { char comp_out[4096]; int finish_comp = 0; - z_stream strm; - if (flags & FASTCGI_COMPRESS) { - int level = NECRONDA_ZLIB_LEVEL; - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - if (deflateInit(&strm, level) != Z_OK) { - print(ERR_STR "Unable to init deflate: %s" CLR_STR, strerror(errno)); - flags &= !FASTCGI_COMPRESS; + z_stream gz_state; + BrotliEncoderState *br_state = NULL; + if (flags & FASTCGI_COMPRESS_BR) { + flags &= !FASTCGI_COMPRESS_GZ; + if (brotli_init(&br_state) != 0) { + print(ERR_STR "Unable to init brotli: %s" CLR_STR, strerror(errno)); + flags &= !FASTCGI_COMPRESS_BR; + } + } else if (flags & FASTCGI_COMPRESS_GZ) { + flags &= !FASTCGI_COMPRESS_BR; + if (gzip_init(&gz_state) != 0) { + print(ERR_STR "Unable to init gzip: %s" CLR_STR, strerror(errno)); + flags &= !FASTCGI_COMPRESS_GZ; } } @@ -455,7 +459,8 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) { content_len = 0; goto out; finish: - deflateEnd(&strm); + if (flags & FASTCGI_COMPRESS_GZ) gzip_free(&gz_state); + if (flags & FASTCGI_COMPRESS_BR) brotli_free(br_state); } if (flags & FASTCGI_CHUNKED) { @@ -466,20 +471,20 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) { } else if (header.type == FCGI_STDERR) { fastcgi_php_error(content, content_len, buf0); } else if (header.type == FCGI_STDOUT) { + unsigned long avail_in = content_len, avail_out; + void *next_in = ptr; out: - if (flags & FASTCGI_COMPRESS) { - strm.avail_in = content_len; - strm.next_in = (unsigned char *) ptr; - } do { int buf_len = content_len; if (flags & FASTCGI_COMPRESS) { - strm.avail_out = sizeof(comp_out); - strm.next_out = (unsigned char *) comp_out; - deflate(&strm, finish_comp ? Z_FINISH : Z_NO_FLUSH); - strm.avail_in = 0; + avail_out = sizeof(comp_out); + if (flags & FASTCGI_COMPRESS_GZ) { + gzip_compress(&gz_state, next_in + content_len - avail_in, &avail_in, comp_out, &avail_out, finish_comp); + } else if (flags & FASTCGI_COMPRESS_BR) { + brotli_compress(br_state, next_in + content_len - avail_in, &avail_in, comp_out, &avail_out, finish_comp); + } ptr = comp_out; - buf_len = (int) (sizeof(comp_out) - strm.avail_out); + buf_len = (int) (sizeof(comp_out) - avail_out); } if (buf_len != 0) { len = sprintf(buf0, "%X\r\n", buf_len); @@ -487,7 +492,7 @@ int fastcgi_send(fastcgi_conn *conn, sock *client, int flags) { sock_send(client, ptr, buf_len, 0); if (flags & FASTCGI_CHUNKED) sock_send(client, "\r\n", 2, 0); } - } while ((flags & FASTCGI_COMPRESS) && strm.avail_out == 0); + } while ((flags & FASTCGI_COMPRESS) && avail_in != 0); if (finish_comp) goto finish; } else { print(ERR_STR "Unknown FastCGI type: %i" CLR_STR, header.type); diff --git a/src/lib/fastcgi.h b/src/lib/fastcgi.h index d2d0e43..4807b34 100644 --- a/src/lib/fastcgi.h +++ b/src/lib/fastcgi.h @@ -13,7 +13,13 @@ #include "uri.h" #define FASTCGI_CHUNKED 1 -#define FASTCGI_COMPRESS 2 +#define FASTCGI_COMPRESS_GZ 2 +#define FASTCGI_COMPRESS_BR 4 +#define FASTCGI_COMPRESS 6 + +#ifndef PHP_FPM_SOCKET +# define PHP_FPM_SOCKET "/var/run/php-fpm/php-fpm.sock" +#endif typedef struct { int socket; diff --git a/src/lib/geoip.c b/src/lib/geoip.c new file mode 100644 index 0000000..1a865c4 --- /dev/null +++ b/src/lib/geoip.c @@ -0,0 +1,69 @@ +/** + * Necronda Web Server + * MaxMind GeoIP Database interface + * src/lib/geoip.c + * Lorenz Stechauner, 2021-05-04 + */ + +#include "geoip.h" + +MMDB_entry_data_list_s *mmdb_json(MMDB_entry_data_list_s *list, char *str, long *str_off, long str_len) { + switch (list->entry_data.type) { + case MMDB_DATA_TYPE_MAP: + *str_off += sprintf(str + *str_off, "{"); + break; + case MMDB_DATA_TYPE_ARRAY: + *str_off += sprintf(str + *str_off, "["); + break; + case MMDB_DATA_TYPE_UTF8_STRING: + *str_off += sprintf(str + *str_off, "\"%.*s\"", list->entry_data.data_size, list->entry_data.utf8_string); + break; + case MMDB_DATA_TYPE_UINT16: + *str_off += sprintf(str + *str_off, "%u", list->entry_data.uint16); + break; + case MMDB_DATA_TYPE_UINT32: + *str_off += sprintf(str + *str_off, "%u", list->entry_data.uint32); + break; + case MMDB_DATA_TYPE_UINT64: + *str_off += sprintf(str + *str_off, "%lu", list->entry_data.uint64); + break; + case MMDB_DATA_TYPE_UINT128: + *str_off += sprintf(str + *str_off, "%llu", (unsigned long long) list->entry_data.uint128); + break; + case MMDB_DATA_TYPE_INT32: + *str_off += sprintf(str + *str_off, "%i", list->entry_data.uint32); + break; + case MMDB_DATA_TYPE_BOOLEAN: + *str_off += sprintf(str + *str_off, "%s", list->entry_data.boolean ? "true" : "false"); + break; + case MMDB_DATA_TYPE_FLOAT: + *str_off += sprintf(str + *str_off, "%f", list->entry_data.float_value); + break; + case MMDB_DATA_TYPE_DOUBLE: + *str_off += sprintf(str + *str_off, "%f", list->entry_data.double_value); + break; + } + if (list->entry_data.type != MMDB_DATA_TYPE_MAP && list->entry_data.type != MMDB_DATA_TYPE_ARRAY) { + return list->next; + } + MMDB_entry_data_list_s *next = list->next; + int stat = 0; + for (int i = 0; i < list->entry_data.data_size; i++) { + next = mmdb_json(next, str, str_off, str_len); + if (list->entry_data.type == MMDB_DATA_TYPE_MAP) { + stat = !stat; + if (stat) { + i--; + *str_off += sprintf(str + *str_off, ":"); + continue; + } + } + if (i != list->entry_data.data_size - 1) *str_off += sprintf(str + *str_off, ","); + } + if (list->entry_data.type == MMDB_DATA_TYPE_MAP) { + *str_off += sprintf(str + *str_off, "}"); + } else { + *str_off += sprintf(str + *str_off, "]"); + } + return next; +} diff --git a/src/lib/geoip.h b/src/lib/geoip.h new file mode 100644 index 0000000..eb000df --- /dev/null +++ b/src/lib/geoip.h @@ -0,0 +1,18 @@ +/** + * Necronda Web Server + * MaxMind GeoIP Database interface (header file) + * src/lib/geoip.h + * Lorenz Stechauner, 2021-05-04 + */ + + +#ifndef NECRONDA_SERVER_GEOIP_H +#define NECRONDA_SERVER_GEOIP_H + +#include + +#define GEOIP_MAX_SIZE 8192 + +MMDB_entry_data_list_s *mmdb_json(MMDB_entry_data_list_s *list, char *str, long *str_off, long str_len); + +#endif //NECRONDA_SERVER_GEOIP_H diff --git a/src/lib/gzip.c b/src/lib/gzip.c new file mode 100644 index 0000000..462345e --- /dev/null +++ b/src/lib/gzip.c @@ -0,0 +1,28 @@ +/** + * + */ + +#include "gzip.h" + +int gzip_init(z_stream *stream) { + stream->zalloc = Z_NULL; + stream->zfree = Z_NULL; + stream->opaque = Z_NULL; + int ret = deflateInit2(stream, GZIP_LEVEL, Z_DEFLATED, 15 + 16, 9, Z_DEFAULT_STRATEGY); + return (ret == Z_OK) ? 0 : -1; +} + +int gzip_compress(z_stream *stream, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish) { + stream->next_in = (unsigned char*) in; + stream->avail_in = *in_len; + stream->next_out = (unsigned char*) out; + stream->avail_out = *out_len; + int ret = deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH); + *in_len = stream->avail_in; + *out_len = stream->avail_out; + return ret; +} + +int gzip_free(z_stream *stream) { + return deflateEnd(stream); +} diff --git a/src/lib/gzip.h b/src/lib/gzip.h new file mode 100644 index 0000000..2d573fb --- /dev/null +++ b/src/lib/gzip.h @@ -0,0 +1,18 @@ +/** + * + */ + +#ifndef NECRONDA_SERVER_GZIP_H +#define NECRONDA_SERVER_GZIP_H + +#include + +#define GZIP_LEVEL 9 + +int gzip_init(z_stream *stream); + +int gzip_compress(z_stream *stream, const char *in, unsigned long *in_len, char *out, unsigned long *out_len, int finish); + +int gzip_free(z_stream *stream); + +#endif //NECRONDA_SERVER_GZIP_H diff --git a/src/lib/http.h b/src/lib/http.h index 8ea2038..7179e34 100644 --- a/src/lib/http.h +++ b/src/lib/http.h @@ -18,11 +18,25 @@ #define HTTP_REMOVE_ALL 1 #define HTTP_REMOVE_LAST 2 +#define HTTP_1XX_STR "\x1B[1;32m" +#define HTTP_2XX_STR "\x1B[1;32m" +#define HTTP_3XX_STR "\x1B[1;33m" +#define HTTP_4XX_STR "\x1B[1;31m" +#define HTTP_5XX_STR "\x1B[1;31m" + #define HTTP_COLOR_SUCCESS "#008000" #define HTTP_COLOR_INFO "#606060" #define HTTP_COLOR_WARNING "#E0C000" #define HTTP_COLOR_ERROR "#C00000" +#ifndef SERVER_STR +# define SERVER_STR "Necronda" +#endif + +#ifndef SERVER_STR_HTML +# define SERVER_STR_HTML "Necronda web server" +#endif + typedef struct { unsigned short code; char type[16]; diff --git a/src/lib/http_static.c b/src/lib/http_static.c index 1e2ef80..46d3c03 100644 --- a/src/lib/http_static.c +++ b/src/lib/http_static.c @@ -6,7 +6,7 @@ */ #include "http.h" -#include "../necronda-server.h" +#include "utils.h" const http_status http_statuses[] = { {100, "Informational", "Continue"}, @@ -137,7 +137,7 @@ const char http_default_document[] = "\t
\n" "\t\t
\n" "%3$s" - "\t\t\t
%7$s - Necronda web server " NECRONDA_VERSION "
\n" + "\t\t\t
%7$s - " SERVER_STR_HTML "
\n" "\t\t
\n" "\t
\n" "\n" diff --git a/src/lib/rev_proxy.c b/src/lib/rev_proxy.c index c1c76e7..7e3e378 100644 --- a/src/lib/rev_proxy.c +++ b/src/lib/rev_proxy.c @@ -7,7 +7,6 @@ #include "rev_proxy.h" #include "utils.h" -#include "../client.h" #include "../necronda-server.h" #include #include diff --git a/src/lib/uri.h b/src/lib/uri.h index b305c97..9a26329 100644 --- a/src/lib/uri.h +++ b/src/lib/uri.h @@ -19,7 +19,8 @@ typedef struct { char etag[64]; char type[24]; char charset[16]; - char filename_comp[256]; + char filename_comp_gz[256]; + char filename_comp_br[256]; struct stat stat; } meta_data; diff --git a/src/lib/utils.c b/src/lib/utils.c index 95717b1..7d1b277 100644 --- a/src/lib/utils.c +++ b/src/lib/utils.c @@ -6,6 +6,7 @@ */ #include "utils.h" +#include #include #include @@ -26,7 +27,7 @@ char *format_duration(unsigned long micros, char *buf) { return buf; } -int url_encode_component(const char *str, char *enc, ssize_t *size) { +int url_encode_component(const char *str, char *enc, long *size) { char *ptr = enc; char ch; memset(enc, 0, *size); @@ -53,7 +54,7 @@ int url_encode_component(const char *str, char *enc, ssize_t *size) { return 0; } -int url_encode(const char *str, char *enc, ssize_t *size) { +int url_encode(const char *str, char *enc, long *size) { char *ptr = enc; unsigned char ch; memset(enc, 0, *size); @@ -76,7 +77,7 @@ int url_encode(const char *str, char *enc, ssize_t *size) { return 0; } -int url_decode(const char *str, char *dec, ssize_t *size) { +int url_decode(const char *str, char *dec, long *size) { char *ptr = dec; char ch, buf[3]; memset(dec, 0, *size); @@ -107,72 +108,21 @@ int mime_is_compressible(const char *type) { strncmp(type, "text/", 5) == 0 || strncmp(type, "message/", 7) == 0 || strstr(type, "+xml") != NULL || + strstr(type, "+json") != NULL || strcmp(type, "application/javascript") == 0 || strcmp(type, "application/json") == 0 || strcmp(type, "application/xml") == 0 || strcmp(type, "application/x-www-form-urlencoded") == 0 || strcmp(type, "application/x-tex") == 0 || strcmp(type, "application/x-httpd-php") == 0 || - strcmp(type, "application/x-latex") == 0; -} - -MMDB_entry_data_list_s *mmdb_json(MMDB_entry_data_list_s *list, char *str, long *str_off, long str_len) { - switch (list->entry_data.type) { - case MMDB_DATA_TYPE_MAP: - *str_off += sprintf(str + *str_off, "{"); - break; - case MMDB_DATA_TYPE_ARRAY: - *str_off += sprintf(str + *str_off, "["); - break; - case MMDB_DATA_TYPE_UTF8_STRING: - *str_off += sprintf(str + *str_off, "\"%.*s\"", list->entry_data.data_size, list->entry_data.utf8_string); - break; - case MMDB_DATA_TYPE_UINT16: - *str_off += sprintf(str + *str_off, "%u", list->entry_data.uint16); - break; - case MMDB_DATA_TYPE_UINT32: - *str_off += sprintf(str + *str_off, "%u", list->entry_data.uint32); - break; - case MMDB_DATA_TYPE_UINT64: - *str_off += sprintf(str + *str_off, "%lu", list->entry_data.uint64); - break; - case MMDB_DATA_TYPE_UINT128: - *str_off += sprintf(str + *str_off, "%llu", (unsigned long long) list->entry_data.uint128); - break; - case MMDB_DATA_TYPE_INT32: - *str_off += sprintf(str + *str_off, "%i", list->entry_data.uint32); - break; - case MMDB_DATA_TYPE_BOOLEAN: - *str_off += sprintf(str + *str_off, "%s", list->entry_data.boolean ? "true" : "false"); - break; - case MMDB_DATA_TYPE_FLOAT: - *str_off += sprintf(str + *str_off, "%f", list->entry_data.float_value); - break; - case MMDB_DATA_TYPE_DOUBLE: - *str_off += sprintf(str + *str_off, "%f", list->entry_data.double_value); - break; - } - if (list->entry_data.type != MMDB_DATA_TYPE_MAP && list->entry_data.type != MMDB_DATA_TYPE_ARRAY) { - return list->next; - } - MMDB_entry_data_list_s *next = list->next; - int stat = 0; - for (int i = 0; i < list->entry_data.data_size; i++) { - next = mmdb_json(next, str, str_off, str_len); - if (list->entry_data.type == MMDB_DATA_TYPE_MAP) { - stat = !stat; - if (stat) { - i--; - *str_off += sprintf(str + *str_off, ":"); - continue; - } - } - if (i != list->entry_data.data_size - 1) *str_off += sprintf(str + *str_off, ","); - } - if (list->entry_data.type == MMDB_DATA_TYPE_MAP) { - *str_off += sprintf(str + *str_off, "}"); - } else { - *str_off += sprintf(str + *str_off, "]"); - } - return next; + strcmp(type, "application/x-latex") == 0 || + strcmp(type, "application/vnd.ms-fontobject") == 0 || + strcmp(type, "application/x-font-ttf") == 0 || + strcmp(type, "application/x-javascript") == 0 || + strcmp(type, "application/x-web-app-manifest+json") == 0 || + strcmp(type, "font/eot") == 0 || + strcmp(type, "font/opentype") == 0 || + strcmp(type, "image/bmp") == 0 || + strcmp(type, "image/vnd.microsoft.icon") == 0 || + strcmp(type, "image/x-icon") == 0; } diff --git a/src/lib/utils.h b/src/lib/utils.h index 0b404c6..74f5c81 100644 --- a/src/lib/utils.h +++ b/src/lib/utils.h @@ -8,7 +8,15 @@ #ifndef NECRONDA_SERVER_UTILS_H #define NECRONDA_SERVER_UTILS_H -#include +#include + +#define ERR_STR "\x1B[1;31m" +#define CLR_STR "\x1B[0m" +#define BLD_STR "\x1B[1m" +#define WRN_STR "\x1B[1;33m" +#define BLUE_STR "\x1B[34m" +#define HTTP_STR "\x1B[1;31m" +#define HTTPS_STR "\x1B[1;32m" extern char *log_prefix; @@ -24,14 +32,12 @@ extern char *log_prefix; char *format_duration(unsigned long micros, char *buf); -int url_encode_component(const char *str, char *enc, ssize_t *size); +int url_encode_component(const char *str, char *enc, long *size); -int url_encode(const char *str, char *enc, ssize_t *size); +int url_encode(const char *str, char *enc, long *size); -int url_decode(const char *str, char *dec, ssize_t *size); +int url_decode(const char *str, char *dec, long *size); int mime_is_compressible(const char *type); -MMDB_entry_data_list_s *mmdb_json(MMDB_entry_data_list_s *list, char *str, long *str_off, long str_len); - #endif //NECRONDA_SERVER_UTILS_H diff --git a/src/necronda-server.c b/src/necronda-server.c index 2017e1b..d324be1 100644 --- a/src/necronda-server.c +++ b/src/necronda-server.c @@ -7,6 +7,7 @@ #define _POSIX_C_SOURCE 199309L +#include "necronda.h" #include "necronda-server.h" #include "client.c" @@ -14,6 +15,7 @@ #include "lib/config.h" #include "lib/sock.h" #include "lib/rev_proxy.h" +#include "lib/geoip.h" #include #include @@ -29,7 +31,6 @@ #include #include #include -#include #include int active = 1; diff --git a/src/necronda-server.h b/src/necronda-server.h index eaa8e0c..c9ddddb 100644 --- a/src/necronda-server.h +++ b/src/necronda-server.h @@ -1,20 +1,19 @@ /** * Necronda Web Server * Main executable (header file) - * src/necronda-server.c + * src/necronda-server.h * Lorenz Stechauner, 2020-12-03 */ #ifndef NECRONDA_SERVER_NECRONDA_SERVER_H #define NECRONDA_SERVER_NECRONDA_SERVER_H -#include +#include #include #define NUM_SOCKETS 2 #define MAX_CHILDREN 1024 #define MAX_MMDB 3 -#define MAX_HOST_CONFIG 64 #define LISTEN_BACKLOG 16 #define REQ_PER_CONNECTION 100 #define CLIENT_TIMEOUT 3600 @@ -22,45 +21,18 @@ #define CHUNK_SIZE 8192 #define CLIENT_MAX_HEADER_SIZE 8192 -#define FILE_CACHE_SIZE 1024 -#define GEOIP_MAX_SIZE 8192 - -#define SHM_KEY_CACHE 255641 -#define SHM_KEY_CONFIG 255642 - -#define ERR_STR "\x1B[1;31m" -#define CLR_STR "\x1B[0m" -#define BLD_STR "\x1B[1m" -#define WRN_STR "\x1B[1;33m" -#define BLUE_STR "\x1B[34m" -#define HTTP_STR "\x1B[1;31m" -#define HTTPS_STR "\x1B[1;32m" - -#define HTTP_1XX_STR "\x1B[1;32m" -#define HTTP_2XX_STR "\x1B[1;32m" -#define HTTP_3XX_STR "\x1B[1;33m" -#define HTTP_4XX_STR "\x1B[1;31m" -#define HTTP_5XX_STR "\x1B[1;31m" - -#define NECRONDA_VERSION "4.3" -#define SERVER_STR "Necronda/" NECRONDA_VERSION -#define NECRONDA_ZLIB_LEVEL 9 #ifndef DEFAULT_HOST -#define DEFAULT_HOST "www.necronda.net" -#endif -#ifndef MAGIC_FILE -#define MAGIC_FILE "/usr/share/file/misc/magic.mgc" -#endif -#ifndef PHP_FPM_SOCKET -#define PHP_FPM_SOCKET "/var/run/php-fpm/php-fpm.sock" -#endif -#ifndef DEFAULT_CONFIG_FILE -#define DEFAULT_CONFIG_FILE "/etc/necronda-server/necronda-server.conf" +# define DEFAULT_HOST "www.necronda.net" #endif extern int sockets[NUM_SOCKETS]; extern pid_t children[MAX_CHILDREN]; extern MMDB_s mmdbs[MAX_MMDB]; +extern int server_keep_alive; +extern char *log_client_prefix, *log_conn_prefix, *log_req_prefix, *client_geoip; +extern char *client_addr_str, *client_addr_str_ptr, *server_addr_str, *server_addr_str_ptr, *client_host_str; +extern struct timeval client_timeout; + #endif //NECRONDA_SERVER_NECRONDA_SERVER_H diff --git a/src/necronda.h b/src/necronda.h new file mode 100644 index 0000000..c912f7f --- /dev/null +++ b/src/necronda.h @@ -0,0 +1,15 @@ +/** + * Necronda Web Server + * Definitions + * src/necronda.h + * Lorenz Stechauner, 2021-05-04 + */ + +#ifndef NECRONDA_SERVER_NECRONDA_H +#define NECRONDA_SERVER_NECRONDA_H + +#define NECRONDA_VERSION "4.3" +#define SERVER_STR "Necronda/" NECRONDA_VERSION +#define SERVER_STR_HTML "Necronda web server " NECRONDA_VERSION + +#endif //NECRONDA_SERVER_NECRONDA_H