From 2ed8451db17fede5f8d084c2284ba4e4a94819c8 Mon Sep 17 00:00:00 2001 From: Lorenz Stechauner Date: Wed, 23 Dec 2020 21:19:40 +0100 Subject: [PATCH] File compression working --- Makefile | 2 +- src/cache.c | 106 +++++++++++++++++++++++++++++++++++++++--- src/cache.h | 5 +- src/client.c | 5 ++ src/necronda-server.h | 1 + 5 files changed, 110 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 7798abf..515be31 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ packages: compile: @mkdir -p bin - gcc src/necronda-server.c -o bin/necronda-server -std=c11 -lssl -lcrypto -lmagic + gcc src/necronda-server.c -o bin/necronda-server -std=c11 -lssl -lcrypto -lmagic -lz install: | packages update compile @echo "Finished!" diff --git a/src/cache.c b/src/cache.c index bde56b8..7107427 100644 --- a/src/cache.c +++ b/src/cache.c @@ -5,6 +5,7 @@ * Lorenz Stechauner, 2020-12-19 */ +#include #include "cache.h" #include "uri.h" @@ -42,7 +43,7 @@ int cache_process() { } cache = shm_rw; - mkdir("/var/necronda-server", 0655); + mkdir("/var/necronda-server", 0755); FILE *cache_file = fopen("/var/necronda-server/cache", "rb"); if (cache_file != NULL) { @@ -56,6 +57,8 @@ int cache_process() { FILE *file; char buf[16384]; + char comp_buf[16384]; + char filename_comp[256]; unsigned long read; int compress; SHA_CTX ctx; @@ -67,12 +70,65 @@ int cache_process() { SHA1_Init(&ctx); file = fopen(cache[i].filename, "rb"); compress = strncmp(cache[i].meta.type, "text/", 5) == 0; + + int level = NECRONDA_ZLIB_LEVEL; + z_stream strm; + FILE *comp_file = NULL; + if (compress) { + sprintf(buf, "%.*s/.necronda-server", cache[i].webroot_len, cache[i].filename); + mkdir(buf, 0755); + sprintf(buf, "%.*s/.necronda-server/cache", cache[i].webroot_len, cache[i].filename); + mkdir(buf, 0700); + char *rel_path = cache[i].filename + cache[i].webroot_len + 1; + for (int j = 0; j < strlen(rel_path); j++) { + char ch = rel_path[j]; + if (ch == '/') { + ch = '_'; + } + buf[j] = ch; + } + buf[strlen(rel_path)] = 0; + sprintf(filename_comp, "%.*s/.necronda-server/cache/%s.z", cache[i].webroot_len, cache[i].filename, buf); + comp_file = fopen(filename_comp, "wb"); + if (comp_file == NULL) { + compress = 0; + fprintf(stderr, ERR_STR "Unable to open cache file: %s" CLR_STR "\n", strerror(errno)); + } else { + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + if (deflateInit(&strm, level) != Z_OK) { + fprintf(stderr, ERR_STR "Unable to init deflate: %s" CLR_STR "\n", strerror(errno)); + compress = 0; + fclose(comp_file); + } + } + } + while ((read = fread(buf, 1, sizeof(buf), file)) > 0) { SHA1_Update(&ctx, buf, read); if (compress) { - // TODO compress text/* files + strm.avail_in = read; + strm.next_in = (unsigned char *) buf; + strm.avail_out = sizeof(comp_buf); + strm.next_out = (unsigned char *) comp_buf; + do { + strm.avail_out = sizeof(comp_buf); + strm.next_out = (unsigned char *) comp_buf; + deflate(&strm, feof(file) ? Z_FINISH : Z_NO_FLUSH); + fwrite(comp_buf, 1, sizeof(comp_buf) - strm.avail_out, comp_file); + strm.avail_in = 0; + } while (strm.avail_out == 0); } } + + if (compress) { + deflateEnd(&strm); + fclose(comp_file); + strcpy(cache[i].meta.filename_comp, filename_comp); + } else { + memset(cache[i].meta.filename_comp, 0, sizeof(cache[i].meta.filename_comp)); + } SHA1_Final(hash, &ctx); memset(cache[i].meta.etag, 0, sizeof(cache[i].meta.etag)); for (int j = 0; j < SHA_DIGEST_LENGTH; j++) { @@ -150,7 +206,7 @@ int cache_unload() { return 0; } -int cache_update_entry(int entry_num, const char *filename) { +int cache_update_entry(int entry_num, const char *filename, const char *webroot) { void *cache_ro = cache; int shm_id = shmget(SHM_KEY, 0, 0); void *shm_rw = shmat(shm_id, NULL, 0); @@ -164,22 +220,27 @@ int cache_update_entry(int entry_num, const char *filename) { stat(filename, &statbuf); memcpy(&cache[entry_num].meta.stat, &statbuf, sizeof(statbuf)); + cache[entry_num].webroot_len = (unsigned char) strlen(webroot); strcpy(cache[entry_num].filename, filename); + magic_setflags(magic, MAGIC_MIME_TYPE); const char *type = magic_file(magic, filename); char type_new[24]; sprintf(type_new, "%s", type); - if (strncmp(type, "text/plain", 10) == 0) { + if (strcmp(type, "text/plain") == 0) { if (strncmp(filename + strlen(filename) - 4, ".css", 4) == 0) { sprintf(type_new, "text/css"); - } else if (strncmp(filename + strlen(filename) - 3, ".js", 3) == 0) { + } else if (strcmp(filename + strlen(filename) - 3, ".js") == 0) { sprintf(type_new, "text/javascript"); } } strcpy(cache[entry_num].meta.type, type_new); + magic_setflags(magic, MAGIC_MIME_ENCODING); strcpy(cache[entry_num].meta.charset, magic_file(magic, filename)); + memset(cache[entry_num].meta.etag, 0, sizeof(cache[entry_num].meta.etag)); + memset(cache[entry_num].meta.filename_comp, 0, sizeof(cache[entry_num].meta.filename_comp)); cache[entry_num].is_updating = 0; shmdt(shm_rw); @@ -187,6 +248,37 @@ int cache_update_entry(int entry_num, const char *filename) { return 0; } +int cache_filename_comp_invalid(const char *filename) { + void *cache_ro = cache; + int shm_id = shmget(SHM_KEY, 0, 0); + void *shm_rw = shmat(shm_id, NULL, 0); + if (shm_rw == (void *) -1) { + print(ERR_STR "Unable to attach shared memory (rw): %s" CLR_STR, strerror(errno)); + return -1; + } + cache = shm_rw; + + int i; + for (i = 0; i < FILE_CACHE_SIZE; i++) { + if (cache[i].filename[0] != 0 && strlen(cache[i].filename) == strlen(filename) && + strcmp(cache[i].filename, filename) == 0) { + if (cache[i].is_updating) { + return 0; + } else { + break; + } + } + } + + memset(cache[i].meta.etag, 0, sizeof(cache[i].meta.etag)); + memset(cache[i].meta.filename_comp, 0, sizeof(cache[i].meta.filename_comp)); + cache[i].is_updating = 0; + + shmdt(shm_rw); + cache = cache_ro; + return 0; +} + int uri_cache_init(http_uri *uri) { if (uri->filename == NULL) { return 0; @@ -208,7 +300,7 @@ int uri_cache_init(http_uri *uri) { if (uri->meta == NULL) { for (i = 0; i < FILE_CACHE_SIZE; i++) { if (cache[i].filename[0] == 0) { - if (cache_update_entry(i, uri->filename) != 0) { + if (cache_update_entry(i, uri->filename, uri->webroot) != 0) { return -1; } uri->meta = &cache[i].meta; @@ -219,7 +311,7 @@ int uri_cache_init(http_uri *uri) { struct stat statbuf; stat(uri->filename, &statbuf); if (memcmp(&uri->meta->stat.st_mtime, &statbuf.st_mtime, sizeof(statbuf.st_mtime)) != 0) { - if (cache_update_entry(i, uri->filename) != 0) { + if (cache_update_entry(i, uri->filename, uri->webroot) != 0) { return -1; } } diff --git a/src/cache.h b/src/cache.h index b1300a4..3e2acd3 100644 --- a/src/cache.h +++ b/src/cache.h @@ -17,6 +17,7 @@ magic_t magic; typedef struct { char filename[256]; + unsigned char webroot_len; unsigned char is_updating:1; meta_data meta; } cache_entry; @@ -35,7 +36,9 @@ int cache_init(); int cache_unload(); -int cache_update_entry(int entry_num, const char *filename); +int cache_update_entry(int entry_num, const char *filename, const char *webroot); + +int cache_filename_comp_invalid(const char *filename); int uri_cache_init(http_uri *uri); diff --git a/src/client.c b/src/client.c index a77769b..1a8ac1f 100644 --- a/src/client.c +++ b/src/client.c @@ -209,8 +209,13 @@ int client_request_handler(sock *client, int req_num) { char *accept_encoding = http_get_header_field(&req.hdr, "Accept-Encoding", HTTP_LOWER); if (uri.meta->filename_comp[0] != 0 && accept_encoding != NULL && strstr(accept_encoding, "deflate") != NULL) { file = fopen(uri.meta->filename_comp, "rb"); + if (file == NULL) { + cache_filename_comp_invalid(uri.filename); + goto not_compressed; + } http_add_header_field(&res.hdr, "Content-Encoding", "deflate"); } else { + not_compressed: file = fopen(uri.filename, "rb"); } fseek(file, 0, SEEK_END); diff --git a/src/necronda-server.h b/src/necronda-server.h index 101c65b..81b682e 100644 --- a/src/necronda-server.h +++ b/src/necronda-server.h @@ -53,6 +53,7 @@ #define NECRONDA_VERSION "4.0" #define SERVER_STR "Necronda/" NECRONDA_VERSION #define NECRONDA_DEFAULT "www.necronda.net" +#define NECRONDA_ZLIB_LEVEL 9 #define MAGIC_FILE "/usr/share/file/misc/magic.mgc"