lorenz.stechauner/sesimos
1
1
Fork 0
Code Issues Pull Requests Projects Releases Activity

Added brotli compression

Browse Source
This commit is contained in:
Lorenz Stechauner
2021-05-04 22:32:21 +02:00
parent 10d405e745
commit 80986325ce
24 changed files with 392 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
src/lib/cache.c
View File

@ -7,9 +7,9 @@
#include "cache.h"
#include "utils.h"
#include "../necronda-server.h"
#include "gzip.h"
#include "brotli.h"
#include <stdio.h>
#include <zlib.h>
#include <magic.h>
#include <sys/ipc.h>
#include <sys/shm.h>
@ -28,7 +28,7 @@ int magic_init() {
fprintf(stderr, ERR_STR "Unable to open magic cookie: %s" CLR_STR "\n", strerror(errno));
return -1;
}
if (magic_load(magic, MAGIC_FILE) != 0) {
if (magic_load(magic, CACHE_MAGIC_FILE) != 0) {
fprintf(stderr, ERR_STR "Unable to load magic cookie: %s" CLR_STR "\n", magic_error(magic));
return -2;
}
@ -43,7 +43,7 @@ int cache_process() {
signal(SIGINT, cache_process_term);
signal(SIGTERM, cache_process_term);
int shm_id = shmget(SHM_KEY_CACHE, FILE_CACHE_SIZE * sizeof(cache_entry), 0);
int shm_id = shmget(CACHE_SHM_KEY, CACHE_ENTRIES * sizeof(cache_entry), 0);
if (shm_id < 0) {
fprintf(stderr, ERR_STR "Unable to create shared memory: %s" CLR_STR "\n", strerror(errno));
return -1;
@ -66,26 +66,28 @@ int cache_process() {
FILE *cache_file = fopen("/var/necronda-server/cache", "rb");
if (cache_file != NULL) {
fread(cache, sizeof(cache_entry), FILE_CACHE_SIZE, cache_file);
fread(cache, sizeof(cache_entry), CACHE_ENTRIES, cache_file);
fclose(cache_file);
}
for (int i = 0; i < FILE_CACHE_SIZE; i++) {
for (int i = 0; i < CACHE_ENTRIES; i++) {
cache[i].is_updating = 0;
}
FILE *file;
char buf[16384];
char comp_buf[16384];
char filename_comp[256];
char filename_comp_gz[256];
char filename_comp_br[256];
unsigned long read;
int compress;
SHA_CTX ctx;
unsigned char hash[SHA_DIGEST_LENGTH];
int cache_changed = 0;
int p_len;
int p_len_gz, p_len_br;
int ret_1, ret_2;
while (cache_continue) {
for (int i = 0; i < FILE_CACHE_SIZE; i++) {
for (int i = 0; i < CACHE_ENTRIES; i++) {
if (cache[i].filename[0] != 0 && cache[i].meta.etag[0] == 0 && !cache[i].is_updating) {
cache[i].is_updating = 1;
fprintf(stdout, "[cache] Hashing file %s\n", cache[i].filename);
@ -93,9 +95,10 @@ int cache_process() {
file = fopen(cache[i].filename, "rb");
compress = mime_is_compressible(cache[i].meta.type);
int level = NECRONDA_ZLIB_LEVEL;
z_stream strm;
FILE *comp_file = NULL;
z_stream gz_state;
BrotliEncoderState *br_state = NULL;
FILE *comp_file_gz = NULL;
FILE *comp_file_br = NULL;
if (compress) {
sprintf(buf, "%.*s/.necronda-server", cache[i].webroot_len, cache[i].filename);
mkdir(buf, 0755);
@ -110,27 +113,41 @@ int cache_process() {
buf[j] = ch;
}
buf[strlen(rel_path)] = 0;
p_len = snprintf(filename_comp, sizeof(filename_comp), "%.*s/.necronda-server/cache/%s.z",
cache[i].webroot_len, cache[i].filename, buf);
if (p_len < 0 || p_len >= sizeof(filename_comp)) {
p_len_gz = snprintf(filename_comp_gz, sizeof(filename_comp_gz),
"%.*s/.necronda-server/cache/%s.gz",
cache[i].webroot_len, cache[i].filename, buf);
p_len_br = snprintf(filename_comp_br, sizeof(filename_comp_br),
"%.*s/.necronda-server/cache/%s.br",
cache[i].webroot_len, cache[i].filename, buf);
if (p_len_gz < 0 || p_len_gz >= sizeof(filename_comp_gz) ||
p_len_br < 0 || p_len_br >= sizeof(filename_comp_br)) {
fprintf(stderr, ERR_STR "Unable to open cached file: "
"File name for compressed file too long" CLR_STR "\n");
goto comp_err;
}
fprintf(stdout, "[cache] Compressing file %s\n", cache[i].filename);
comp_file = fopen(filename_comp, "wb");
if (comp_file == NULL) {
comp_file_gz = fopen(filename_comp_gz, "wb");
comp_file_br = fopen(filename_comp_br, "wb");
if (comp_file_gz == NULL || comp_file_br == NULL) {
fprintf(stderr, ERR_STR "Unable to open cached file: %s" CLR_STR "\n", strerror(errno));
comp_err:
compress = 0;
} else {
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
if (deflateInit(&strm, level) != Z_OK) {
fprintf(stderr, ERR_STR "Unable to init deflate: %s" CLR_STR "\n", strerror(errno));
ret_1 = gzip_init(&gz_state);
ret_2 = brotli_init(&br_state);
if (ret_1 != 0) {
fprintf(stderr, ERR_STR "Unable to init gzip: %s" CLR_STR "\n", strerror(errno));
compress = 0;
fclose(comp_file);
fclose(comp_file_gz);
fclose(comp_file_br);
} else if (ret_2 != 0) {
fprintf(stderr, ERR_STR "Unable to init brotli: %s" CLR_STR "\n", strerror(errno));
compress = 0;
fclose(comp_file_gz);
fclose(comp_file_br);
}
}
}
@ -138,25 +155,33 @@ int cache_process() {
while ((read = fread(buf, 1, sizeof(buf), file)) > 0) {
SHA1_Update(&ctx, buf, read);
if (compress) {
strm.avail_in = read;
strm.next_in = (unsigned char *) buf;
unsigned long avail_in, avail_out;
avail_in = read;
do {
strm.avail_out = sizeof(comp_buf);
strm.next_out = (unsigned char *) comp_buf;
deflate(&strm, feof(file) ? Z_FINISH : Z_NO_FLUSH);
fwrite(comp_buf, 1, sizeof(comp_buf) - strm.avail_out, comp_file);
strm.avail_in = 0;
} while (strm.avail_out == 0);
avail_out = sizeof(comp_buf);
gzip_compress(&gz_state, buf + read - avail_in, &avail_in, comp_buf, &avail_out, feof(file));
fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_gz);
} while (avail_in != 0);
avail_in = read;
do {
avail_out = sizeof(comp_buf);
brotli_compress(br_state, buf + read - avail_in, &avail_in, comp_buf, &avail_out, feof(file));
fwrite(comp_buf, 1, sizeof(comp_buf) - avail_out, comp_file_br);
} while (avail_in != 0);
}
}
if (compress) {
deflateEnd(&strm);
fclose(comp_file);
gzip_free(&gz_state);
brotli_free(br_state);
fclose(comp_file_gz);
fclose(comp_file_br);
fprintf(stdout, "[cache] Finished compressing file %s\n", cache[i].filename);
strcpy(cache[i].meta.filename_comp, filename_comp);
strcpy(cache[i].meta.filename_comp_gz, filename_comp_gz);
strcpy(cache[i].meta.filename_comp_br, filename_comp_br);
} else {
memset(cache[i].meta.filename_comp, 0, sizeof(cache[i].meta.filename_comp));
memset(cache[i].meta.filename_comp_gz, 0, sizeof(cache[i].meta.filename_comp_gz));
memset(cache[i].meta.filename_comp_br, 0, sizeof(cache[i].meta.filename_comp_br));
}
SHA1_Final(hash, &ctx);
memset(cache[i].meta.etag, 0, sizeof(cache[i].meta.etag));
@ -173,7 +198,11 @@ int cache_process() {
if (cache_changed) {
cache_changed = 0;
cache_file = fopen("/var/necronda-server/cache", "wb");
fwrite(cache, sizeof(cache_entry), FILE_CACHE_SIZE, cache_file);
if (cache_file == NULL) {
fprintf(stderr, ERR_STR "Unable to open cache file: %s" CLR_STR "\n", strerror(errno));
return -1;
}
fwrite(cache, sizeof(cache_entry), CACHE_ENTRIES, cache_file);
fclose(cache_file);
} else {
sleep(1);
@ -187,7 +216,7 @@ int cache_init() {
return -1;
}
int shm_id = shmget(SHM_KEY_CACHE, FILE_CACHE_SIZE * sizeof(cache_entry), IPC_CREAT | IPC_EXCL | 0600);
int shm_id = shmget(CACHE_SHM_KEY, CACHE_ENTRIES * sizeof(cache_entry), IPC_CREAT | IPC_EXCL | 0600);
if (shm_id < 0) {
fprintf(stderr, ERR_STR "Unable to create shared memory: %s" CLR_STR "\n", strerror(errno));
return -2;
@ -206,7 +235,7 @@ int cache_init() {
return -4;
}
cache = shm_rw;
memset(cache, 0, FILE_CACHE_SIZE * sizeof(cache_entry));
memset(cache, 0, CACHE_ENTRIES * sizeof(cache_entry));
shmdt(shm_rw);
cache = shm;
@ -229,7 +258,7 @@ int cache_init() {
}
int cache_unload() {
int shm_id = shmget(SHM_KEY_CACHE, 0, 0);
int shm_id = shmget(CACHE_SHM_KEY, 0, 0);
if (shm_id < 0) {
fprintf(stderr, ERR_STR "Unable to get shared memory id: %s" CLR_STR "\n", strerror(errno));
shmdt(cache);
@ -245,7 +274,7 @@ int cache_unload() {
int cache_update_entry(int entry_num, const char *filename, const char *webroot) {
void *cache_ro = cache;
int shm_id = shmget(SHM_KEY_CACHE, 0, 0);
int shm_id = shmget(CACHE_SHM_KEY, 0, 0);
void *shm_rw = shmat(shm_id, NULL, 0);
if (shm_rw == (void *) -1) {
print(ERR_STR "Unable to attach shared memory (rw): %s" CLR_STR, strerror(errno));
@ -277,7 +306,8 @@ int cache_update_entry(int entry_num, const char *filename, const char *webroot)
strcpy(cache[entry_num].meta.charset, magic_file(magic, filename));
memset(cache[entry_num].meta.etag, 0, sizeof(cache[entry_num].meta.etag));
memset(cache[entry_num].meta.filename_comp, 0, sizeof(cache[entry_num].meta.filename_comp));
memset(cache[entry_num].meta.filename_comp_gz, 0, sizeof(cache[entry_num].meta.filename_comp_gz));
memset(cache[entry_num].meta.filename_comp_br, 0, sizeof(cache[entry_num].meta.filename_comp_br));
cache[entry_num].is_updating = 0;
shmdt(shm_rw);
@ -287,7 +317,7 @@ int cache_update_entry(int entry_num, const char *filename, const char *webroot)
int cache_filename_comp_invalid(const char *filename) {
void *cache_ro = cache;
int shm_id = shmget(SHM_KEY_CACHE, 0, 0);
int shm_id = shmget(CACHE_SHM_KEY, 0, 0);
void *shm_rw = shmat(shm_id, NULL, 0);
if (shm_rw == (void *) -1) {
print(ERR_STR "Unable to attach shared memory (rw): %s" CLR_STR, strerror(errno));
@ -296,7 +326,7 @@ int cache_filename_comp_invalid(const char *filename) {
cache = shm_rw;
int i;
for (i = 0; i < FILE_CACHE_SIZE; i++) {
for (i = 0; i < CACHE_ENTRIES; i++) {
if (cache[i].filename[0] != 0 && strlen(cache[i].filename) == strlen(filename) &&
strcmp(cache[i].filename, filename) == 0) {
if (cache[i].is_updating) {
@ -308,7 +338,8 @@ int cache_filename_comp_invalid(const char *filename) {
}
memset(cache[i].meta.etag, 0, sizeof(cache[i].meta.etag));
memset(cache[i].meta.filename_comp, 0, sizeof(cache[i].meta.filename_comp));
memset(cache[i].meta.filename_comp_gz, 0, sizeof(cache[i].meta.filename_comp_gz));
memset(cache[i].meta.filename_comp_br, 0, sizeof(cache[i].meta.filename_comp_br));
cache[i].is_updating = 0;
shmdt(shm_rw);
@ -322,7 +353,7 @@ int uri_cache_init(http_uri *uri) {
}
int i;
for (i = 0; i < FILE_CACHE_SIZE; i++) {
for (i = 0; i < CACHE_ENTRIES; i++) {
if (cache[i].filename[0] != 0 && strlen(cache[i].filename) == strlen(uri->filename) &&
strcmp(cache[i].filename, uri->filename) == 0) {
uri->meta = &cache[i].meta;
@ -335,7 +366,7 @@ int uri_cache_init(http_uri *uri) {
}
if (uri->meta == NULL) {
for (i = 0; i < FILE_CACHE_SIZE; i++) {
for (i = 0; i < CACHE_ENTRIES; i++) {
if (cache[i].filename[0] == 0) {
if (cache_update_entry(i, uri->filename, uri->webroot) != 0) {
return -1;