mpmc: Use volatile modifier on buffer differently

src/async.c

@@ -21,7 +21,8 @@
 #include <unistd.h>
 #include <openssl/ssl.h>
 
-#define ASYNC_MAX_EVENTS 16
+#define ASYNC_QUEUE_MAX_EVENTS 256
+#define ASYNC_EPOLL_MAX_EVENTS 64
 
 typedef struct {
     int fd;
@@ -36,7 +37,7 @@ typedef struct {
 
 typedef struct {
     int n;
-    evt_listen_t *q[ASYNC_MAX_EVENTS];
+    evt_listen_t *q[ASYNC_QUEUE_MAX_EVENTS];
 } listen_queue_t;
 
 static volatile listen_queue_t listen1, listen2, *listen_q = &listen1;
@@ -123,7 +124,13 @@ static int async_add_to_queue(evt_listen_t *evt) {
         }
     }
 
-    evt_listen_t *ptr = malloc(sizeof(evt_listen_t));
+    if (listen_q->n >= ASYNC_QUEUE_MAX_EVENTS) {
+        sem_post(&lock);
+        errno = ENOBUFS;
+        return -1;
+    }
+
+    evt_listen_t *ptr = malloc(sizeof(*evt));
     if (ptr == NULL) {
         sem_post(&lock);
         return -1;
@@ -198,9 +205,12 @@ static int async_add(evt_listen_t *evt) {
     if (async_check(evt) == 1)
         return 0;
 
-    int ret = async_add_to_queue(evt);
-    if (ret == 0 && thread != -1)
+    int ret;
+    if ((ret = async_add_to_queue(evt)) != 0) {
+        alert("Unable to add event to async queue");
+    } else if (thread != -1) {
         pthread_kill(thread, SIGUSR1);
+    }
 
     return ret;
 }
@@ -257,7 +267,7 @@ void async_free(void) {
 }
 
 void async_thread(void) {
-    struct epoll_event ev, events[ASYNC_MAX_EVENTS];
+    struct epoll_event ev, events[ASYNC_EPOLL_MAX_EVENTS];
     int num_fds, idx;
     long ts, min_ts, cur_ts;
     volatile listen_queue_t *l;
@@ -343,7 +353,7 @@ void async_thread(void) {
         }
 
         // epoll is used in level-triggered mode, so buffers are taken into account
-        if ((num_fds = epoll_wait(epoll_fd, events, ASYNC_MAX_EVENTS, (int) (min_ts / 1000))) == -1) {
+        if ((num_fds = epoll_wait(epoll_fd, events, ASYNC_EPOLL_MAX_EVENTS, (int) (min_ts / 1000))) == -1) {
             if (errno == EINTR) {
                 // interrupt
                 errno = 0;

src/lib/mpmc.c

@@ -10,7 +10,7 @@
 
 static void *mpmc_worker(void *arg);
 
-int mpmc_init(mpmc_t *ctx, int n_workers, int buf_size, void (*consumer)(void *obj), const char *name) {
+int mpmc_init(mpmc_t *ctx, const int n_workers, const int buf_size, void (*consumer)(void *), const char *name) {
     ctx->alive = 1;
     ctx->n_workers = n_workers;
     ctx->size = buf_size, ctx->max_size = buf_size;
@@ -35,8 +35,8 @@ int mpmc_init(mpmc_t *ctx, int n_workers, int buf_size, void (*consumer)(void *o
         return -1;
     }
 
-    memset(ctx->buffer,   0, ctx->size      * sizeof(void *));
-    memset(ctx->workers, -1, ctx->n_workers * sizeof(pthread_t));
+    memset((void *)ctx->buffer, 0, ctx->size      * sizeof(void *));
+    memset(ctx->workers,       -1, ctx->n_workers * sizeof(pthread_t));
 
     for (int i = 0; i < ctx->n_workers; i++) {
         int ret;
@@ -72,7 +72,7 @@ int mpmc_queue(mpmc_t *ctx, void *obj) {
         }
     }
 
-    int p = ctx->wr;
+    const int p = ctx->wr;
     ctx->wr = (ctx->wr + 1) % ctx->size;
 
     // unlock wr field
@@ -121,7 +121,7 @@ static void *mpmc_worker(void *arg) {
             }
         }
 
-        int p = ctx->rd;
+        const int p = ctx->rd;
         ctx->rd = (ctx->rd + 1) % ctx->size;
 
         // unlock rd field
@@ -143,9 +143,9 @@ void mpmc_stop(mpmc_t *ctx) {
 }
 
 void mpmc_destroy(mpmc_t *ctx) {
-    int e = errno;
+    const int e = errno;
 
-    // stop threads, if running
+    // stop threads if running
     mpmc_stop(ctx);
     for (int i = 0; i < ctx->n_workers; i++) {
         if (ctx->workers[i] == -1) break;
@@ -158,7 +158,7 @@ void mpmc_destroy(mpmc_t *ctx) {
     sem_destroy(&ctx->used);
     sem_destroy(&ctx->lck_rd);
     sem_destroy(&ctx->lck_wr);
-    free(ctx->buffer);
+    free((void *)ctx->buffer);
     free(ctx->workers);
 
     // reset errno

src/lib/mpmc.h

@@ -5,18 +5,18 @@
 #include <semaphore.h>
 
 typedef struct {
-    unsigned char alive;
+    volatile unsigned int alive:1;
     int n_workers;
-    int rd, wr;
+    volatile int rd, wr;
     sem_t free, used, lck_rd, lck_wr;
     int size, max_size;
-    void **buffer;
+    void *volatile *buffer;
     pthread_t *workers;
     void (*consumer)(void *obj);
     const char* name;
 } mpmc_t;
 
-int mpmc_init(mpmc_t *ctx, int n_workers, int buf_size, void (*consumer)(void *obj), const char *name);
+int mpmc_init(mpmc_t *ctx, int n_workers, int buf_size, void (*consumer)(void *), const char *name);
 
 int mpmc_queue(mpmc_t *ctx, void *obj);
 

src/server.c

@@ -411,7 +411,19 @@ int main(int argc, char *const argv[]) {
 
     logger_set_name("main");
 
-    workers_init();
+    if (workers_init() != 0) {
+        critical("Unable to initialize workers");
+        ssl_free();
+        list_free(clients);
+        sem_destroy(&sem_clients_lock);
+        geoip_free();
+        proxy_unload();
+        cache_join();
+        async_free();
+        logger_stop();
+        logger_join();
+        return 1;
+    }
 
     for (int i = 0; i < NUM_SOCKETS; i++) {
         async_fd(sockets[i], ASYNC_WAIT_READ, ASYNC_KEEP, &sockets[i], accept_cb, accept_err_cb, accept_err_cb);

src/server.h

@@ -12,8 +12,8 @@
 #include "worker/func.h"
 
 #define NUM_SOCKETS 2
-#define LISTEN_BACKLOG 16
-#define REQ_PER_CONNECTION 200
+#define LISTEN_BACKLOG 256
+#define REQ_PER_CONNECTION 256
 
 #define SOCKET_TIMEOUT 1
 #define CLIENT_TIMEOUT 3600
@@ -21,9 +21,6 @@
 #define SERVER_SOCKET_TIMEOUT_RES 60
 #define SERVER_TIMEOUT 3600
 
-#define CNX_HANDLER_WORKERS 8
-#define REQ_HANDLER_WORKERS 16
-
 void server_free_client(client_ctx_t *ctx);
 
 #endif //SESIMOS_SERVER_H

src/worker/fastcgi_handler.c

@@ -34,6 +34,9 @@ void fastcgi_handler_func(client_ctx_t *ctx) {
         fastcgi_close(ctx->fcgi_ctx);
         ctx->fcgi_ctx = NULL;
     }
+
+    request_complete(ctx);
+    handle_request(ctx);
 }
 
 static int fastcgi_handler_1(client_ctx_t *ctx) {

src/workers.c

@@ -16,16 +16,26 @@ static mpmc_t tcp_acceptor_ctx, request_handler_ctx, local_handler_ctx, fastcgi_
               proxy_peer_handler_ctx, ws_frame_handler_ctx, chunk_handler_ctx, fastcgi_frame_handler_ctx;
 
 int workers_init(void) {
-    mpmc_init(&tcp_acceptor_ctx,          8, 64, (void (*)(void *)) tcp_acceptor_func,          "tcp");
-    mpmc_init(&request_handler_ctx,       8, 64, (void (*)(void *)) request_handler_func,       "req");
-    mpmc_init(&local_handler_ctx,         8, 64, (void (*)(void *)) local_handler_func,         "local");
-    mpmc_init(&fastcgi_handler_ctx,       8, 64, (void (*)(void *)) fastcgi_handler_func,       "fcgi");
-    mpmc_init(&proxy_handler_ctx,         8, 64, (void (*)(void *)) proxy_handler_func,         "proxy");
-    mpmc_init(&proxy_peer_handler_ctx,    1,  8, (void (*)(void *)) proxy_peer_handler_func,    "prxy_p");
-    mpmc_init(&ws_frame_handler_ctx,      8, 64, (void (*)(void *)) ws_frame_handler_func,      "ws");
-    mpmc_init(&chunk_handler_ctx,         8, 64, (void (*)(void *)) chunk_handler_func,         "chunk");
-    mpmc_init(&fastcgi_frame_handler_ctx, 8, 64, (void (*)(void *)) fastcgi_frame_handler_func, "fcgi_f");
-    return -1;
+    int ret;
+    if ((ret = mpmc_init(&tcp_acceptor_ctx,          8, 64, (void (*)(void *)) tcp_acceptor_func,          "tcp")) != 0)
+        return ret;
+    if ((ret = mpmc_init(&request_handler_ctx,       8, 64, (void (*)(void *)) request_handler_func,       "req")) != 0)
+        return ret;
+    if ((ret = mpmc_init(&local_handler_ctx,         8, 64, (void (*)(void *)) local_handler_func,         "local")) != 0)
+        return ret;
+    if ((ret = mpmc_init(&fastcgi_handler_ctx,       8, 64, (void (*)(void *)) fastcgi_handler_func,       "fcgi")) != 0)
+        return ret;
+    if ((ret = mpmc_init(&proxy_handler_ctx,         8, 64, (void (*)(void *)) proxy_handler_func,         "proxy")) != 0)
+        return ret;
+    if ((ret = mpmc_init(&proxy_peer_handler_ctx,    1,  8, (void (*)(void *)) proxy_peer_handler_func,    "prxy_p")) != 0)
+        return ret;
+    if ((ret = mpmc_init(&ws_frame_handler_ctx,      8, 64, (void (*)(void *)) ws_frame_handler_func,      "ws")) != 0)
+        return ret;
+    if ((ret = mpmc_init(&chunk_handler_ctx,         8, 64, (void (*)(void *)) chunk_handler_func,         "chunk")) != 0)
+        return ret;
+    if ((ret = mpmc_init(&fastcgi_frame_handler_ctx, 8, 64, (void (*)(void *)) fastcgi_frame_handler_func, "fcgi_f")) != 0)
+        return ret;
+    return 0;
 }
 
 void workers_stop(void) {