summaryrefslogtreecommitdiff
path: root/src/lib/ssm/pool.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/ssm/pool.c')
-rw-r--r--src/lib/ssm/pool.c122
1 files changed, 53 insertions, 69 deletions
diff --git a/src/lib/ssm/pool.c b/src/lib/ssm/pool.c
index 5c98b515..705de147 100644
--- a/src/lib/ssm/pool.c
+++ b/src/lib/ssm/pool.c
@@ -24,6 +24,7 @@
#include "config.h"
+#include <ouroboros/atomics.h>
#include <ouroboros/errno.h>
#include <ouroboros/pthread.h>
#include <ouroboros/ssm_pool.h>
@@ -37,10 +38,20 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <time.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
+static __inline__ uint64_t pool_now_ns(void)
+{
+ struct timespec ts;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+
+ return (uint64_t) ts.tv_sec * 1000000000ULL + (uint64_t) ts.tv_nsec;
+}
+
/* Global Shared Packet Pool (GSPP) configuration */
static const struct ssm_size_class_cfg ssm_gspp_cfg[SSM_POOL_MAX_CLASSES] = {
{ (1 << 8), SSM_GSPP_256_BLOCKS },
@@ -75,26 +86,6 @@ static const struct ssm_size_class_cfg ssm_pup_cfg[SSM_POOL_MAX_CLASSES] = {
#define GET_SHARD_FOR_PID(pid) ((int)((pid) % SSM_POOL_SHARDS))
-#define LOAD_RELAXED(ptr) \
- (__atomic_load_n(ptr, __ATOMIC_RELAXED))
-
-#define LOAD_ACQUIRE(ptr) \
- (__atomic_load_n(ptr, __ATOMIC_ACQUIRE))
-
-#define STORE_RELEASE(ptr, val) \
- (__atomic_store_n(ptr, val, __ATOMIC_RELEASE))
-
-#define LOAD(ptr) \
- (__atomic_load_n(ptr, __ATOMIC_SEQ_CST))
-
-#define STORE(ptr, val) \
- (__atomic_store_n(ptr, val, __ATOMIC_SEQ_CST))
-
-#define FETCH_ADD(ptr, val) \
- (__atomic_fetch_add(ptr, val, __ATOMIC_SEQ_CST))
-
-#define FETCH_SUB(ptr, val) \
- (__atomic_fetch_sub(ptr, val, __ATOMIC_SEQ_CST))
#define SSM_FILE_SIZE (SSM_POOL_TOTAL_SIZE + sizeof(struct _ssm_pool_hdr))
#define SSM_GSPP_FILE_SIZE (SSM_GSPP_TOTAL_SIZE + sizeof(struct _ssm_pool_hdr))
@@ -107,6 +98,8 @@ static const struct ssm_size_class_cfg ssm_pup_cfg[SSM_POOL_MAX_CLASSES] = {
: SSM_PUP_FILE_SIZE)
#define GET_POOL_CFG(uid) (IS_GSPP(uid) ? ssm_gspp_cfg : ssm_pup_cfg)
+#define NEEDS_CHOWN(uid, gid) ((uid) != geteuid() || (gid) != getegid())
+
struct ssm_pool {
uint8_t * shm_base; /* start of blocks */
struct _ssm_pool_hdr * hdr; /* shared memory header */
@@ -163,29 +156,6 @@ static __inline__ void list_add_head(struct _ssm_list_head * head,
STORE(&head->count, LOAD(&head->count) + 1);
}
-static __inline__ int select_size_class(struct ssm_pool * pool,
- size_t len)
-{
- size_t sz;
- int i;
-
- assert(pool != NULL);
-
- /* Total space needed: header + headspace + data + tailspace */
- sz = sizeof(struct ssm_pk_buff) + SSM_PK_BUFF_HEADSPACE + len
- + SSM_PK_BUFF_TAILSPACE;
-
- for (i = 0; i < SSM_POOL_MAX_CLASSES; i++) {
- struct _ssm_size_class * sc;
-
- sc = &pool->hdr->size_classes[i];
- if (sc->object_size > 0 && sz <= sc->object_size)
- return i;
- }
-
- return -1;
-}
-
static __inline__ int find_size_class_for_offset(struct ssm_pool * pool,
size_t offset)
{
@@ -276,6 +246,7 @@ static void init_size_classes(struct ssm_pool * pool)
STORE(&blk->refcount, 0);
blk->allocator_pid = 0;
+ blk->alloc_ts = 0;
STORE(&blk->next_offset, 0);
list_add_head(&sc->shards[0].free_list, blk,
@@ -306,19 +277,31 @@ static size_t reclaim_pid_from_sc(struct _ssm_size_class * sc,
size_t i;
size_t recovered = 0;
struct ssm_pk_buff * blk;
+ uint64_t now;
+ uint64_t min_age_ns;
- region = (uint8_t *) pool_base + sc->pool_start;
+ region = (uint8_t *) pool_base + sc->pool_start;
+ now = pool_now_ns();
+ min_age_ns = (uint64_t) SSM_POOL_RECLAIM_AGE_S * 1000000000ULL;
for (i = 0; i < sc->object_count; ++i) {
blk = (struct ssm_pk_buff *)(region + i * sc->object_size);
- if (blk->allocator_pid == pid && LOAD(&blk->refcount) > 0) {
- STORE(&blk->refcount, 0);
- blk->allocator_pid = 0;
- list_add_head(&shard->free_list, blk, pool_base);
- FETCH_ADD(&shard->free_count, 1);
- recovered++;
- }
+ if (blk->allocator_pid != pid)
+ continue;
+
+ if (LOAD(&blk->refcount) == 0)
+ continue;
+
+ /* Recent: a live consumer may still hold the handoff. */
+ if (now - blk->alloc_ts < min_age_ns)
+ continue;
+
+ STORE(&blk->refcount, 0);
+ blk->allocator_pid = 0;
+ list_add_head(&shard->free_list, blk, pool_base);
+ FETCH_ADD(&shard->free_count, 1);
+ recovered++;
}
return recovered;
@@ -379,6 +362,7 @@ static __inline__ ssize_t init_block(struct ssm_pool * pool,
{
STORE(&blk->refcount, 1);
blk->allocator_pid = getpid();
+ blk->alloc_ts = pool_now_ns();
blk->size = (uint32_t) (sc->object_size -
sizeof(struct ssm_pk_buff));
blk->pk_head = SSM_PK_BUFF_HEADSPACE;
@@ -548,7 +532,7 @@ static struct ssm_pool * __pool_create(const char * name,
if (flags & O_CREAT) {
if (ftruncate(fd, (off_t) file_size) < 0)
goto fail_truncate;
- if (uid != geteuid() && fchown(fd, uid, gid) < 0)
+ if (NEEDS_CHOWN(uid, gid) && fchown(fd, uid, gid) < 0)
goto fail_truncate;
}
@@ -700,7 +684,7 @@ ssize_t ssm_pool_alloc(struct ssm_pool * pool,
assert(pool != NULL);
assert(spb != NULL);
- idx = select_size_class(pool, count);
+ idx = select_size_class(pool->hdr, count);
if (idx >= 0)
return alloc_from_sc(pool, idx, count, ptr, spb);
@@ -718,7 +702,7 @@ ssize_t ssm_pool_alloc_b(struct ssm_pool * pool,
assert(pool != NULL);
assert(spb != NULL);
- idx = select_size_class(pool, count);
+ idx = select_size_class(pool->hdr, count);
if (idx >= 0)
return alloc_from_sc_b(pool, idx, count, ptr, spb, abstime);
@@ -744,7 +728,7 @@ ssize_t ssm_pool_read(uint8_t ** dst,
}
struct ssm_pk_buff * ssm_pool_get(struct ssm_pool * pool,
- size_t off)
+ size_t off)
{
struct ssm_pk_buff * blk;
@@ -823,36 +807,36 @@ int ssm_pool_remove(struct ssm_pool * pool,
return 0;
}
-size_t ssm_pk_buff_get_idx(struct ssm_pk_buff * spb)
+size_t ssm_pk_buff_get_off(const struct ssm_pk_buff * spb)
{
assert(spb != NULL);
return spb->off;
}
-uint8_t * ssm_pk_buff_head(struct ssm_pk_buff * spb)
+uint8_t * ssm_pk_buff_head(const struct ssm_pk_buff * spb)
{
assert(spb != NULL);
- return spb->data + spb->pk_head;
+ return (uint8_t *) spb->data + spb->pk_head;
}
-uint8_t * ssm_pk_buff_tail(struct ssm_pk_buff * spb)
+uint8_t * ssm_pk_buff_tail(const struct ssm_pk_buff * spb)
{
assert(spb != NULL);
- return spb->data + spb->pk_tail;
+ return (uint8_t *) spb->data + spb->pk_tail;
}
-size_t ssm_pk_buff_len(struct ssm_pk_buff * spb)
+size_t ssm_pk_buff_len(const struct ssm_pk_buff * spb)
{
assert(spb != NULL);
return spb->pk_tail - spb->pk_head;
}
-uint8_t * ssm_pk_buff_head_alloc(struct ssm_pk_buff * spb,
- size_t size)
+uint8_t * ssm_pk_buff_push(struct ssm_pk_buff * spb,
+ size_t size)
{
assert(spb != NULL);
@@ -864,8 +848,8 @@ uint8_t * ssm_pk_buff_head_alloc(struct ssm_pk_buff * spb,
return spb->data + spb->pk_head;
}
-uint8_t * ssm_pk_buff_tail_alloc(struct ssm_pk_buff * spb,
- size_t size)
+uint8_t * ssm_pk_buff_push_tail(struct ssm_pk_buff * spb,
+ size_t size)
{
uint8_t * buf;
@@ -881,8 +865,8 @@ uint8_t * ssm_pk_buff_tail_alloc(struct ssm_pk_buff * spb,
return buf;
}
-uint8_t * ssm_pk_buff_head_release(struct ssm_pk_buff * spb,
- size_t size)
+uint8_t * ssm_pk_buff_pop(struct ssm_pk_buff * spb,
+ size_t size)
{
uint8_t * buf;
@@ -896,8 +880,8 @@ uint8_t * ssm_pk_buff_head_release(struct ssm_pk_buff * spb,
return buf;
}
-uint8_t * ssm_pk_buff_tail_release(struct ssm_pk_buff * spb,
- size_t size)
+uint8_t * ssm_pk_buff_pop_tail(struct ssm_pk_buff * spb,
+ size_t size)
{
assert(spb != NULL);
assert(!(size > spb->pk_tail - spb->pk_head));
generated by cgit v1.2.3 (git 2.39.1) at 2026-08-05 07:50:46 +0000