diff options
Diffstat (limited to 'src/ipcpd/unicast')
63 files changed, 5930 insertions, 1126 deletions
diff --git a/src/ipcpd/unicast/CMakeLists.txt b/src/ipcpd/unicast/CMakeLists.txt index a9155353..2373f877 100644 --- a/src/ipcpd/unicast/CMakeLists.txt +++ b/src/ipcpd/unicast/CMakeLists.txt @@ -1,44 +1,12 @@ -get_filename_component(CURRENT_SOURCE_PARENT_DIR - ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) -get_filename_component(CURRENT_BINARY_PARENT_DIR - ${CMAKE_CURRENT_BINARY_DIR} DIRECTORY) +# Unicast IPCP build configuration -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) +protobuf_generate_c(DHT_PROTO_SRCS DHT_PROTO_HDRS + "${CMAKE_CURRENT_SOURCE_DIR}/dir/dht.proto") -include_directories(${CURRENT_SOURCE_PARENT_DIR}) -include_directories(${CURRENT_BINARY_PARENT_DIR}) - -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_BINARY_DIR}/include) - -set(IPCP_UNICAST_TARGET ipcpd-unicast CACHE INTERNAL "") -set(IPCP_UNICAST_MPL 10000 CACHE STRING - "Default maximum packet lifetime for the unicast IPCP, in ms") -set(DEBUG_PROTO_DHT FALSE CACHE BOOL - "Add DHT protocol message output to debug logging") -set(DEBUG_PROTO_LS FALSE CACHE BOOL - "Add link state protocol message output to debug logging") - -protobuf_generate_c(DHT_PROTO_SRCS DHT_PROTO_HDRS dir/dht.proto) - -math(EXPR PFT_EXPR "1 << 12") -set(PFT_SIZE ${PFT_EXPR} CACHE STRING - "Size of the PDU forwarding table") -if (HAVE_FUSE) - set(IPCP_FLOW_STATS TRUE CACHE BOOL - "Enable flow statistics tracking in IPCP") - if (IPCP_FLOW_STATS) - message(STATUS "IPCP flow statistics enabled") - else () - message(STATUS "IPCP flow statistics disabled") - endif () -endif () - -set(IPCP_UNICAST_SOURCE_FILES - # Add source files here +set(UNICAST_SOURCES addr-auth.c ca.c + cap.c connmgr.c dir.c dt.c @@ -47,7 +15,6 @@ set(IPCP_UNICAST_SOURCE_FILES pff.c routing.c psched.c - # Add policies last addr-auth/flat.c ca/mb-ecn.c ca/nop.c @@ -58,21 +25,28 @@ set(IPCP_UNICAST_SOURCE_FILES pff/pft.c routing/link-state.c routing/graph.c - ) +) + +add_executable(${IPCP_UNICAST_TARGET} + ${UNICAST_SOURCES} + ${IPCP_SOURCES} + ${COMMON_SOURCES} + ${DHT_PROTO_SRCS} +) -add_executable(ipcpd-unicast ${IPCP_UNICAST_SOURCE_FILES} ${IPCP_SOURCES} ${COMMON_SOURCES} - ${DHT_PROTO_SRCS} ${LAYER_CONFIG_PROTO_SRCS}) -target_link_libraries(ipcpd-unicast LINK_PUBLIC ouroboros-dev) +target_include_directories(${IPCP_UNICAST_TARGET} PRIVATE ${IPCP_INCLUDE_DIRS}) +target_include_directories(${IPCP_UNICAST_TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(${IPCP_UNICAST_TARGET} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries(${IPCP_UNICAST_TARGET} PRIVATE ouroboros-dev) -include(AddCompileFlags) -if (CMAKE_BUILD_TYPE MATCHES "Debug*") - add_compile_flags(ipcpd-unicast -DCONFIG_OUROBOROS_DEBUG) -endif () +ouroboros_target_debug_definitions(${IPCP_UNICAST_TARGET}) -install(TARGETS ipcpd-unicast RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}) +install(TARGETS ${IPCP_UNICAST_TARGET} RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}) if(BUILD_TESTS) + add_subdirectory(ca/tests) + add_subdirectory(dir/tests) add_subdirectory(pff/tests) add_subdirectory(routing/tests) - add_subdirectory(dir/tests) -endif () + add_subdirectory(tests) +endif() diff --git a/src/ipcpd/unicast/addr-auth.c b/src/ipcpd/unicast/addr-auth.c index 908a4aa1..27671d09 100644 --- a/src/ipcpd/unicast/addr-auth.c +++ b/src/ipcpd/unicast/addr-auth.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Address authority * diff --git a/src/ipcpd/unicast/addr-auth.h b/src/ipcpd/unicast/addr-auth.h index 0d2cd4c0..65567dc3 100644 --- a/src/ipcpd/unicast/addr-auth.h +++ b/src/ipcpd/unicast/addr-auth.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Address authority * diff --git a/src/ipcpd/unicast/addr-auth/flat.c b/src/ipcpd/unicast/addr-auth/flat.c index 34ca1cef..cf832279 100644 --- a/src/ipcpd/unicast/addr-auth/flat.c +++ b/src/ipcpd/unicast/addr-auth/flat.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Policy for flat addresses in a distributed way * diff --git a/src/ipcpd/unicast/addr-auth/flat.h b/src/ipcpd/unicast/addr-auth/flat.h index d4b672c7..b5c7e525 100644 --- a/src/ipcpd/unicast/addr-auth/flat.h +++ b/src/ipcpd/unicast/addr-auth/flat.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Policy for flat addresses in a distributed way * diff --git a/src/ipcpd/unicast/addr-auth/ops.h b/src/ipcpd/unicast/addr-auth/ops.h index 06b24cec..f5c4c611 100644 --- a/src/ipcpd/unicast/addr-auth/ops.h +++ b/src/ipcpd/unicast/addr-auth/ops.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Address authority policy ops * diff --git a/src/ipcpd/unicast/addr-auth/pol.h b/src/ipcpd/unicast/addr-auth/pol.h index 844308c6..a33f1dd6 100644 --- a/src/ipcpd/unicast/addr-auth/pol.h +++ b/src/ipcpd/unicast/addr-auth/pol.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Address Authority policies * diff --git a/src/ipcpd/unicast/ca.c b/src/ipcpd/unicast/ca.c index 1fcc9bb2..d0ee2f73 100644 --- a/src/ipcpd/unicast/ca.c +++ b/src/ipcpd/unicast/ca.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Congestion Avoidance * @@ -22,17 +22,47 @@ #define OUROBOROS_PREFIX "ca" +#include "config.h" + +#include <ouroboros/list.h> #include <ouroboros/logs.h> #include "ca.h" #include "ca/pol.h" +#include <pthread.h> +#include <stdlib.h> + +/* + * A ca_ctx holds congestion state for a (peer address, qos cube) PATH, + * not for a flow. In the default build the façade interns one ctx per + * (addr, qc) and shares it across every flow on that path; the policy + * runs on the shared ctx and cannot tell one flow from many. Per-flow + * ctx (IPCP_CA_PER_FLOW) is a testing build only: it skips interning so + * every flow gets its own ctx. + */ + +struct ca_ctx { + uint64_t addr; + qoscube_t qc; + size_t refs; + void * pol; /* policy ctx (ops->ctx_create result) */ + struct list_head next; +}; + struct { - struct ca_ops * ops; + struct ca_ops * ops; +#ifndef IPCP_CA_PER_FLOW + struct list_head buckets[CA_BUCKETS]; + pthread_mutex_t mtx; +#endif } ca; int ca_init(enum pol_cong_avoid pol) { +#ifndef IPCP_CA_PER_FLOW + size_t i; +#endif switch(pol) { case CA_NONE: log_dbg("Disabling congestion control."); @@ -46,63 +76,167 @@ int ca_init(enum pol_cong_avoid pol) return -1; } +#ifndef IPCP_CA_PER_FLOW + for (i = 0; i < CA_BUCKETS; i++) + list_head_init(&ca.buckets[i]); + + if (pthread_mutex_init(&ca.mtx, NULL) != 0) + return -1; +#endif return 0; } void ca_fini(void) { +#ifndef IPCP_CA_PER_FLOW + size_t i; + + /* Data path is stopped; drain any ctx a flow left interned. */ + for (i = 0; i < CA_BUCKETS; i++) { + struct list_head * p; + struct list_head * h; + + list_for_each_safe(p, h, &ca.buckets[i]) { + struct ca_ctx * ctx; + ctx = list_entry(p, struct ca_ctx, next); + list_del(&ctx->next); + ca.ops->ctx_destroy(ctx->pol); + free(ctx); + } + } + + pthread_mutex_destroy(&ca.mtx); +#endif ca.ops = NULL; } -void * ca_ctx_create(void) +#ifndef IPCP_CA_PER_FLOW +static size_t ca_bucket(uint64_t addr, + qoscube_t qc) { - return ca.ops->ctx_create(); + return (addr ^ (addr >> 32) ^ (uint64_t) qc) & (CA_BUCKETS - 1); +} +#endif + +void * ca_ctx_get(uint64_t addr, + qoscube_t qc) +{ + struct ca_ctx * ctx; +#ifndef IPCP_CA_PER_FLOW + struct list_head * p; + size_t b = ca_bucket(addr, qc); + + pthread_mutex_lock(&ca.mtx); + + list_for_each(p, &ca.buckets[b]) { + ctx = list_entry(p, struct ca_ctx, next); + if (ctx->addr == addr && ctx->qc == qc) { + ctx->refs++; + pthread_mutex_unlock(&ca.mtx); + return ctx; + } + } +#endif + ctx = malloc(sizeof(*ctx)); + if (ctx == NULL) + goto fail_ctx; + + ctx->pol = ca.ops->ctx_create(); + if (ctx->pol == NULL) + goto fail_pol; + + ctx->addr = addr; + ctx->qc = qc; + ctx->refs = 1; + +#ifndef IPCP_CA_PER_FLOW + list_add(&ctx->next, &ca.buckets[b]); + + pthread_mutex_unlock(&ca.mtx); +#endif + return ctx; + fail_pol: + free(ctx); + fail_ctx: +#ifndef IPCP_CA_PER_FLOW + pthread_mutex_unlock(&ca.mtx); +#endif + return NULL; } -void ca_ctx_destroy(void * ctx) +void ca_ctx_put(void * _ctx) { - return ca.ops->ctx_destroy(ctx); + struct ca_ctx * ctx = _ctx; + +#ifndef IPCP_CA_PER_FLOW + pthread_mutex_lock(&ca.mtx); + + if (--ctx->refs > 0) { + pthread_mutex_unlock(&ca.mtx); + return; + } + + list_del(&ctx->next); + + pthread_mutex_unlock(&ca.mtx); +#endif + ca.ops->ctx_destroy(ctx->pol); + + free(ctx); } -ca_wnd_t ca_ctx_update_snd(void * ctx, - size_t len) +time_t ca_ctx_update_snd(void * _ctx, + size_t len, + uint8_t lecn, + uint64_t * ftag) { - return ca.ops->ctx_update_snd(ctx, len); + struct ca_ctx * ctx = _ctx; + + return ca.ops->ctx_update_snd(ctx->pol, len, lecn, ftag); } -bool ca_ctx_update_rcv(void * ctx, +bool ca_ctx_update_rcv(void * _ctx, size_t len, uint8_t ecn, - uint16_t * ece) + uint8_t cap, + uint16_t * ece, + uint8_t * fcap) { - return ca.ops->ctx_update_rcv(ctx, len, ecn, ece); -} + struct ca_ctx * ctx = _ctx; -void ca_ctx_update_ece(void * ctx, - uint16_t ece) -{ - return ca.ops->ctx_update_ece(ctx, ece); + return ca.ops->ctx_update_rcv(ctx->pol, len, ecn, cap, ece, fcap); } -void ca_wnd_wait(ca_wnd_t wnd) +void ca_ctx_update_ece(void * _ctx, + uint16_t ece, + uint8_t cap) { - return ca.ops->wnd_wait(wnd); + struct ca_ctx * ctx = _ctx; + + return ca.ops->ctx_update_ece(ctx->pol, ece, cap); } -int ca_calc_ecn(int fd, +int ca_calc_ecn(size_t queued, uint8_t * ecn, qoscube_t qc, size_t len) { - return ca.ops->calc_ecn(fd, ecn, qc, len); + return ca.ops->calc_ecn(queued, ecn, qc, len); } -ssize_t ca_print_stats(void * ctx, +bool ca_marks_ecn(void) +{ + return ca.ops->marks_ecn; +} + +ssize_t ca_print_stats(void * _ctx, char * buf, size_t len) { + struct ca_ctx * ctx = _ctx; + if (ca.ops->print_stats == NULL) return 0; - return ca.ops->print_stats(ctx, buf, len); + return ca.ops->print_stats(ctx->pol, buf, len); } diff --git a/src/ipcpd/unicast/ca.h b/src/ipcpd/unicast/ca.h index ea803e17..d73d35f5 100644 --- a/src/ipcpd/unicast/ca.h +++ b/src/ipcpd/unicast/ca.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Congestion avoidance * @@ -29,38 +29,40 @@ #include <stdbool.h> #include <sys/types.h> -typedef union { - time_t wait; -} ca_wnd_t; - int ca_init(enum pol_cong_avoid ca); void ca_fini(void); /* OPS */ -void * ca_ctx_create(void); +void * ca_ctx_get(uint64_t addr, + qoscube_t qc); -void ca_ctx_destroy(void * ctx); +void ca_ctx_put(void * ctx); -ca_wnd_t ca_ctx_update_snd(void * ctx, - size_t len); +time_t ca_ctx_update_snd(void * ctx, + size_t len, + uint8_t lecn, + uint64_t * ftag); bool ca_ctx_update_rcv(void * ctx, size_t len, uint8_t ecn, - uint16_t * ece); + uint8_t cap, + uint16_t * ece, + uint8_t * fcap); void ca_ctx_update_ece(void * ctx, - uint16_t ece); + uint16_t ece, + uint8_t cap); -void ca_wnd_wait(ca_wnd_t wnd); - -int ca_calc_ecn(int fd, +int ca_calc_ecn(size_t queued, uint8_t * ecn, qoscube_t qc, size_t len); +bool ca_marks_ecn(void); + ssize_t ca_print_stats(void * ctx, char * buf, size_t len); diff --git a/src/ipcpd/unicast/ca/mb-ecn.c b/src/ipcpd/unicast/ca/mb-ecn.c index d9a204b0..e59aac88 100644 --- a/src/ipcpd/unicast/ca/mb-ecn.c +++ b/src/ipcpd/unicast/ca/mb-ecn.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Multi-bit ECN Congestion Avoidance * @@ -28,9 +28,10 @@ #include "config.h" -#include <ouroboros/ipcp-dev.h> #include <ouroboros/time.h> +#include <ouroboros/utils.h> +#include "cap.h" #include "mb-ecn.h" #include <inttypes.h> @@ -38,31 +39,125 @@ #include <string.h> #include <stdio.h> -/* congestion avoidance constants */ -#define CA_SHFT 5 /* Average over 32 pkts */ -#define CA_WND (1 << CA_SHFT) /* 32 pkts receiver wnd */ -#define CA_UPD (1 << (CA_SHFT - 2)) /* Update snd every 8 pkt */ -#define CA_SLOT 24 /* Initial slot = 16 ms */ -#define CA_INC 1UL << 16 /* ~4MiB/s^2 additive inc */ -#define CA_IWL 1UL << 16 /* Initial limit ~4MiB/s */ -#define CA_MINPS 8 /* Mimimum pkts / slot */ -#define CA_MAXPS 64 /* Maximum pkts / slot */ -#define ECN_Q_SHFT 4 -#define ts_to_ns(ts) ((size_t) ts.tv_sec * BILLION + ts.tv_nsec) +/* + * The sender paces with a token bucket at a rate driven by Δt-scaled + * AIMD: each step changes the rate proportional to elapsed wall-clock + * time, so the per-second dynamics do not depend on how often packets + * arrive. The receiver's averaging window and the sender's feedback + * staleness both stretch with the flow's byte rate, so a slow flow + * is measured and controlled like a fast one; CA_RATE_MIN only + * bounds those horizons (window <= CA_TW_ABSMAX, TTL ~8 s). There + * is no per-flow timer; the control runs on packet sends. + * + * The floor and the AI slope scale with the path: forwarders stamp + * their measured link capacity into the PCI (cap.c), the receiver + * feeds the path MIN back with the ece, and the sender derives + * rate_min = ai_rate = C / 32, clamped to [CA_RATE_MIN, CA_RMIN_MAX], + * falling back to those defaults when the signal goes stale. + */ + +#define CA_SHFT 5 /* ece fixed point: 32 * ecn */ +#define CA_TW_MIN (1ULL << 20) /* min mean window ~1.05 ms */ +#define CA_TW_INIT (1ULL << 26) /* initial mean window ~67ms */ +#define CA_TW_ABSMAX (1ULL << 32) /* window ceiling ~4.3 s */ +/* Quiet horizon, in windows (1 << shift): gap restart and the TTLs. */ +#define CA_TW_GAP_SHFT 2 +#define CA_N_TARGET 16 /* target packets per window */ +#define CA_RX_WBYTES (CA_N_TARGET * 1000ULL) /* target bytes/window */ +#define CA_RX_WCLOSE (2 * CA_RX_WBYTES) /* byte-triggered early close */ +#define CA_TW_SM_SHFT 2 /* window EWMA weight 1/4 */ +#define CA_MARK_Q 4 /* mark quantum (packets) */ + +#define CA_RATE_MIN (1ULL << 13) /* 8 KiB/s rate floor */ +#define CA_RATE_INIT (1ULL << 16) /* slow start seed 64 KiB/s */ +/* Rate cap; also keeps rate * dt and rate * rise below 2^64. */ +#define CA_RATE_MAX (1ULL << 37) +#define CA_INV_SHFT 32 /* reciprocal-rate fixp */ +#define CA_AI_RATE (1ULL << 16) /* 64 KiB/s^2 additive inc */ +#define CA_PROBE_TC (8ULL * BILLION) /* proportional probe TC 8s */ +#define CA_ECE_REF (16 << CA_SHFT) /* full congestion: ecn 16 */ +#define CA_MD_KD_DIV 4 /* one-sided lead gain 1/4 */ +/* Must stay >= 1 ms: the MD term scales by dtc / MILLION (truncates). */ +#define CA_DT_CTRL (BILLION / 1000) +#define CA_DT_CAP (BILLION / 20) /* idle-resume Δt clamp 50ms */ +/* Floor of the rate-relative feedback staleness (ctx->ece_ttl). */ +#define CA_ECE_TTL ((1 << CA_TW_GAP_SHFT) * CA_TW_INIT) +#define CA_SS_TC (BILLION / 50) +#define CA_WASH_BKT (BILLION / 32) /* washout bucket ~31 ms */ +#define CA_WASH_SHFT 2 /* damp 1/4 of bucket change */ + +#define CA_CAP_SHFT 5 /* floor = capacity / 32 */ +#define CA_CAP_SM_SHFT 1 /* capacity EWMA weight 1/2 */ +/* Outlives ece_ttl 16x: onset-fresh fcap re-seeds each episode. */ +#define CA_CAP_TTL_SHFT 4 +#define CA_RMIN_MAX (1ULL << 32) /* derived floor ceiling */ + +#define CA_SND_WIN CA_TW_INIT /* sender util window ~67ms */ +#define CA_USE_NUM 3 /* backlogged: offered >= */ +#define CA_USE_DEN 4 /* 3/4 * window-start rate */ +#define CA_HDRM_MARKS 4 /* ceiling ~2x offered load */ +#define CA_SND_DEC_SHFT 4 /* offered max-filter 1/16 */ +#define CA_SND_DEC_CAP 16 /* bound gapped-close decay */ +#define CA_SND_BYT_MAX (1ULL << 33) /* offered-byte saturation */ + +/* + * Retuning invariants (pinned by the unit tests): + * - (1 << CA_TW_GAP_SHFT) * CA_TW_INIT > S * BILLION / CA_RATE_MIN, or + * a floor-rate flow's onset restart-loops (S ~ one MTU; both ns). + * - CA_RX_WBYTES * BILLION / CA_RATE_MIN < CA_TW_ABSMAX: the + * floor-rate window must clear the ceiling. + * - CA_RATE_MAX * CA_DT_CAP, the folded lead * inv_rate at + * CA_RATE_MIN, and owed * BILLION (owed clamped in mb_ecn_snd) all + * keep the pacer arithmetic below 2^64. + * - CA_DT_CTRL < CA_WASH_BKT < CA_DT_CAP: control cadence under the + * washout bucket under the sparse-step cutoff. + * - CA_RATE_MIN <= CA_RATE_INIT and CA_RMIN_MAX < CA_RATE_MAX. + */ struct mb_ecn_ctx { - uint16_t rx_ece; /* Level of congestion (upstream) */ - size_t rx_ctr; /* Receiver side packet counter */ - - uint16_t tx_ece; /* Level of congestion (downstream) */ - size_t tx_ctr; /* Sender side packet counter */ - size_t tx_wbc; /* Window byte count */ - size_t tx_wpc; /* Window packet count */ - size_t tx_wbl; /* Window byte limit */ - bool tx_cav; /* Congestion avoidance */ - size_t tx_mul; /* Slot size multiplier */ - size_t tx_inc; /* Additive increase */ - size_t tx_slot; + uint16_t rx_ece; /* smoothed congestion echo (32 * ecn) */ + uint64_t rx_acc; /* window integral of ecn * dt */ + uint64_t rx_byt; /* bytes arrived in current window */ + uint64_t rx_ts; /* last packet arrival (ns) */ + uint64_t rx_win; /* window start (ns) */ + uint64_t rx_tw; /* adaptive averaging window (ns) */ + uint8_t rx_cap; /* window bottleneck capacity code */ + + uint16_t tx_ece; /* congestion reported from downstream */ + uint16_t tx_ecp; /* previous tx_ece (rise detection) */ + uint8_t tx_loc; /* local first-hop ecn mark (fallback) */ + uint8_t tx_cap; /* path capacity code fed back to us */ + bool tx_cav; /* past slow start */ + uint64_t rate; /* paced send rate (bytes/s) */ + uint64_t rate_min; /* capacity-derived rate floor (B/s) */ + uint64_t ai_rate; /* additive-increase slope (B/s^2) */ + uint64_t ece_ttl; /* how long feedback stays valid (ns) */ + uint64_t r_bkt; /* rate snapshot at last washout bucket */ + uint64_t wash_acc; /* washout bucket time accumulator (ns) */ + uint64_t inv_rate; /* fixed-point 1/rate for pacing */ + uint64_t vt; /* virtual service clock (bytes) */ + uint64_t lead; /* pacer lead of last send (bytes) */ + uint64_t last_ts; /* last clock advance (ns) */ + uint64_t last_ctrl; /* last rate update (ns) */ + uint64_t last_fb; /* last feedback applied (ns) */ + uint64_t last_loc; /* last local mark seen (ns) */ + uint64_t last_cap; /* last capacity applied (ns) */ + + uint64_t n_ctrl; /* control steps taken */ + uint64_t t_ctrl; /* wall time covered by steps (ns) */ + uint64_t t_bank; /* increase time banked in steps (ns) */ + uint64_t n_fb; /* feedback updates received */ + uint64_t n_ttl; /* feedback aged out (TTL) */ + uint64_t n_cap; /* capacity updates applied */ + uint64_t ss_peak; /* peak rate in slow start (bytes/s) */ + + uint64_t snd_byt; /* bytes offered this window (capped) */ + uint64_t snd_win; /* utilisation window start (ns) */ + uint64_t snd_r0; /* rate at window start */ + uint64_t snd_rate; /* max-filter of offered rate (B/s) */ + bool backlogged; /* offered load keeps the pacer busy */ + bool src_limited; /* rate held at offered-load ceiling */ + bool started; /* a real send has occurred */ }; struct ca_ops mb_ecn_ca_ops = { @@ -71,14 +166,34 @@ struct ca_ops mb_ecn_ca_ops = { .ctx_update_snd = mb_ecn_ctx_update_snd, .ctx_update_rcv = mb_ecn_ctx_update_rcv, .ctx_update_ece = mb_ecn_ctx_update_ece, - .wnd_wait = mb_ecn_wnd_wait, .calc_ecn = mb_ecn_calc_ecn, + .marks_ecn = true, .print_stats = mb_ecn_print_stats }; +static uint64_t mb_ecn_rate_inv(uint64_t rate) +{ + return ((uint64_t) BILLION << CA_INV_SHFT) / rate; +} + +/* + * Feedback arrives once per receiver window, and the window tracks + * the flow's byte rate. Mirror it: age the signal out only past the + * quiet horizon at the current rate, floored for fast flows. + */ +static uint64_t mb_ecn_ece_ttl(uint64_t rate) +{ + uint64_t ttl; + + ttl = (1 << CA_TW_GAP_SHFT) * CA_RX_WBYTES * BILLION / rate; + + return ttl > (uint64_t) CA_ECE_TTL ? ttl : (uint64_t) CA_ECE_TTL; +} + void * mb_ecn_ctx_create(void) { struct timespec now; + uint64_t t; struct mb_ecn_ctx * ctx; ctx = malloc(sizeof(*ctx)); @@ -89,10 +204,27 @@ void * mb_ecn_ctx_create(void) memset(ctx, 0, sizeof(*ctx)); - ctx->tx_mul = CA_SLOT; - ctx->tx_wbl = CA_IWL; - ctx->tx_inc = CA_INC; - ctx->tx_slot = ts_to_ns(now) >> ctx->tx_mul; + t = TS_TO_UINT64(now); + + ctx->rate = CA_RATE_INIT; + ctx->rate_min = CA_RATE_MIN; + ctx->ai_rate = CA_AI_RATE; + ctx->ece_ttl = mb_ecn_ece_ttl(CA_RATE_INIT); + ctx->r_bkt = CA_RATE_INIT; + ctx->inv_rate = mb_ecn_rate_inv(CA_RATE_INIT); + ctx->rx_ts = t; + ctx->rx_win = t; + ctx->rx_tw = CA_TW_INIT; + ctx->last_ts = t; + ctx->last_ctrl = t; + ctx->last_fb = t; + ctx->last_loc = t; + ctx->last_cap = t; + + /* snd_win/last_ts re-seeded lazily on the first real send. */ + ctx->snd_r0 = CA_RATE_INIT; + ctx->snd_rate = CA_RATE_INIT; + ctx->backlogged = true; return (void *) ctx; } @@ -102,158 +234,519 @@ void mb_ecn_ctx_destroy(void * ctx) free(ctx); } -#define _slot_after(new, old) ((int64_t) (old - new) < 0) +/* Local first-hop mark exits slow start and covers dead feedback. */ +static void mb_ecn_loc(struct mb_ecn_ctx * ctx, + uint8_t lecn, + uint64_t t) +{ + if (lecn == 0) + return; + + ctx->tx_loc = lecn; + ctx->tx_cav = true; + ctx->last_loc = t; +} + +/* Slow start: ramp only while backlogged. */ +static void mb_ecn_slow_start(struct mb_ecn_ctx * ctx, + uint64_t dta) +{ + if (ctx->backlogged) + ctx->rate += ctx->rate * dta / CA_SS_TC; + + ctx->r_bkt = ctx->rate; +} + +/* Additive increase plus a rate-independent proportional probe. */ +static void mb_ecn_increase(struct mb_ecn_ctx * ctx, + uint64_t dta) +{ + if (!ctx->backlogged) + return; + + ctx->rate += ctx->ai_rate * dta / BILLION; + ctx->rate += ctx->rate * dta / CA_PROBE_TC; +} + +/* Multiplicative decrease: proportional cut plus a one-sided lead. */ +static void mb_ecn_decrease(struct mb_ecn_ctx * ctx, + uint64_t dtc) +{ + uint64_t dtm; + uint64_t mark; + uint64_t rise; + uint64_t cut; + uint16_t m; + + m = ctx->tx_ece > 0 ? ctx->tx_ece + : (uint16_t) (ctx->tx_loc << CA_SHFT); + if (m == 0) { + ctx->tx_ecp = 0; + return; + } + + mark = MIN(m, CA_ECE_REF); + rise = 0; + if (m > ctx->tx_ecp) + rise = MIN(m - ctx->tx_ecp, CA_ECE_REF); + + cut = ctx->rate * rise / (CA_ECE_REF * CA_MD_KD_DIV); + + /* Honest elapsed ms, so a starved sender still cuts. */ + dtm = dtc / MILLION; + if (mark * dtm >= CA_ECE_REF * 500) + cut += ctx->rate / 2; + else + cut += ctx->rate * mark * dtm / (CA_ECE_REF * 1000); + + if (cut > ctx->rate / 2) + cut = ctx->rate / 2; + + ctx->rate -= cut; + ctx->tx_ecp = m; +} + +/* + * Washout: once per wall-clock bucket, damp a fixed fraction of the + * rate change over that bucket. Bucketed (not per-step) so it stays + * cadence-independent; bounded so it cannot reverse a ramp. A sparse + * step resets it, so a starved sender keeps its cut. + */ +static void mb_ecn_washout(struct mb_ecn_ctx * ctx, + uint64_t dtc, + uint64_t dta) +{ + if (dtc > (uint64_t) CA_DT_CAP) { + ctx->r_bkt = ctx->rate; + ctx->wash_acc = 0; + return; + } + + ctx->wash_acc += dta; + if (ctx->wash_acc < (uint64_t) CA_WASH_BKT) + return; + + if (ctx->rate > ctx->r_bkt) + ctx->rate -= (ctx->rate - ctx->r_bkt) >> CA_WASH_SHFT; + else + ctx->rate += (ctx->r_bkt - ctx->rate) >> CA_WASH_SHFT; + + ctx->r_bkt = ctx->rate; + ctx->wash_acc = 0; +} + +/* Offered-load ceiling backstop while source-limited. */ +static void mb_ecn_ceiling(struct mb_ecn_ctx * ctx) +{ + unsigned code; + uint64_t hi; + + if (ctx->backlogged) { + ctx->src_limited = false; + return; + } + + code = (unsigned) cap_enc(ctx->snd_rate) + CA_HDRM_MARKS; + if (code > UINT8_MAX) /* keep the cast lossless */ + code = UINT8_MAX; + + hi = cap_dec((uint8_t) code); + if (hi > CA_RATE_MAX) + hi = CA_RATE_MAX; + + if (hi < CA_RATE_MIN) + hi = CA_RATE_MIN; + + ctx->src_limited = ctx->rate > hi; + if (ctx->src_limited) { + ctx->rate = hi; + ctx->r_bkt = ctx->rate; + } +} + +static void mb_ecn_ctrl(struct mb_ecn_ctx * ctx, + uint64_t dtc) +{ + uint64_t dta; + uint64_t lo; + + /* AI and slow start bank at most CA_DT_CAP of idle time. */ + dta = MIN(dtc, (uint64_t) CA_DT_CAP); + + ctx->n_ctrl++; + ctx->t_ctrl += dtc; + ctx->t_bank += dta; + + if (ctx->tx_cav) { + mb_ecn_increase(ctx, dta); + mb_ecn_decrease(ctx, dtc); + mb_ecn_washout(ctx, dtc, dta); + } else { + mb_ecn_slow_start(ctx, dta); + } + + mb_ecn_ceiling(ctx); + + /* Capacity floor only while backlogged; else the absolute floor. */ + lo = ctx->backlogged ? ctx->rate_min : (uint64_t) CA_RATE_MIN; + if (ctx->rate < lo) + ctx->rate = lo; + + if (ctx->rate > CA_RATE_MAX) + ctx->rate = CA_RATE_MAX; + + ctx->inv_rate = mb_ecn_rate_inv(ctx->rate); + ctx->ece_ttl = mb_ecn_ece_ttl(ctx->rate); + + if (!ctx->tx_cav && ctx->rate > ctx->ss_peak) + ctx->ss_peak = ctx->rate; +} + +/* Fold offered into the max filter: rise at once, decay 1/16 per window. */ +static void mb_ecn_offered(struct mb_ecn_ctx * ctx, + uint64_t offered, + uint64_t elapsed) +{ + uint64_t n; + + if (offered >= ctx->snd_rate) { + ctx->snd_rate = offered; + return; + } + + n = MIN(elapsed / CA_SND_WIN, CA_SND_DEC_CAP); + while (n-- > 0 && ctx->snd_rate > offered) + ctx->snd_rate -= (ctx->snd_rate - offered) >> CA_SND_DEC_SHFT; +} + +/* + * Close the utilisation window: set backlogged from the level test, + * fold offered into the max filter, then reset the window. + */ +static void mb_ecn_win(struct mb_ecn_ctx * ctx, + uint64_t t) +{ + uint64_t elapsed = t - ctx->snd_win; + uint64_t offered; + + offered = ctx->snd_byt * BILLION / elapsed; + + ctx->backlogged = offered * CA_USE_DEN >= ctx->snd_r0 * CA_USE_NUM; -ca_wnd_t mb_ecn_ctx_update_snd(void * _ctx, - size_t len) + mb_ecn_offered(ctx, offered, elapsed); + + if (ctx->backlogged) + ctx->src_limited = false; + + ctx->snd_win = t; + ctx->snd_byt = 0; + ctx->snd_r0 = ctx->rate; +} + +/* Age out congestion, local-mark and capacity signals once stale. */ +static void mb_ecn_age(struct mb_ecn_ctx * ctx, + uint64_t t) +{ + if (t - ctx->last_fb > ctx->ece_ttl) { + if (ctx->tx_ece > 0) + ctx->n_ttl++; + ctx->tx_ece = 0; + } + + if (t - ctx->last_loc > ctx->ece_ttl) + ctx->tx_loc = 0; + + /* Stale capacity: fall back to the compile-time defaults. */ + if (t - ctx->last_cap > ctx->ece_ttl << CA_CAP_TTL_SHFT) { + ctx->rate_min = CA_RATE_MIN; + ctx->ai_rate = CA_AI_RATE; + ctx->tx_cap = 0; + } +} + +/* Advance the virtual clock; a gap past CA_DT_CAP credits a burst. */ +static void mb_ecn_advance(struct mb_ecn_ctx * ctx, + uint64_t dt, + size_t len, + uint64_t ftag) +{ + uint64_t burst; + uint64_t owed; + + if (dt <= (uint64_t) CA_DT_CAP) { + ctx->vt += ctx->rate * dt / BILLION; + return; + } + + burst = ctx->rate * CA_DT_CAP / BILLION; + if (burst < (uint64_t) len) + burst = len; + + owed = ftag > ctx->vt ? ftag - ctx->vt + burst : burst; + + /* Clamp so owed * BILLION cannot wrap (2^33 B backlog). */ + if (owed > (1ULL << 33)) + owed = 1ULL << 33; + + if (dt >= owed * BILLION / ctx->rate) + ctx->vt += owed; + else + ctx->vt += ctx->rate * dt / BILLION; +} + +static time_t mb_ecn_snd(struct mb_ecn_ctx * ctx, + size_t len, + uint64_t t, + uint64_t * ftag) +{ + uint64_t dt; + uint64_t dtc; + uint64_t s; + + /* Lazy warm-up seed: packet #1 is never an idle resume. */ + if (!ctx->started) { + ctx->started = true; + ctx->last_ts = t; + ctx->snd_win = t; + ctx->snd_r0 = ctx->rate; + } + + dt = t - ctx->last_ts; + ctx->last_ts = t; + + mb_ecn_age(ctx, t); + + /* Offered-load estimator: accumulate, gate growth, size ceiling. */ + ctx->snd_byt += len; + if (ctx->snd_byt > (uint64_t) CA_SND_BYT_MAX) + ctx->snd_byt = CA_SND_BYT_MAX; + + if (dt > (uint64_t) CA_DT_CAP) + ctx->backlogged = false; + + if (t - ctx->snd_win >= (uint64_t) CA_SND_WIN) + mb_ecn_win(ctx, t); + + /* Rate update before the vt advance: burst uses the clamped rate. */ + dtc = t - ctx->last_ctrl; + if (dtc >= (uint64_t) CA_DT_CTRL) { + ctx->last_ctrl = t; + mb_ecn_ctrl(ctx, dtc); + } + + mb_ecn_advance(ctx, dt, len, *ftag); + + /* SFQ start tag: behind the clock starts now, ahead waits. */ + s = *ftag > ctx->vt ? *ftag : ctx->vt; + *ftag = s + len; + + ctx->lead = s - ctx->vt; + + /* Reciprocal pacing; folded so any lead * rate stays in range. */ + if (s > ctx->vt) + return (time_t) ((ctx->lead * (ctx->inv_rate >> 16)) + >> (CA_INV_SHFT - 16)); + + return 0; +} + +time_t mb_ecn_ctx_update_snd(void * _ctx, + size_t len, + uint8_t lecn, + uint64_t * ftag) { struct timespec now; - size_t slot; - ca_wnd_t wnd; + uint64_t t; struct mb_ecn_ctx * ctx = _ctx; clock_gettime(PTHREAD_COND_CLOCK, &now); - slot = ts_to_ns(now) >> ctx->tx_mul; + t = TS_TO_UINT64(now); - ctx->tx_ctr++; - ctx->tx_wpc++; - ctx->tx_wbc += len; + mb_ecn_loc(ctx, lecn, t); - if (ctx->tx_ctr > CA_WND) - ctx->tx_ece = 0; + return mb_ecn_snd(ctx, len, t, ftag); +} - if (_slot_after(slot, ctx->tx_slot)) { - bool carry = false; /* may carry over if window increases */ - - ctx->tx_slot = slot; - - if (!ctx->tx_cav) { /* Slow start */ - if (ctx->tx_wbc > ctx->tx_wbl) - ctx->tx_wbl <<= 1; - } else { - if (ctx->tx_ece) /* Mult. Decrease */ - ctx->tx_wbl -= (ctx->tx_wbl * ctx->tx_ece) - >> (CA_SHFT + 8); - else /* Add. Increase */ - ctx->tx_wbl = ctx->tx_wbc + ctx->tx_inc; - } - - /* Window scaling */ - if (ctx->tx_wpc < CA_MINPS) { - size_t fact = 0; /* factor to scale the window up */ - size_t pkts = ctx->tx_wpc; - while (pkts < CA_MINPS) { - pkts <<= 1; - fact++; - } - ctx->tx_mul += fact; - ctx->tx_slot >>= fact; - if ((ctx->tx_slot & ((1 << fact) - 1)) == 0) { - carry = true; - ctx->tx_slot += 1; - } - ctx->tx_wbl <<= fact; - ctx->tx_inc <<= fact; - } else if (ctx->tx_wpc > CA_MAXPS) { - size_t fact = 0; /* factor to scale the window down */ - size_t pkts = ctx->tx_wpc; - while (pkts > CA_MAXPS) { - pkts >>= 1; - fact++; - } - ctx->tx_mul -= fact; - ctx->tx_slot <<= fact; - ctx->tx_wbl >>= fact; - ctx->tx_inc >>= fact; - } else { - ctx->tx_slot = slot; - } - - if (!carry) { - ctx->tx_wbc = 0; - ctx->tx_wpc = 0; - } - } +/* Estimator idle, or a gap past ~4 current windows: restart fresh. */ +static bool mb_ecn_rcv_fresh(const struct mb_ecn_ctx * ctx, + uint64_t dt) +{ + if (ctx->rx_ece == 0 && ctx->rx_acc == 0) + return true; - if (ctx->tx_wbc > ctx->tx_wbl) - wnd.wait = ((ctx->tx_slot + 1) << ctx->tx_mul) - ts_to_ns(now); + return dt > ctx->rx_tw << CA_TW_GAP_SHFT; +} + +/* Size the next averaging window to ~CA_N_TARGET packets at this rate. */ +static void mb_ecn_resize(struct mb_ecn_ctx * ctx, + uint64_t win) +{ + uint64_t tw = CA_RX_WBYTES * win / ctx->rx_byt; + + if (tw > ctx->rx_tw) + ctx->rx_tw += (tw - ctx->rx_tw) >> CA_TW_SM_SHFT; else - wnd.wait = 0; + ctx->rx_tw -= (ctx->rx_tw - tw) >> CA_TW_SM_SHFT; - return wnd; + if (ctx->rx_tw < CA_TW_MIN) + ctx->rx_tw = CA_TW_MIN; + + if (ctx->rx_tw > CA_TW_ABSMAX) + ctx->rx_tw = CA_TW_ABSMAX; } -void mb_ecn_wnd_wait(ca_wnd_t wnd) +static bool mb_ecn_rcv(struct mb_ecn_ctx * ctx, + size_t len, + uint8_t ecn, + uint8_t cap, + uint16_t * ece, + uint8_t * fcap, + uint64_t t) { - if (wnd.wait > 0) { - struct timespec s = TIMESPEC_INIT_S(0); - if (wnd.wait > BILLION) /* Don't care throttling < 1s */ - s.tv_sec = 1; - else - s.tv_nsec = wnd.wait; + uint64_t dt; + uint64_t win; - nanosleep(&s, NULL); + dt = t - ctx->rx_ts; + ctx->rx_ts = t; + + if (ctx->rx_ece == 0 && ctx->rx_acc == 0 && ecn == 0) + return false; + + /* Onset, or ~4 windows of silence: emit fresh, undiluted. */ + if (mb_ecn_rcv_fresh(ctx, dt)) { + ctx->rx_win = t; + ctx->rx_acc = 0; + ctx->rx_byt = len; + ctx->rx_cap = cap; /* fresh, seeds the new window */ + ctx->rx_ece = (uint16_t) (ecn << CA_SHFT); + *ece = ctx->rx_ece; + *fcap = ctx->rx_cap; + return true; } + + /* Dwell clamp: one packet weighs at most one window of mark. */ + ctx->rx_acc += ecn * MIN(dt, ctx->rx_tw); + ctx->rx_byt += len; + ctx->rx_cap = cap_min(ctx->rx_cap, cap); + + *ece = ctx->rx_ece; + + win = t - ctx->rx_win; + if (win < ctx->rx_tw) { + /* Early close once 2x target bytes arrive (speed-up). */ + if (ctx->rx_byt < CA_RX_WCLOSE) + return false; + + if (win < CA_TW_MIN) + return false; + } + + /* Time-integral mean over the actual window elapsed (never rx_tw). */ + ctx->rx_ece = (uint16_t) ((ctx->rx_acc << CA_SHFT) / win); + + if (ctx->rx_byt > 0) + mb_ecn_resize(ctx, win); + + *fcap = ctx->rx_cap; + + ctx->rx_win = t; + ctx->rx_acc = 0; + ctx->rx_byt = 0; + ctx->rx_cap = 0; /* the next window starts unknown */ + + *ece = ctx->rx_ece; + + return true; } bool mb_ecn_ctx_update_rcv(void * _ctx, size_t len, uint8_t ecn, - uint16_t * ece) + uint8_t cap, + uint16_t * ece, + uint8_t * fcap) { - struct mb_ecn_ctx* ctx = _ctx; - bool update; + struct timespec now; + struct mb_ecn_ctx * ctx = _ctx; - (void) len; + clock_gettime(PTHREAD_COND_CLOCK, &now); - if ((ctx->rx_ece | ecn) == 0) - return false; + return mb_ecn_rcv(ctx, len, ecn, cap, ece, fcap, TS_TO_UINT64(now)); +} - if (ecn == 0) { /* End of congestion */ - ctx->rx_ece >>= 2; - update = ctx->rx_ece == 0; - } else { - if (ctx->rx_ece == 0) { /* Start of congestion */ - ctx->rx_ece = ecn; - ctx->rx_ctr = 0; - update = true; - } else { /* Congestion update */ - ctx->rx_ece -= ctx->rx_ece >> CA_SHFT; - ctx->rx_ece += ecn; - update = (ctx->rx_ctr++ & (CA_UPD - 1)) == true; - } +static void mb_ecn_ece(struct mb_ecn_ctx * ctx, + uint16_t ece, + uint8_t cap, + uint64_t t) +{ + uint64_t tgt; + + ctx->tx_ece = ece; + ctx->tx_cav = true; + ctx->last_fb = t; + ctx->n_fb++; + + /* Scale the floor and AI slope to the path bottleneck. */ + if (cap != 0) { + tgt = cap_dec(cap) >> CA_CAP_SHFT; + if (tgt < CA_RATE_MIN) + tgt = CA_RATE_MIN; + + if (tgt > CA_RMIN_MAX) + tgt = CA_RMIN_MAX; + + if (tgt > ctx->rate_min) + ctx->rate_min += (tgt - ctx->rate_min) + >> CA_CAP_SM_SHFT; + else + ctx->rate_min -= (ctx->rate_min - tgt) + >> CA_CAP_SM_SHFT; + + ctx->ai_rate = ctx->rate_min; + ctx->tx_cap = cap; + ctx->last_cap = t; + ctx->n_cap++; } - *ece = ctx->rx_ece; + /* Control from the feedback path: a starved sender recovers. */ + if (t - ctx->last_ctrl < (uint64_t) CA_DT_CTRL) + return; - return update; -} + mb_ecn_ctrl(ctx, t - ctx->last_ctrl); + ctx->last_ctrl = t; +} void mb_ecn_ctx_update_ece(void * _ctx, - uint16_t ece) + uint16_t ece, + uint8_t cap) { - struct mb_ecn_ctx* ctx = _ctx; + struct timespec now; + struct mb_ecn_ctx * ctx = _ctx; - ctx->tx_ece = ece; - ctx->tx_ctr = 0; - ctx->tx_cav = true; + clock_gettime(PTHREAD_COND_CLOCK, &now); + + mb_ecn_ece(ctx, ece, cap, TS_TO_UINT64(now)); } -int mb_ecn_calc_ecn(int fd, +int mb_ecn_calc_ecn(size_t queued, uint8_t * ecn, qoscube_t qc, size_t len) { - size_t q; + size_t q; + uint8_t mark; (void) len; (void) qc; - q = ipcp_flow_queued(fd); + /* Saturate: a queue past 255 quanta must not wrap to a low mark. */ + q = queued / CA_MARK_Q; + mark = q > 255 ? (uint8_t) 255 : (uint8_t) q; - *ecn |= (uint8_t) (q >> ECN_Q_SHFT); + if (mark > *ecn) + *ecn = mark; return 0; } @@ -262,35 +755,64 @@ ssize_t mb_ecn_print_stats(void * _ctx, char * buf, size_t len) { - struct mb_ecn_ctx* ctx = _ctx; - char * regime; + struct mb_ecn_ctx * ctx = _ctx; + char * regime; + uint64_t rate; + uint64_t peak; + int code; + uint16_t m; if (len < 1024) return 0; - if (!ctx->tx_cav) + /* No signal seen: the rate is unconstrained drift, not a target. */ + rate = ctx->tx_cav ? ctx->rate : 0; + peak = ctx->tx_cav ? ctx->ss_peak : 0; + + /* Match the controller: MD fires on m, incl. the local fallback. */ + m = ctx->tx_ece > 0 ? ctx->tx_ece + : (uint16_t) (ctx->tx_loc << CA_SHFT); + + if (!ctx->tx_cav) { regime = "Slow start"; - else if (ctx->tx_ece) - regime = "Multiplicative dec"; - else + code = 0; + } else if (ctx->src_limited) { + regime = "Source limited"; + code = 3; + } else if (m > 0) { + regime = "Proportional dec"; + code = 2; + } else { regime = "Additive inc"; + code = 1; + } sprintf(buf, "Congestion avoidance algorithm: %20s\n" "Upstream congestion level: %20u\n" - "Upstream packet counter: %20zu\n" "Downstream congestion level: %20u\n" - "Downstream packet counter: %20zu\n" - "Congestion window size (ns): %20" PRIu64 "\n" - "Packets in this window: %20zu\n" - "Bytes in this window: %20zu\n" - "Max bytes in this window: %20zu\n" - "Current congestion regime: %20s\n", + "Paced rate (bytes/s): %20" PRIu64 "\n" + "Pacer lead (bytes): %20" PRIu64 "\n" + "Congestion regime (code): %20d\n" + "Current congestion regime: %20s\n" + "Control steps (count): %20" PRIu64 "\n" + "Control time elapsed (ns): %20" PRIu64 "\n" + "Control time banked (ns): %20" PRIu64 "\n" + "Feedback updates (count): %20" PRIu64 "\n" + "Feedback timeouts (count): %20" PRIu64 "\n" + "Path capacity (bytes/s): %20" PRIu64 "\n" + "Capacity rate floor (bytes/s): %20" PRIu64 "\n" + "Capacity updates (count): %20" PRIu64 "\n" + "Slow start peak rate (bytes/s): %20" PRIu64 "\n", "Multi-bit ECN", - ctx->tx_ece, ctx->tx_ctr, - ctx->rx_ece, ctx->rx_ctr, (uint64_t) (1ULL << ctx->tx_mul), - ctx->tx_wpc, ctx->tx_wbc, ctx->tx_wbl, - regime); + ctx->tx_ece, + ctx->rx_ece, + rate, ctx->lead, code, + regime, + ctx->n_ctrl, ctx->t_ctrl, ctx->t_bank, + ctx->n_fb, ctx->n_ttl, + cap_dec(ctx->tx_cap), ctx->rate_min, ctx->n_cap, + peak); return strlen(buf); } diff --git a/src/ipcpd/unicast/ca/mb-ecn.h b/src/ipcpd/unicast/ca/mb-ecn.h index 9a2c8b49..7bf0b29f 100644 --- a/src/ipcpd/unicast/ca/mb-ecn.h +++ b/src/ipcpd/unicast/ca/mb-ecn.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Multi-bit ECN Congestion Avoidance * @@ -29,20 +29,23 @@ void * mb_ecn_ctx_create(void); void mb_ecn_ctx_destroy(void * ctx); -ca_wnd_t mb_ecn_ctx_update_snd(void * ctx, - size_t len); +time_t mb_ecn_ctx_update_snd(void * ctx, + size_t len, + uint8_t lecn, + uint64_t * ftag); bool mb_ecn_ctx_update_rcv(void * ctx, size_t len, uint8_t ecn, - uint16_t * ece); + uint8_t cap, + uint16_t * ece, + uint8_t * fcap); void mb_ecn_ctx_update_ece(void * ctx, - uint16_t ece); - -void mb_ecn_wnd_wait(ca_wnd_t wnd); + uint16_t ece, + uint8_t cap); -int mb_ecn_calc_ecn(int fd, +int mb_ecn_calc_ecn(size_t queued, uint8_t * ecn, qoscube_t qc, size_t len); diff --git a/src/ipcpd/unicast/ca/nop.c b/src/ipcpd/unicast/ca/nop.c index 617fc15b..e6965297 100644 --- a/src/ipcpd/unicast/ca/nop.c +++ b/src/ipcpd/unicast/ca/nop.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Dummy Congestion Avoidance * @@ -30,8 +30,8 @@ struct ca_ops nop_ca_ops = { .ctx_update_snd = nop_ctx_update_snd, .ctx_update_rcv = nop_ctx_update_rcv, .ctx_update_ece = nop_ctx_update_ece, - .wnd_wait = nop_wnd_wait, .calc_ecn = nop_calc_ecn, + .marks_ecn = false, .print_stats = NULL }; @@ -45,51 +45,52 @@ void nop_ctx_destroy(void * ctx) (void) ctx; } -ca_wnd_t nop_ctx_update_snd(void * ctx, - size_t len) +time_t nop_ctx_update_snd(void * ctx, + size_t len, + uint8_t lecn, + uint64_t * ftag) { - ca_wnd_t wnd; - (void) ctx; (void) len; + (void) lecn; + (void) ftag; - memset(&wnd, 0, sizeof(wnd)); - - return wnd; -} - -void nop_wnd_wait(ca_wnd_t wnd) -{ - (void) wnd; + return 0; } bool nop_ctx_update_rcv(void * ctx, size_t len, uint8_t ecn, - uint16_t * ece) + uint8_t cap, + uint16_t * ece, + uint8_t * fcap) { (void) ctx; (void) len; (void) ecn; + (void) cap; (void) ece; + (void) fcap; return false; } void nop_ctx_update_ece(void * ctx, - uint16_t ece) + uint16_t ece, + uint8_t cap) { (void) ctx; (void) ece; + (void) cap; } -int nop_calc_ecn(int fd, +int nop_calc_ecn(size_t queued, uint8_t * ecn, qoscube_t qc, size_t len) { - (void) fd; + (void) queued; (void) len; (void) ecn; (void) qc; diff --git a/src/ipcpd/unicast/ca/nop.h b/src/ipcpd/unicast/ca/nop.h index 248b198d..6ca206df 100644 --- a/src/ipcpd/unicast/ca/nop.h +++ b/src/ipcpd/unicast/ca/nop.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Dummy Congestion Avoidance * @@ -29,20 +29,23 @@ void * nop_ctx_create(void); void nop_ctx_destroy(void * ctx); -ca_wnd_t nop_ctx_update_snd(void * ctx, - size_t len); +time_t nop_ctx_update_snd(void * ctx, + size_t len, + uint8_t lecn, + uint64_t * ftag); bool nop_ctx_update_rcv(void * ctx, size_t len, uint8_t ecn, - uint16_t * ece); + uint8_t cap, + uint16_t * ece, + uint8_t * fcap); void nop_ctx_update_ece(void * ctx, - uint16_t ece); - -void nop_wnd_wait(ca_wnd_t wnd); + uint16_t ece, + uint8_t cap); -int nop_calc_ecn(int fd, +int nop_calc_ecn(size_t queued, uint8_t * ecn, qoscube_t qc, size_t len); diff --git a/src/ipcpd/unicast/ca/ops.h b/src/ipcpd/unicast/ca/ops.h index 3a7b7248..a86c1b3b 100644 --- a/src/ipcpd/unicast/ca/ops.h +++ b/src/ipcpd/unicast/ca/ops.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Congestion avoidance policy ops * @@ -30,24 +30,30 @@ struct ca_ops { void (* ctx_destroy)(void * ctx); - ca_wnd_t (* ctx_update_snd)(void * ctx, - size_t len); + time_t (* ctx_update_snd)(void * ctx, + size_t len, + uint8_t lecn, + uint64_t * ftag); bool (* ctx_update_rcv)(void * ctx, size_t len, uint8_t ecn, - uint16_t * ece); + uint8_t cap, + uint16_t * ece, + uint8_t * fcap); void (* ctx_update_ece)(void * ctx, - uint16_t ece); - - void (* wnd_wait)(ca_wnd_t wnd); + uint16_t ece, + uint8_t cap); - int (* calc_ecn)(int fd, + int (* calc_ecn)(size_t queued, uint8_t * ecn, qoscube_t qc, size_t len); + /* True if calc_ecn inspects the queue; gates the lookup. */ + bool marks_ecn; + /* Optional, can be NULL */ ssize_t (* print_stats)(void * ctx, char * buf, diff --git a/src/ipcpd/unicast/ca/pol.h b/src/ipcpd/unicast/ca/pol.h index db0a1a11..bfb9cc2d 100644 --- a/src/ipcpd/unicast/ca/pol.h +++ b/src/ipcpd/unicast/ca/pol.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Congestion avoidance policies * diff --git a/src/ipcpd/unicast/ca/tests/CMakeLists.txt b/src/ipcpd/unicast/ca/tests/CMakeLists.txt new file mode 100644 index 00000000..6e42163d --- /dev/null +++ b/src/ipcpd/unicast/ca/tests/CMakeLists.txt @@ -0,0 +1,44 @@ +get_filename_component(CURRENT_SOURCE_PARENT_DIR + ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) +get_filename_component(CURRENT_BINARY_PARENT_DIR + ${CMAKE_CURRENT_BINARY_DIR} DIRECTORY) + +get_filename_component(UNICAST_SOURCE_DIR ${CURRENT_SOURCE_PARENT_DIR} DIRECTORY) +get_filename_component(UNICAST_BINARY_DIR ${CURRENT_BINARY_PARENT_DIR} DIRECTORY) + +get_filename_component(PARENT_PATH ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) +get_filename_component(PARENT_DIR ${PARENT_PATH} NAME) + +compute_test_prefix() + +create_test_sourcelist(${PARENT_DIR}_tests test_suite.c + # Add new tests here + mb_ecn_test.c + ca_test.c + ) + +add_executable(${PARENT_DIR}_test ${${PARENT_DIR}_tests} + ${UNICAST_SOURCE_DIR}/ca.c + ${UNICAST_SOURCE_DIR}/cap.c + ${CURRENT_SOURCE_PARENT_DIR}/nop.c + ) + +target_include_directories(${PARENT_DIR}_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CURRENT_SOURCE_PARENT_DIR} + ${CURRENT_BINARY_PARENT_DIR} + ${UNICAST_SOURCE_DIR} + ${UNICAST_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_BINARY_DIR}/include + ${CMAKE_SOURCE_DIR}/src/ipcpd + ${CMAKE_BINARY_DIR}/src/ipcpd +) + +disable_test_logging_for_target(${PARENT_DIR}_test) +target_link_libraries(${PARENT_DIR}_test PRIVATE ouroboros-common) + +add_dependencies(build_tests ${PARENT_DIR}_test) + +ouroboros_register_tests(TARGET ${PARENT_DIR}_test TESTS ${${PARENT_DIR}_tests}) diff --git a/src/ipcpd/unicast/ca/tests/ca_test.c b/src/ipcpd/unicast/ca/tests/ca_test.c new file mode 100644 index 00000000..4c44d29e --- /dev/null +++ b/src/ipcpd/unicast/ca/tests/ca_test.c @@ -0,0 +1,392 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Unit tests for the congestion-avoidance interface + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "config.h" + +#include "ca.h" + +#include <test/test.h> + +#define ADDR_A 0x1111ULL +#define ADDR_B 0x2222ULL + +static const struct { + enum pol_cong_avoid pol; + const char * name; +} ca_pols[] = { + { CA_NONE, "none" }, + { CA_MB_ECN, "mb-ecn" } +}; + +#define CA_POLS (sizeof(ca_pols) / sizeof(ca_pols[0])) + +static int test_ca_init_fini(enum pol_cong_avoid pol, + const char * name) +{ + TEST_START("(%s)", name); + + if (ca_init(pol) < 0) { + printf("Failed to init ca for %s.\n", name); + goto fail; + } + + ca_fini(); + + TEST_SUCCESS("(%s)", name); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL("(%s)", name); + return TEST_RC_FAIL; +} + +static int test_ca_init_fini_all(void) +{ + int ret = 0; + size_t i; + + for (i = 0; i < CA_POLS; i++) + ret |= test_ca_init_fini(ca_pols[i].pol, ca_pols[i].name); + + return ret; +} + +static int test_ca_init_invalid(void) +{ + TEST_START(); + + if (ca_init(CA_INVALID) == 0) { + printf("Init accepted an invalid policy.\n"); + ca_fini(); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ca_ctx_share(enum pol_cong_avoid pol, + const char * name) +{ + void * c1; + void * c2; + + TEST_START("(%s)", name); + + if (ca_init(pol) < 0) { + printf("Failed to init ca for %s.\n", name); + goto fail; + } + + c1 = ca_ctx_get(ADDR_A, QOS_CUBE_BE); + if (c1 == NULL) { + printf("Failed to get ctx.\n"); + goto fail_init; + } + + c2 = ca_ctx_get(ADDR_A, QOS_CUBE_BE); + if (c2 == NULL) { + printf("Failed to get second ctx.\n"); + goto fail_c1; + } + +#ifdef IPCP_CA_PER_FLOW + if (c1 == c2) { + printf("Per-flow build shared a ctx across flows.\n"); + goto fail_c2; + } +#else + if (c1 != c2) { + printf("Aggregate build did not share ctx per (addr, qc).\n"); + goto fail_c2; + } +#endif + ca_ctx_put(c2); + ca_ctx_put(c1); + + ca_fini(); + + TEST_SUCCESS("(%s)", name); + + return TEST_RC_SUCCESS; + fail_c2: + ca_ctx_put(c2); + fail_c1: + ca_ctx_put(c1); + fail_init: + ca_fini(); + fail: + TEST_FAIL("(%s)", name); + return TEST_RC_FAIL; +} + +static int test_ca_ctx_share_all(void) +{ + int ret = 0; + size_t i; + + for (i = 0; i < CA_POLS; i++) + ret |= test_ca_ctx_share(ca_pols[i].pol, ca_pols[i].name); + + return ret; +} + +static int test_ca_ctx_distinct(void) +{ + void * a_be; + void * b_be; + void * a_video; + + TEST_START(); + + if (ca_init(CA_NONE) < 0) { + printf("Failed to init ca.\n"); + goto fail; + } + + a_be = ca_ctx_get(ADDR_A, QOS_CUBE_BE); + if (a_be == NULL) { + printf("Failed to get ctx.\n"); + goto fail_init; + } + + b_be = ca_ctx_get(ADDR_B, QOS_CUBE_BE); + if (b_be == NULL) { + printf("Failed to get ctx.\n"); + goto fail_a_be; + } + + a_video = ca_ctx_get(ADDR_A, QOS_CUBE_VIDEO); + if (a_video == NULL) { + printf("Failed to get ctx.\n"); + goto fail_b_be; + } + + if (a_be == b_be) { + printf("Distinct addresses shared a ctx.\n"); + goto fail_a_video; + } + + if (a_be == a_video) { + printf("Distinct qos cubes shared a ctx.\n"); + goto fail_a_video; + } + + ca_ctx_put(a_video); + ca_ctx_put(b_be); + ca_ctx_put(a_be); + + ca_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_a_video: + ca_ctx_put(a_video); + fail_b_be: + ca_ctx_put(b_be); + fail_a_be: + ca_ctx_put(a_be); + fail_init: + ca_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Refcount survival is an aggregate-only property. */ +#ifndef IPCP_CA_PER_FLOW +static int test_ca_ctx_refcount(void) +{ + void * c1; + void * c3; + + TEST_START(); + + if (ca_init(CA_NONE) < 0) { + printf("Failed to init ca.\n"); + goto fail; + } + + c1 = ca_ctx_get(ADDR_A, QOS_CUBE_BE); /* refs = 1 */ + if (c1 == NULL) { + printf("Failed to get ctx.\n"); + goto fail_init; + } + + if (ca_ctx_get(ADDR_A, QOS_CUBE_BE) == NULL) { /* refs = 2 */ + printf("Failed to get second ref.\n"); + goto fail_c1; + } + + ca_ctx_put(c1); /* refs = 1 */ + + c3 = ca_ctx_get(ADDR_A, QOS_CUBE_BE); /* refs = 2 */ + if (c3 == NULL) { + printf("Failed to get third ref.\n"); + goto fail_c1; + } + + if (c3 != c1) { + printf("Refcounted ctx freed while still referenced.\n"); + goto fail_c3; + } + + ca_ctx_put(c3); + ca_ctx_put(c1); + + ca_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_c3: + ca_ctx_put(c3); + fail_c1: + ca_ctx_put(c1); + fail_init: + ca_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The last put frees the interned ctx; a fresh get recreates it. */ +static int test_ca_ctx_recreate(void) +{ + void * c1; + void * c2; + void * c3; + + TEST_START(); + + if (ca_init(CA_NONE) < 0) { + printf("Failed to init ca.\n"); + goto fail; + } + + c1 = ca_ctx_get(ADDR_A, QOS_CUBE_BE); /* refs = 1 */ + if (c1 == NULL) { + printf("Failed to get ctx.\n"); + goto fail_init; + } + + ca_ctx_put(c1); /* refs = 0: freed and de-interned */ + + c2 = ca_ctx_get(ADDR_A, QOS_CUBE_BE); /* fresh entry */ + if (c2 == NULL) { + printf("Get after release did not recreate.\n"); + goto fail_init; + } + + c3 = ca_ctx_get(ADDR_A, QOS_CUBE_BE); /* refs = 2: shares */ + if (c3 == NULL) { + printf("Failed to share recreated ctx.\n"); + goto fail_c2; + } + + if (c3 != c2) { + printf("Recreated ctx did not intern.\n"); + goto fail_c3; + } + + ca_ctx_put(c3); + ca_ctx_put(c2); + + ca_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_c3: + ca_ctx_put(c3); + fail_c2: + ca_ctx_put(c2); + fail_init: + ca_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* ca_fini drains a ctx a flow left interned, with no leak. */ +static int test_ca_fini_drains(void) +{ + void * c1; + void * c2; + + TEST_START(); + + if (ca_init(CA_NONE) < 0) { + printf("Failed to init ca.\n"); + goto fail; + } + + c1 = ca_ctx_get(ADDR_A, QOS_CUBE_BE); /* refs = 1 */ + if (c1 == NULL) { + printf("Failed to get ctx.\n"); + goto fail_init; + } + + c2 = ca_ctx_get(ADDR_A, QOS_CUBE_BE); /* refs = 2 */ + if (c2 == NULL) { + printf("Failed to get second ref.\n"); + goto fail_init; + } + + /* Leave both refs live: ca_fini must drain and free the ctx. */ + ca_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + ca_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} +#endif /* !IPCP_CA_PER_FLOW */ + +int ca_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_ca_init_fini_all(); + ret |= test_ca_init_invalid(); + ret |= test_ca_ctx_share_all(); + ret |= test_ca_ctx_distinct(); +#ifndef IPCP_CA_PER_FLOW + ret |= test_ca_ctx_refcount(); + ret |= test_ca_ctx_recreate(); + ret |= test_ca_fini_drains(); +#endif + return ret; +} diff --git a/src/ipcpd/unicast/ca/tests/mb_ecn_test.c b/src/ipcpd/unicast/ca/tests/mb_ecn_test.c new file mode 100644 index 00000000..8e3a73da --- /dev/null +++ b/src/ipcpd/unicast/ca/tests/mb_ecn_test.c @@ -0,0 +1,2588 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Unit tests for multi-bit ECN congestion avoidance + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "mb-ecn.c" + +#include <test/test.h> + +#define MS (MILLION) /* one millisecond in ns */ +#define LEN 1000 /* default packet size (bytes) */ + +/* Create a context with the clock zeroed for deterministic time steps. */ +static struct mb_ecn_ctx * mk_ctx(void) +{ + struct mb_ecn_ctx * ctx; + + ctx = mb_ecn_ctx_create(); + if (ctx == NULL) + return NULL; + + ctx->rx_ts = 0; + ctx->rx_win = 0; + ctx->last_ts = 0; + ctx->last_ctrl = 0; + ctx->last_fb = 0; + ctx->last_loc = 0; + ctx->last_cap = 0; + + ctx->snd_byt = 0; + ctx->snd_win = 0; + ctx->snd_r0 = CA_RATE_INIT; + ctx->snd_rate = CA_RATE_INIT; + ctx->backlogged = true; + ctx->src_limited = false; + ctx->started = false; + + return ctx; +} + +/* + * Drive ctx as a fully backlogged flow: offer a packet every paced + * wait, so the offered load tracks the paced rate. Returns end time. + */ +static uint64_t drive_backlogged(struct mb_ecn_ctx * ctx, + uint64_t * ftag, + uint64_t t, + uint64_t dur, + size_t len) +{ + uint64_t end = t + dur; + time_t w; + + while (t < end) { + w = mb_ecn_snd(ctx, len, t, ftag); + t += w > 0 ? (uint64_t) w : 1; + } + + return t; +} + +static int test_mb_ecn_ctx_create_destroy(void) +{ + struct mb_ecn_ctx * ctx; + + TEST_START(); + + ctx = mb_ecn_ctx_create(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + if (ctx->rate != CA_RATE_INIT) { + printf("Bad initial rate %" PRIu64 ".\n", ctx->rate); + goto fail_ctx; + } + + if (ctx->rate_min != CA_RATE_MIN) { + printf("Bad initial floor %" PRIu64 ".\n", ctx->rate_min); + goto fail_ctx; + } + + if (ctx->vt != 0) { + printf("Bad initial virtual clock %" PRIu64 ".\n", ctx->vt); + goto fail_ctx; + } + + if (ctx->tx_cav) { + printf("Context did not start in slow start.\n"); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_mb_ecn_calc_ecn(void) +{ + uint8_t ecn; + + TEST_START(); + + /* A queue below one ECN quantum marks nothing. */ + ecn = 0; + mb_ecn_calc_ecn(CA_MARK_Q - 1, &ecn, QOS_CUBE_BE, 0); + if (ecn != 0) { + printf("Sub-quantum queue marked %u.\n", ecn); + goto fail; + } + + /* Queue depth maps to ecn = queued / CA_MARK_Q. */ + ecn = 0; + mb_ecn_calc_ecn(5 * CA_MARK_Q, &ecn, QOS_CUBE_BE, 0); + if (ecn != 5) { + printf("Expected ecn 5, got %u.\n", ecn); + goto fail; + } + + /* MAX keeps the larger value; a smaller mark cannot raise it. */ + ecn = 0x80; + mb_ecn_calc_ecn(CA_MARK_Q, &ecn, QOS_CUBE_BE, 0); + if (ecn != 0x80) { + printf("Expected ecn 0x80, got 0x%x.\n", ecn); + goto fail; + } + + ecn = 3; + mb_ecn_calc_ecn(4 * CA_MARK_Q, &ecn, QOS_CUBE_BE, 0); + if (ecn != 4) { + printf("Expected ecn 4, got %u.\n", ecn); + goto fail; + } + + /* A queue past 255 quanta saturates, it does not wrap to a low mark. */ + ecn = 0; + mb_ecn_calc_ecn(256 * CA_MARK_Q, &ecn, QOS_CUBE_BE, 0); + if (ecn != 255) { + printf("Deep queue wrapped: exp 255, got %u.\n", ecn); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The first mark after idle emits the raw value with zero latency. */ +static int test_mb_ecn_rcv_onset_immediate(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + if (!mb_ecn_rcv(ctx, LEN, 4, 0, &ece, &fcap, MS)) { + printf("Onset did not update.\n"); + goto fail_ctx; + } + + if (ece != 4 << CA_SHFT) { + printf("Onset ece: exp %u, got %u.\n", 4 << CA_SHFT, ece); + goto fail_ctx; + } + + if (mb_ecn_rcv(ctx, LEN, 4, 0, &ece, &fcap, 2 * MS)) { + printf("Mid-window packet updated.\n"); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A 50% duty mark square wave emits the time mean, not the last peak. */ +static int test_mb_ecn_rcv_window_mean(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + size_t upd; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + mb_ecn_rcv(ctx, LEN, 8, 0, &ece, &fcap, MS); + + /* The byte trigger closes every ~32 packets: two windows. */ + upd = 0; + for (i = 1; i <= 68; i++) { + time_t ecn = (i & 1) ? 8 : 0; + time_t t = MS + i * MS; + if (mb_ecn_rcv(ctx, LEN, ecn, 0, &ece, &fcap, t)) + upd++; + } + + if (upd != 2) { + printf("%zu updates in two windows.\n", upd); + goto fail_ctx; + } + + if (ece < 120 || ece > 136) { + printf("window mean: exp ~128, got %u.\n", ece); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * Two flows on the same wall-clock mark timeline, 15x apart in byte + * rate: the same congestion estimate, but the faster flow's window is + * shorter, so it feeds back more often (cadence tracks byte rate). + */ +static int test_mb_ecn_rcv_rate_independent(void) +{ + struct mb_ecn_ctx * a; + struct mb_ecn_ctx * b; + uint16_t ea; + uint16_t eb; + uint8_t fcap; + size_t ua; + size_t ub; + size_t i; + + TEST_START(); + + a = mk_ctx(); + b = mk_ctx(); + if (a == NULL || b == NULL) { + printf("Failed to create contexts.\n"); + goto fail_ctx; + } + + ea = 0; + eb = 0; + ua = 0; + ub = 0; + + /* 300 ms of sustained mark 8; a at 1 kpps, b at ~66 pps. */ + for (i = 1; i <= 300; i++) { + ua += mb_ecn_rcv(a, LEN, 8, 0, &ea, &fcap, i * MS) ? 1 : 0; + if (i % 15 != 0) + continue; + + ub += mb_ecn_rcv(b, LEN, 8, 0, &eb, &fcap, i * MS) ? 1 : 0; + } + + if (ea > eb + 32 || eb > ea + 32) { + printf("estimates diverge: %u vs %u.\n", ea, eb); + goto fail_ctx; + } + + if (ua < ub + 2) { + printf("cadence not rate-scaled: %zu vs %zu.\n", ua, ub); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * Two flows on one bottleneck, equal byte rate but 7.5x apart in + * packet size: the same congestion estimate and the same feedback + * cadence. Framing does not skew the control signal (fair share). + */ +static int test_mb_ecn_rcv_size_fair(void) +{ + struct mb_ecn_ctx * a; + struct mb_ecn_ctx * b; + uint16_t ea; + uint16_t eb; + uint8_t fcap; + size_t ua; + size_t ub; + uint64_t ta; + uint64_t tb; + + TEST_START(); + + a = mk_ctx(); + b = mk_ctx(); + if (a == NULL || b == NULL) { + printf("Failed to create contexts.\n"); + goto fail_ctx; + } + + ea = 0; + eb = 0; + ua = 0; + ub = 0; + ta = 0; + tb = 0; + + /* 1 MB/s each: a at 200 B / 200 us, b at 1500 B / 1.5 ms. */ + while (ta < 500 * MS) { + ta += 200 * 1000; + ua += mb_ecn_rcv(a, 200, 8, 0, &ea, &fcap, ta) ? 1 : 0; + } + + while (tb < 500 * MS) { + tb += 1500 * 1000; + ub += mb_ecn_rcv(b, 1500, 8, 0, &eb, &fcap, tb) ? 1 : 0; + } + + if (ea != 8 << CA_SHFT || eb != 8 << CA_SHFT) { + printf("size-skewed estimate: %u vs %u.\n", ea, eb); + goto fail_ctx; + } + + if (ua > ub + 3 || ub > ua + 3) { + printf("cadence skewed by size: %zu vs %zu.\n", ua, ub); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Release emits exactly one 0 and leaves the estimator fully idle. */ +static int test_mb_ecn_rcv_release_exact_zero(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + size_t ends; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + for (i = 1; i <= 140; i++) + mb_ecn_rcv(ctx, LEN, 15, 0, &ece, &fcap, i * MS); + + ends = 0; + for (i = 141; i <= 350; i++) { + if (!mb_ecn_rcv(ctx, LEN, 0, 0, &ece, &fcap, i * MS)) + continue; + + if (ece == 0) + ends++; + } + + if (ends != 1) { + printf("end of congestion fired %zu times.\n", ends); + goto fail_ctx; + } + + if (ctx->rx_ece != 0 || ctx->rx_acc != 0) { + printf("estimator not idle: ece %u acc %" PRIu64 ".\n", + ctx->rx_ece, ctx->rx_acc); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A gap past the window restarts fresh: no stale, diluted estimate. */ +static int test_mb_ecn_rcv_gap_restart(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + uint64_t t; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + mb_ecn_rcv(ctx, LEN, 6, 0, &ece, &fcap, MS); + mb_ecn_rcv(ctx, LEN, 6, 0, &ece, &fcap, 2 * MS); + + t = 2 * MS + 10 * CA_TW_INIT; + if (!mb_ecn_rcv(ctx, LEN, 5, 0, &ece, &fcap, t)) { + printf("gap restart did not update.\n"); + goto fail_ctx; + } + + if (ece != 5 << CA_SHFT) { + printf("gap restart: exp %u, got %u.\n", 5 << CA_SHFT, ece); + goto fail_ctx; + } + + t += 10 * CA_TW_INIT; + if (!mb_ecn_rcv(ctx, LEN, 0, 0, &ece, &fcap, t) || ece != 0) { + printf("gap with clean packet did not end: %u.\n", ece); + goto fail_ctx; + } + + if (mb_ecn_rcv(ctx, LEN, 0, 0, &ece, &fcap, t + MS)) { + printf("idle packet updated.\n"); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Max marks at max gaps: exact ceiling, no overflow past the edge. */ +static int test_mb_ecn_rcv_accum_bounds(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + uint64_t t; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + mb_ecn_rcv(ctx, LEN, 15, 0, &ece, &fcap, MS); + + /* Two packets at dt just under CA_TW_INIT straddle the boundary. */ + t = MS + CA_TW_INIT - 1; + if (mb_ecn_rcv(ctx, LEN, 15, 0, &ece, &fcap, t)) { + printf("update before the window closed.\n"); + goto fail_ctx; + } + + t += CA_TW_INIT - 1; + if (!mb_ecn_rcv(ctx, LEN, 15, 0, &ece, &fcap, t)) { + printf("no update at the window boundary.\n"); + goto fail_ctx; + } + + if (ece != 15 << CA_SHFT) { + printf("ceiling: exp %u, got %u.\n", 15 << CA_SHFT, ece); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Scale-free density: the window holds ~CA_N_TARGET packets at any rate. */ +static int test_mb_ecn_rcv_window_holds_target(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + uint16_t last_ece; + uint64_t rates[4]; + uint64_t ia; + uint64_t t; + size_t closes; + size_t since; + size_t count; + size_t ri; + + TEST_START(); + + rates[0] = 5000000; + rates[1] = 10000000; + rates[2] = 50000000; + rates[3] = 100000000; + + ctx = NULL; + for (ri = 0; ri < 4; ri++) { + ia = 8000ULL * BILLION / rates[ri]; + t = 0; + closes = 0; + since = 0; + count = 0; + last_ece = 0; + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* Warm past the ramp from CA_TW_INIT, then time one gap. */ + while (closes < 42) { + t += ia; + since++; + if (!mb_ecn_rcv(ctx, LEN, 8, 0, &ece, &fcap, t)) + continue; + closes++; + if (closes == 41) { + since = 0; + } else if (closes == 42) { + count = since - 1; + last_ece = ece; + } + } + + if (count < 8 || count > 32) { + printf("rate %" PRIu64 ": %zu pkts/window.\n", + rates[ri], count); + goto fail_ctx; + } + + if (last_ece < 224 || last_ece > 288) { + printf("rate %" PRIu64 ": ece %u ~256.\n", + rates[ri], last_ece); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + ctx = NULL; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * The window floors at CA_TW_MIN, tracks the rate below the old knee, + * and only a pathological fold hits the CA_TW_ABSMAX ceiling. + */ +static int test_mb_ecn_rcv_window_clip_bounds(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + uint64_t ia; + uint64_t t; + size_t closes; + + TEST_START(); + + /* 1 GbE is above the high knee: the window floors at CA_TW_MIN. */ + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ia = 8000ULL * BILLION / 1000000000ULL; + t = 0; + closes = 0; + while (closes < 40) { + t += ia; + if (mb_ecn_rcv(ctx, LEN, 8, 0, &ece, &fcap, t)) + closes++; + } + + if (ctx->rx_tw != CA_TW_MIN) { + printf("high-rate window: exp %" PRIu64 ", got %" PRIu64 + ".\n", (uint64_t) CA_TW_MIN, ctx->rx_tw); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + /* 1 Mbps: past the old knee, ~16 pkts = 16 * 8 ms = 131 ms. */ + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ia = 8000ULL * BILLION / 1000000ULL; + t = 0; + closes = 0; + while (closes < 40) { + t += ia; + if (mb_ecn_rcv(ctx, LEN, 8, 0, &ece, &fcap, t)) + closes++; + } + + if (ctx->rx_tw < 120 * MS || ctx->rx_tw > 140 * MS) { + printf("low-rate window: exp ~131 ms, got %" PRIu64 ".\n", + ctx->rx_tw); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + /* A near-empty window folds a huge target: ceiling holds. */ + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + mb_ecn_rcv(ctx, 10, 8, 0, &ece, &fcap, MS); + mb_ecn_rcv(ctx, 10, 8, 0, &ece, &fcap, MS + CA_TW_INIT); + + if (ctx->rx_tw != CA_TW_ABSMAX) { + printf("window ceiling breached: %" PRIu64 ".\n", + ctx->rx_tw); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A CA-limited slow flow grows its window to hold ~16 packets. */ +static int test_mb_ecn_rcv_slow_window(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* 1400 B every 171 ms (~8 KB/s), sustained mark 8. */ + for (i = 1; i <= 100; i++) + mb_ecn_rcv(ctx, 1400, 8, 0, &ece, &fcap, i * 171 * MS); + + /* Target window 16 * 1000 B at 8187 B/s ~= 2.0 s. */ + if (ctx->rx_tw < 1400 * MS || ctx->rx_tw > 2800 * MS) { + printf("slow window: exp ~2 s, got %" PRIu64 ".\n", + ctx->rx_tw); + goto fail_ctx; + } + + /* Steady mark 8 emits exactly 256 once the window settles. */ + if (ece != 8 << CA_SHFT) { + printf("slow-flow estimate: exp %u, got %u.\n", + 8 << CA_SHFT, ece); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A physically maximal window must fold without overflow or wrap. */ +static int test_mb_ecn_rcv_no_overflow_highrate(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + bool ok; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* Open a window, then inject a maximal byte count and span. */ + mb_ecn_rcv(ctx, LEN, 15, 0, &ece, &fcap, 0); + ctx->rx_byt = CA_RATE_MAX / 8; + ctx->rx_ts = 2 * CA_TW_INIT - 2; + + ok = mb_ecn_rcv(ctx, LEN, 15, 0, &ece, &fcap, 2 * CA_TW_INIT - 1); + + if (!ok) { + printf("max-window close did not fire.\n"); + goto fail_ctx; + } + + if (ece > 8160) { + printf("estimate %u wrapped.\n", ece); + goto fail_ctx; + } + + /* A wrapped numerator drives rx_tw to MAX; it must descend. */ + if (ctx->rx_tw >= CA_TW_INIT || ctx->rx_tw < CA_TW_MIN) { + printf("window %" PRIu64 " did not descend.\n", ctx->rx_tw); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * The sender holds a mark across the full inter-feedback gap (TTL > + * 2 * CA_TW_INIT) and a repeated mark must not re-fire the one-sided lead. + */ +static int test_mb_ecn_ece_ttl_covers_cadence(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t prev; + uint64_t t; + uint64_t ftag = 0; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = (uint64_t) 100 << 20; + ctx->tx_cav = true; + + mb_ecn_ece(ctx, 100, 0, MS); + mb_ecn_snd(ctx, LEN, MS, &ftag); + + /* Sends between feedbacks spaced 2 * CA_TW_INIT + 5 ms apart. */ + t = MS; + for (i = 0; i < 4; i++) { + t += (2 * CA_TW_INIT + 5 * MS) / 4; + mb_ecn_snd(ctx, LEN, t, &ftag); + if (ctx->tx_ece == 0) { + printf("mark cleared inside the feedback gap.\n"); + goto fail_ctx; + } + } + + /* Same mark again: rise 0, so only the proportional cut. */ + prev = ctx->rate; + t += MS; + mb_ecn_ece(ctx, 100, 0, t); + mb_ecn_snd(ctx, LEN, t, &ftag); + + if (prev - ctx->rate > prev / 100) { + printf("phantom lead cut: %" PRIu64 " -> %" PRIu64 ".\n", + prev, ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_mb_ecn_slow_start(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t prev; + uint64_t t; + uint64_t ftag = 0; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + prev = ctx->rate; + t = 0; + + /* No feedback: the flow stays in slow start and grows each step. */ + for (i = 0; i < 16; i++) { + t += MS; + mb_ecn_snd(ctx, LEN, t, &ftag); + if (ctx->rate <= prev) { + printf("rate did not grow: %" PRIu64 ".\n", ctx->rate); + goto fail_ctx; + } + + prev = ctx->rate; + } + + /* Exponential ramp doubles in ~ln2 * CA_SS_TC ~= 14 ms. */ + if (ctx->rate < 2 * CA_RATE_INIT) { + printf("slow start too slow: %" PRIu64 ".\n", ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_mb_ecn_dt_scaling_invariant(void) +{ + struct mb_ecn_ctx * a; + struct mb_ecn_ctx * b; + uint64_t inc_a; + uint64_t inc_b; + uint64_t t; + uint64_t fta = 0; + uint64_t ftb = 0; + size_t i; + + TEST_START(); + + a = mk_ctx(); + b = mk_ctx(); + if (a == NULL || b == NULL) { + printf("Failed to create contexts.\n"); + goto fail_ctx; + } + + /* Leave slow start; seed a realistic rate (truncation-free). */ + mb_ecn_ece(a, 0, 0, 0); + mb_ecn_ece(b, 0, 0, 0); + a->rate = (uint64_t) 10 << 20; + b->rate = (uint64_t) 10 << 20; + a->r_bkt = a->rate; + b->r_bkt = b->rate; + + /* a: one 30 ms step (under one washout bucket, so it stays out). */ + mb_ecn_snd(a, LEN, 30 * MS, &fta); + + /* b: thirty 1 ms steps over the same 30 ms. */ + t = 0; + for (i = 0; i < 30; i++) { + t += MS; + mb_ecn_snd(b, LEN, t, &ftb); + } + + inc_a = a->rate - ((uint64_t) 10 << 20); + inc_b = b->rate - ((uint64_t) 10 << 20); + + /* Equal within 1 %; the small gap is per-step integer truncation. */ + if (inc_a == 0 || inc_b == 0) { + printf("no additive increase: %" PRIu64 " %" PRIu64 ".\n", + inc_a, inc_b); + goto fail_ctx; + } + + if (inc_a > inc_b + inc_a / 100 || inc_b > inc_a + inc_a / 100) { + printf("cadence-dependent AI: %" PRIu64 " vs %" PRIu64 ".\n", + inc_a, inc_b); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_mb_ecn_multiplicative_decrease(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t prev; + uint64_t t; + uint64_t ftag = 0; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = (uint64_t) 100 << 20; + prev = ctx->rate; + t = 0; + + for (i = 0; i < 10; i++) { + t += MS; + mb_ecn_ece(ctx, CA_ECE_REF, 0, t); + mb_ecn_snd(ctx, LEN, t, &ftag); + if (ctx->rate >= prev) { + printf("rate did not shrink: %" PRIu64 ".\n", + ctx->rate); + goto fail_ctx; + } + + prev = ctx->rate; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_mb_ecn_rate_floor(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* Cut larger than headroom must clamp; stay inside CA_ECE_TTL. */ + ctx->rate = CA_RATE_MIN + 1000; + mb_ecn_ece(ctx, CA_ECE_REF, 0, 0); + mb_ecn_snd(ctx, LEN, 30 * MS, &ftag); + + if (ctx->rate != CA_RATE_MIN) { + printf("rate floor breached: %" PRIu64 ".\n", ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_mb_ecn_fixed_point(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t exp; + uint64_t t; + uint64_t ftag = 0; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + exp = CA_AI_RATE * CA_ECE_REF / + (128 - CA_ECE_REF * BILLION / CA_PROBE_TC); + t = 0; + + /* ~20 s: the probe raises the loop time constant to CA_PROBE_TC. */ + for (i = 0; i < 20000; i++) { + t += MS; + mb_ecn_ece(ctx, 128, 0, t); + mb_ecn_snd(ctx, LEN, t, &ftag); + } + + if (ctx->rate < exp - exp / 4 || ctx->rate > exp + exp / 4) { + printf("no fixed point: exp ~%" PRIu64 ", got %" PRIu64 ".\n", + exp, ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * One-sided lead: a mark rise cuts rate * rise / (REF * DEN) once; + * a flat or falling mark leaves only the dt-scaled proportional cut. + */ +static int test_mb_ecn_lead_cut(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t prev; + uint64_t drop; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = (uint64_t) 100 << 20; + ctx->tx_cav = true; + + /* Rise 0 -> 256: lead cuts ~rise/(REF*DEN) = 1/8 of the rate. */ + prev = ctx->rate; + mb_ecn_ece(ctx, 256, 0, MS); + mb_ecn_snd(ctx, LEN, MS, &ftag); + drop = prev - ctx->rate; + if (drop < prev / 10) { + printf("lead cut missing: dropped %" PRIu64 ".\n", drop); + goto fail_ctx; + } + + /* Flat mark: rise 0, only the ~0.05%% proportional cut. */ + prev = ctx->rate; + mb_ecn_ece(ctx, 256, 0, 2 * MS); + mb_ecn_snd(ctx, LEN, 2 * MS, &ftag); + drop = prev - ctx->rate; + if (drop > prev / 100) { + printf("flat mark over-cut: dropped %" PRIu64 ".\n", drop); + goto fail_ctx; + } + + /* Falling mark: one-sided lead must not fire. */ + prev = ctx->rate; + mb_ecn_ece(ctx, 64, 0, 3 * MS); + mb_ecn_snd(ctx, LEN, 3 * MS, &ftag); + drop = prev > ctx->rate ? prev - ctx->rate : 0; + if (drop > prev / 100) { + printf("falling mark cut: dropped %" PRIu64 ".\n", drop); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A local first-hop mark exits slow start with no feedback needed. */ +static int test_mb_ecn_slow_start_local_brake(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t prev; + uint64_t ftag = 0; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + for (i = 1; i <= 50; i++) + mb_ecn_snd(ctx, LEN, i * MS, &ftag); + + if (ctx->tx_cav) { + printf("Left slow start without any signal.\n"); + goto fail_ctx; + } + + prev = ctx->rate; + mb_ecn_loc(ctx, 1, 50 * MS); + if (!ctx->tx_cav) { + printf("Local mark did not exit slow start.\n"); + goto fail_ctx; + } + + mb_ecn_snd(ctx, LEN, 51 * MS, &ftag); + if (ctx->rate > prev + prev / 20) { + printf("SS ramp survived the brake: %" PRIu64 ".\n", + ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A clean path still ramps to line rate in well under a second. */ +static int test_mb_ecn_slow_start_clean_ramp(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* Backlogged, no marks: slow start sprints in a couple windows. */ + drive_backlogged(ctx, &ftag, MS, 2 * CA_SND_WIN, LEN); + + if (ctx->rate < (1ULL << 24)) { + printf("backlogged slow start too slow: %" PRIu64 ".\n", + ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * A sender starved of send-path control steps recovers through the + * feedback path: honest elapsed time, at most a 50% cut per step. + */ +static int test_mb_ecn_starved_decrease_escape(void) +{ + struct mb_ecn_ctx * ctx; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = (uint64_t) 100 << 20; + ctx->tx_cav = true; + + /* Feedback arrives once per second; no sends at all. */ + for (i = 1; i <= 6; i++) + mb_ecn_ece(ctx, 480, 0, i * BILLION); + + if (ctx->rate > (5ULL << 19)) { + printf("still wedged at %" PRIu64 " B/s.\n", ctx->rate); + goto fail_ctx; + } + + if (ctx->rate < CA_RATE_MIN) { + printf("rate floor breached: %" PRIu64 ".\n", ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* With feedback fully dead, the local mark alone recovers the rate. */ +static int test_mb_ecn_starved_local_fallback(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t t; + uint64_t ftag = 0; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = (uint64_t) 100 << 20; + + for (i = 1; i <= 7; i++) { + t = i * BILLION; + mb_ecn_loc(ctx, 15, t); + mb_ecn_snd(ctx, LEN, t, &ftag); + } + + if (!ctx->tx_cav) { + printf("Local mark did not exit slow start.\n"); + goto fail_ctx; + } + + if (ctx->rate > (5ULL << 19)) { + printf("still wedged at %" PRIu64 " B/s.\n", ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * MD Δt-invariance: the same elapsed time under the same mark cuts + * the same, in one big step or five small ones. + */ +static int test_mb_ecn_decrease_dt_invariant(void) +{ + struct mb_ecn_ctx * a; + struct mb_ecn_ctx * b; + uint64_t cut_a; + uint64_t cut_b; + uint64_t r0; + uint64_t fta = 0; + uint64_t ftb = 0; + size_t i; + + TEST_START(); + + a = mk_ctx(); + b = mk_ctx(); + if (a == NULL || b == NULL) { + printf("Failed to create contexts.\n"); + goto fail_ctx; + } + + r0 = (uint64_t) 100 << 20; + a->rate = r0; + b->rate = r0; + + mb_ecn_ece(a, 256, 0, 0); + mb_ecn_ece(b, 256, 0, 0); + + /* a: one 50 ms step; b: five 10 ms steps (both within DT_CAP). */ + mb_ecn_snd(a, LEN, 50 * MS, &fta); + + for (i = 1; i <= 5; i++) + mb_ecn_snd(b, LEN, i * 10 * MS, &ftb); + + cut_a = r0 - a->rate; + cut_b = r0 - b->rate; + + if (cut_a == 0 || cut_b == 0) { + printf("no cut: %" PRIu64 " %" PRIu64 ".\n", cut_a, cut_b); + goto fail_ctx; + } + + /* Within 10%: residual is Euler compounding of MD and the probe. */ + if (cut_a > cut_b + cut_a / 10 || cut_b > cut_a + cut_a / 10) { + printf("cadence-dependent MD: %" PRIu64 " vs %" PRIu64 + ".\n", cut_a, cut_b); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Resuming after a long idle gap: bounded AI, no cut from stale marks. */ +static int test_mb_ecn_idle_resume_bounded(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t prev; + uint64_t bump; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = (uint64_t) 10 << 20; + + mb_ecn_loc(ctx, 15, MS); + mb_ecn_ece(ctx, 480, 0, MS); + + prev = ctx->rate; + + /* 600 s later: both signals stale; one capped AI + probe step. */ + mb_ecn_snd(ctx, LEN, 600 * BILLION, &ftag); + + if (ctx->rate < prev) { + printf("stale mark cut the rate: %" PRIu64 ".\n", + ctx->rate); + goto fail_ctx; + } + + bump = CA_AI_RATE * CA_DT_CAP / BILLION; + bump += (prev + bump) * CA_DT_CAP / CA_PROBE_TC; + + if (ctx->rate > prev + bump + 2) { + printf("idle resume cap breached: %" PRIu64 ".\n", ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The congestion signal ages out on wall-clock time, not packet count. */ +static int test_mb_ecn_ece_staleness(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* A fast flow's control step caches the floor TTL. */ + ctx->rate = (uint64_t) 1 << 20; + ctx->tx_cav = true; + mb_ecn_snd(ctx, LEN, MS, &ftag); + + if (ctx->ece_ttl != CA_ECE_TTL) { + printf("Fast-flow TTL: exp %" PRIu64 ", got %" PRIu64 ".\n", + (uint64_t) CA_ECE_TTL, ctx->ece_ttl); + goto fail_ctx; + } + + mb_ecn_ece(ctx, CA_ECE_REF, 0, 2 * MS); + + /* Just inside the TTL: the signal is still held. */ + mb_ecn_snd(ctx, LEN, 2 * MS + ctx->ece_ttl, &ftag); + if (ctx->tx_ece == 0) { + printf("signal aged out too early.\n"); + goto fail_ctx; + } + + /* Past the TTL without feedback: the signal is cleared. */ + mb_ecn_snd(ctx, LEN, 2 * MS + ctx->ece_ttl + 1, &ftag); + if (ctx->tx_ece != 0) { + printf("stale signal not cleared: %u.\n", ctx->tx_ece); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The staleness horizon stretches with a slow flow's window. */ +static int test_mb_ecn_ece_ttl_tracks_rate(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t want; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = 8192; + ctx->rate_min = 8192; + ctx->ai_rate = 0; + ctx->tx_cav = true; + mb_ecn_snd(ctx, 1400, MS, &ftag); + + want = (1 << CA_TW_GAP_SHFT) * CA_RX_WBYTES * BILLION / ctx->rate; + if (ctx->ece_ttl != want) { + printf("Slow-flow TTL: exp %" PRIu64 ", got %" PRIu64 ".\n", + want, ctx->ece_ttl); + goto fail_ctx; + } + + if (ctx->ece_ttl < 7 * (uint64_t) BILLION) { + printf("TTL did not stretch: %" PRIu64 ".\n", + ctx->ece_ttl); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * First packet of a flow starts at the clock; a same-instant second + * packet leads by its length and is paced by lead / rate. + */ +static int test_mb_ecn_sfq_pace(void) +{ + struct mb_ecn_ctx * ctx; + time_t wait; + uint64_t ftag = 0; + time_t want; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = 1U << 20; + ctx->inv_rate = mb_ecn_rate_inv(ctx->rate); + + /* First send: start tag equals the clock, so no wait. */ + wait = mb_ecn_snd(ctx, 1500, 0, &ftag); + if (wait != 0) { + printf("first packet waited %ld, expected 0.\n", (long) wait); + goto fail_ctx; + } + + /* Same instant (dt = 0): the flow now leads by 1500 B. */ + wait = mb_ecn_snd(ctx, 1500, 0, &ftag); + want = (time_t) ((uint64_t) 1500 * BILLION / ctx->rate); + + if (wait != want) { + printf("paced wait %ld, expected %ld.\n", + (long) wait, (long) want); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * The proportional probe grows the rate by the same fraction per unit + * time regardless of the absolute rate: two clean flows 100x apart in + * rate grow by the same ratio. Deleting the probe leaves only the tiny + * additive increase, failing the growth floor. + */ +static int test_mb_ecn_probe_scale_invariant(void) +{ + struct mb_ecn_ctx * a; + struct mb_ecn_ctx * b; + uint64_t ra0; + uint64_t rb0; + double ga; + double gb; + uint64_t t; + size_t i; + + TEST_START(); + + a = mk_ctx(); + b = mk_ctx(); + if (a == NULL || b == NULL) { + printf("Failed to create contexts.\n"); + goto fail_ctx; + } + + /* Clean path (mark 0), out of slow start, backlogged, 100x apart. */ + mb_ecn_ece(a, 0, 0, 0); + mb_ecn_ece(b, 0, 0, 0); + a->rate = (uint64_t) 10 << 20; + b->rate = (uint64_t) 1000 << 20; + a->r_bkt = a->rate; + b->r_bkt = b->rate; + a->backlogged = true; + b->backlogged = true; + ra0 = a->rate; + rb0 = b->rate; + + /* Drive control via the feedback path so backlogged stays set. */ + t = 0; + for (i = 0; i < 500; i++) { + t += MS; + mb_ecn_ece(a, 0, 0, t); + mb_ecn_ece(b, 0, 0, t); + } + + ga = (double) a->rate / ra0; + gb = (double) b->rate / rb0; + + if (ga < gb - gb / 50 || gb < ga - ga / 50) { + printf("probe not scale-invariant: %.4f vs %.4f.\n", ga, gb); + goto fail_ctx; + } + + /* And it must actually grow: the probe is present, not deleted. */ + if (ga < 1.05) { + printf("probe did not grow the rate: %.4f.\n", ga); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(a); + mb_ecn_ctx_destroy(b); + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * The proportional probe e-folds the rate over CA_PROBE_TC: a clean flow + * grows by ~e in 8 s. Pinned to a literal e-band so a mistuned + * CA_PROBE_TC (e.g. 4 s gives e^2) fails. + */ +static int test_mb_ecn_probe_time_constant(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t r0; + double ratio; + uint64_t t; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* Clean path, out of slow start, backlogged, below the ceiling. */ + mb_ecn_ece(ctx, 0, 0, 0); + ctx->rate = (uint64_t) 1 << 30; + ctx->r_bkt = ctx->rate; + ctx->backlogged = true; + r0 = ctx->rate; + + /* 8000 x 1 ms of clean growth, driven via the feedback path. */ + t = 0; + for (i = 0; i < 8000; i++) { + t += MS; + mb_ecn_ece(ctx, 0, 0, t); + } + + /* Washout damps the probe to ~4/3 TC, so 8 s -> ~2.12x. */ + ratio = (double) ctx->rate / r0; + if (ratio < 2.0 || ratio > 2.25) { + printf("probe TC off: exp ~2.12, got %.3fx over 8 s.\n", + ratio); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The fed-back capacity is the MIN of the nonzero caps in the window. */ +static int test_mb_ecn_rcv_cap_window_min(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + uint64_t t; + bool upd; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* Onset packet carries no capacity: feed back unknown. */ + if (!mb_ecn_rcv(ctx, LEN, 8, 0, &ece, &fcap, MS)) { + printf("Onset did not update.\n"); + goto fail_ctx; + } + + if (fcap != 0) { + printf("Onset fed back cap: exp 0, got %u.\n", fcap); + goto fail_ctx; + } + + mb_ecn_rcv(ctx, LEN, 8, 40, &ece, &fcap, 2 * MS); + mb_ecn_rcv(ctx, LEN, 8, 36, &ece, &fcap, 3 * MS); + + t = 3 * MS + CA_TW_INIT; + if (!mb_ecn_rcv(ctx, LEN, 8, 0, &ece, &fcap, t)) { + printf("Window did not close.\n"); + goto fail_ctx; + } + + if (fcap != 36) { + printf("Window min cap: exp 36, got %u.\n", fcap); + goto fail_ctx; + } + + /* The next window starts unknown; follow the adapted rx_tw. */ + upd = false; + for (i = 0; i < 128 && !upd; i++) { + t += CA_TW_INIT; + upd = mb_ecn_rcv(ctx, LEN, 8, 0, &ece, &fcap, t); + } + + if (!upd) { + printf("Second window did not close.\n"); + goto fail_ctx; + } + + if (fcap != 0) { + printf("Stale cap %u leaked into the next window.\n", fcap); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Onset and gap restarts emit the triggering packet's cap, fresh. */ +static int test_mb_ecn_rcv_cap_onset_fresh(void) +{ + struct mb_ecn_ctx * ctx; + uint16_t ece; + uint8_t fcap; + uint64_t t; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + if (!mb_ecn_rcv(ctx, LEN, 4, 77, &ece, &fcap, MS)) { + printf("Onset did not update.\n"); + goto fail_ctx; + } + + if (fcap != 77) { + printf("Onset cap: exp 77, got %u.\n", fcap); + goto fail_ctx; + } + + mb_ecn_rcv(ctx, LEN, 4, 50, &ece, &fcap, 2 * MS); + + /* A gap restart must not fold in the stale window min. */ + t = 2 * MS + 5 * CA_TW_INIT; + if (!mb_ecn_rcv(ctx, LEN, 4, 90, &ece, &fcap, t)) { + printf("Gap restart did not update.\n"); + goto fail_ctx; + } + + if (fcap != 90) { + printf("Gap restart cap: exp 90, got %u.\n", fcap); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Fed-back capacity derives the floor and slope: EWMA toward C/32. */ +static int test_mb_ecn_ece_cap_derives_rates(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t tgt; + uint64_t want; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* Code 120 = 2^30 B/s; target floor = 2^25 B/s. */ + tgt = cap_dec(120) >> CA_CAP_SHFT; + + mb_ecn_ece(ctx, 100, 120, MS); + + want = CA_RATE_MIN + ((tgt - CA_RATE_MIN) >> CA_CAP_SM_SHFT); + if (ctx->rate_min != want) { + printf("Floor: exp %" PRIu64 ", got %" PRIu64 ".\n", + want, ctx->rate_min); + goto fail_ctx; + } + + if (ctx->ai_rate != ctx->rate_min) { + printf("AI slope did not track the floor.\n"); + goto fail_ctx; + } + + mb_ecn_ece(ctx, 100, 120, 2 * MS); + + want += (tgt - want) >> CA_CAP_SM_SHFT; + if (ctx->rate_min != want) { + printf("Floor EWMA: exp %" PRIu64 ", got %" PRIu64 ".\n", + want, ctx->rate_min); + goto fail_ctx; + } + + if (ctx->n_cap != 2) { + printf("Capacity updates: exp 2, got %" PRIu64 ".\n", + ctx->n_cap); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Feedback without a capacity leaves the derived rates untouched. */ +static int test_mb_ecn_ece_cap_zero_keeps_rates(void) +{ + struct mb_ecn_ctx * ctx; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + mb_ecn_ece(ctx, 100, 0, MS); + + if (ctx->rate_min != CA_RATE_MIN || ctx->ai_rate != CA_AI_RATE) { + printf("Unknown cap moved the derived rates.\n"); + goto fail_ctx; + } + + if (ctx->n_cap != 0) { + printf("Unknown cap counted as an update.\n"); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The derived floor clamps to [CA_RATE_MIN, CA_RMIN_MAX]. */ +static int test_mb_ecn_ece_cap_clamps(void) +{ + struct mb_ecn_ctx * ctx; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* A path slower than the default floor cannot lower it. */ + mb_ecn_ece(ctx, 100, 1, MS); + + if (ctx->rate_min != CA_RATE_MIN) { + printf("Slow path lowered the floor: %" PRIu64 ".\n", + ctx->rate_min); + goto fail_ctx; + } + + /* A absurdly fast path saturates at the ceiling. */ + for (i = 1; i <= 40; i++) + mb_ecn_ece(ctx, 100, 255, (1 + i) * MS); + + if (ctx->rate_min > CA_RMIN_MAX) { + printf("Floor above the ceiling: %" PRIu64 ".\n", + ctx->rate_min); + goto fail_ctx; + } + + if (ctx->rate_min < CA_RMIN_MAX - 4) { + printf("Floor did not reach the ceiling: %" PRIu64 ".\n", + ctx->rate_min); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Stale capacity reverts the derived rates to the defaults. */ +static int test_mb_ecn_cap_ttl_reverts(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + mb_ecn_ece(ctx, 100, 120, MS); + + if (ctx->rate_min == CA_RATE_MIN) { + printf("Capacity did not derive a floor.\n"); + goto fail_ctx; + } + + /* Just inside the TTL: the derived rates hold. */ + mb_ecn_snd(ctx, LEN, MS + (ctx->ece_ttl << CA_CAP_TTL_SHFT), &ftag); + + if (ctx->rate_min == CA_RATE_MIN) { + printf("Derived rates reverted too early.\n"); + goto fail_ctx; + } + + /* Past the TTL: back to the defaults. */ + mb_ecn_snd(ctx, LEN, MS + (ctx->ece_ttl << CA_CAP_TTL_SHFT) + 1, + &ftag); + + if (ctx->rate_min != CA_RATE_MIN || ctx->ai_rate != CA_AI_RATE) { + printf("Stale capacity kept the derived rates.\n"); + goto fail_ctx; + } + + if (ctx->tx_cap != 0) { + printf("Stale capacity code not cleared.\n"); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The control law uses the per-ctx AI slope. */ +static int test_mb_ecn_ctrl_per_ctx_ai(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t want; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* Leave slow start; raise the slope as capacity would. */ + mb_ecn_ece(ctx, 0, 0, 0); + ctx->rate = (uint64_t) 10 << 20; + ctx->r_bkt = ctx->rate; + ctx->ai_rate = 16 * CA_AI_RATE; + + want = ctx->rate + ctx->ai_rate * (30 * MS) / BILLION; + want += want * (30 * MS) / CA_PROBE_TC; + + mb_ecn_snd(ctx, LEN, 30 * MS, &ftag); + + if (ctx->rate != want) { + printf("AI not per-ctx: exp %" PRIu64 ", got %" PRIu64 ".\n", + want, ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The rate clamp honours the per-ctx derived floor. */ +static int test_mb_ecn_ctrl_per_ctx_floor(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate_min = (uint64_t) 1 << 20; + ctx->rate = ((uint64_t) 1 << 20) + 1000; + + mb_ecn_ece(ctx, CA_ECE_REF, 0, 0); + mb_ecn_snd(ctx, LEN, 30 * MS, &ftag); + + if (ctx->rate != ctx->rate_min) { + printf("Floor not per-ctx: %" PRIu64 ".\n", ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * A paced flow slower than one packet per CA_DT_CAP must not decay: + * the gap credit law grants its true elapsed service, so the lead + * stays pinned at ~one packet instead of growing without bound. + */ +static int test_mb_ecn_snd_slow_rate_paced(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + uint64_t t = 0; + time_t wait; + size_t i; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* 8 KB/s, 1400 B packets: inter-send gap ~171 ms > CA_DT_CAP. */ + ctx->rate = 8192; + ctx->rate_min = 8192; + ctx->ai_rate = 0; + ctx->inv_rate = mb_ecn_rate_inv(8192); + ctx->tx_cav = true; + + for (i = 0; i < 50; i++) { + wait = mb_ecn_snd(ctx, 1400, t, &ftag); + t += wait > 0 ? (uint64_t) wait : 1; + } + + if (ctx->lead > 2 * 1400) { + printf("Pacer starves a slow flow: lead %" PRIu64 ".\n", + ctx->lead); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * A long idle makes the aggregate source-limited, so the offered-load + * ceiling bounds the resume rate (hence the burst) well below the + * pre-idle rate. + */ +static int test_mb_ecn_snd_idle_burst_bound(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = (uint64_t) 1 << 20; + ctx->inv_rate = mb_ecn_rate_inv(ctx->rate); + ctx->tx_cav = true; + + mb_ecn_snd(ctx, 1400, 0, &ftag); /* warm-up: sets started */ + + /* 600 s idle. */ + mb_ecn_snd(ctx, 1400, 600 * BILLION, &ftag); + + if (ctx->backlogged) { + printf("long idle did not clear backlogged.\n"); + goto fail_ctx; + } + + if (ctx->rate >= ((uint64_t) 1 << 20)) { + printf("idle resume rate not ceiling-bounded: %" PRIu64 + ".\n", ctx->rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A deep backlog at a low rate must not wrap the wait computation. */ +static int test_mb_ecn_snd_wait_no_overflow(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t want; + uint64_t ftag; + time_t wait; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->rate = 8192; + ctx->inv_rate = mb_ecn_rate_inv(8192); + + /* 128 flows x 1400 B of SFQ lead at the floor rate. */ + ftag = 128 * 1400; + + wait = mb_ecn_snd(ctx, 1400, 0, &ftag); + want = (uint64_t) 128 * 1400 * BILLION / 8192; + + if ((uint64_t) wait < want - want / 100 || + (uint64_t) wait > want + want / 100) { + printf("Wait wrapped: exp ~%" PRIu64 ", got %" PRIu64 ".\n", + want, (uint64_t) wait); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A fully paced-backlogged flow reads backlogged after a window. */ +static int test_mb_ecn_backlogged_paced(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->backlogged = false; /* prove a window close re-earns it */ + + drive_backlogged(ctx, &ftag, MS, 4 * CA_SND_WIN, LEN); + + if (!ctx->backlogged) { + printf("paced-backlogged flow read source-limited.\n"); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* + * A source-limited flow is capped to the next quarter-log2 headroom + * above the offered estimate, and NOT re-floored to a high capacity + * rate_min. + */ +static int test_mb_ecn_source_limited_ceiling(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + uint64_t t = 10 * MS; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->tx_cav = true; + ctx->started = true; + ctx->backlogged = false; + ctx->rate = (uint64_t) 100 << 20; + ctx->inv_rate = mb_ecn_rate_inv(ctx->rate); + ctx->rate_min = (uint64_t) 50 << 20; + ctx->snd_rate = (uint64_t) 1 << 20; + ctx->snd_r0 = ctx->rate; + ctx->snd_win = t; + ctx->last_ts = t; + ctx->last_ctrl = t; + + mb_ecn_snd(ctx, LEN, t + 2 * MS, &ftag); + + if (ctx->rate != ((uint64_t) 2 << 20)) { + printf("ceiling: exp %" PRIu64 ", got %" PRIu64 ".\n", + (uint64_t) 2 << 20, ctx->rate); + goto fail_ctx; + } + + if (!ctx->src_limited) { + printf("ceiling bound but src_limited not set.\n"); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* One quiet window must not collapse the max-filter; it decays ~1/16. */ +static int test_mb_ecn_max_filter(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + uint64_t hi = (uint64_t) 10 << 20; + uint64_t t = 10 * MS; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->started = true; + ctx->snd_rate = hi; + ctx->snd_r0 = hi; + ctx->snd_byt = 0; + ctx->snd_win = t; + ctx->last_ts = t; + ctx->last_ctrl = t; + + /* Close one window with almost no bytes offered. */ + mb_ecn_snd(ctx, LEN, t + CA_SND_WIN + 1, &ftag); + + if (ctx->snd_rate >= hi || ctx->snd_rate < hi - hi / 8) { + printf("max-filter: exp ~15/16 of %" PRIu64 ", got %" + PRIu64 " after one quiet window.\n", + hi, ctx->snd_rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* A >CA_DT_CAP gap clears backlogged without touching the estimate. */ +static int test_mb_ecn_idle_clears_backlogged(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + uint64_t snd_rate = (uint64_t) 5 << 20; + uint64_t t = 10 * MS; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + ctx->started = true; + ctx->backlogged = true; + ctx->snd_rate = snd_rate; + ctx->snd_win = t; + ctx->last_ts = t; + ctx->last_ctrl = t; + + /* 55 ms gap: past CA_DT_CAP, under CA_SND_WIN (no window close). */ + mb_ecn_snd(ctx, LEN, t + 55 * MS, &ftag); + + if (ctx->backlogged) { + printf("idle gap did not clear backlogged.\n"); + goto fail_ctx; + } + + if (ctx->snd_rate != snd_rate) { + printf("idle step altered snd_rate %" PRIu64 ".\n", + ctx->snd_rate); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* The first send is never misread as idle, whatever the wall clock. */ +static int test_mb_ecn_first_send_warmup(void) +{ + struct mb_ecn_ctx * ctx; + uint64_t ftag = 0; + + TEST_START(); + + ctx = mk_ctx(); + if (ctx == NULL) { + printf("Failed to create context.\n"); + goto fail; + } + + /* started == false; a large first timestamp must not look idle. */ + mb_ecn_snd(ctx, LEN, 500 * MS, &ftag); + + if (!ctx->started) { + printf("first send did not set the warm-up sentinel.\n"); + goto fail_ctx; + } + + if (!ctx->backlogged) { + printf("first send misclassified as idle.\n"); + goto fail_ctx; + } + + mb_ecn_ctx_destroy(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + mb_ecn_ctx_destroy(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int mb_ecn_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_mb_ecn_ctx_create_destroy(); + ret |= test_mb_ecn_calc_ecn(); + ret |= test_mb_ecn_rcv_onset_immediate(); + ret |= test_mb_ecn_rcv_window_mean(); + ret |= test_mb_ecn_rcv_rate_independent(); + ret |= test_mb_ecn_rcv_size_fair(); + ret |= test_mb_ecn_rcv_release_exact_zero(); + ret |= test_mb_ecn_rcv_gap_restart(); + ret |= test_mb_ecn_rcv_accum_bounds(); + ret |= test_mb_ecn_rcv_window_holds_target(); + ret |= test_mb_ecn_rcv_window_clip_bounds(); + ret |= test_mb_ecn_rcv_slow_window(); + ret |= test_mb_ecn_rcv_no_overflow_highrate(); + ret |= test_mb_ecn_ece_ttl_covers_cadence(); + ret |= test_mb_ecn_slow_start(); + ret |= test_mb_ecn_dt_scaling_invariant(); + ret |= test_mb_ecn_probe_scale_invariant(); + ret |= test_mb_ecn_multiplicative_decrease(); + ret |= test_mb_ecn_fixed_point(); + ret |= test_mb_ecn_lead_cut(); + ret |= test_mb_ecn_slow_start_local_brake(); + ret |= test_mb_ecn_slow_start_clean_ramp(); + ret |= test_mb_ecn_starved_decrease_escape(); + ret |= test_mb_ecn_starved_local_fallback(); + ret |= test_mb_ecn_decrease_dt_invariant(); + ret |= test_mb_ecn_idle_resume_bounded(); + ret |= test_mb_ecn_rate_floor(); + ret |= test_mb_ecn_ece_staleness(); + ret |= test_mb_ecn_ece_ttl_tracks_rate(); + ret |= test_mb_ecn_sfq_pace(); + ret |= test_mb_ecn_probe_time_constant(); + ret |= test_mb_ecn_rcv_cap_window_min(); + ret |= test_mb_ecn_rcv_cap_onset_fresh(); + ret |= test_mb_ecn_ece_cap_derives_rates(); + ret |= test_mb_ecn_ece_cap_zero_keeps_rates(); + ret |= test_mb_ecn_ece_cap_clamps(); + ret |= test_mb_ecn_cap_ttl_reverts(); + ret |= test_mb_ecn_ctrl_per_ctx_ai(); + ret |= test_mb_ecn_ctrl_per_ctx_floor(); + ret |= test_mb_ecn_snd_slow_rate_paced(); + ret |= test_mb_ecn_snd_idle_burst_bound(); + ret |= test_mb_ecn_snd_wait_no_overflow(); + ret |= test_mb_ecn_backlogged_paced(); + ret |= test_mb_ecn_source_limited_ceiling(); + ret |= test_mb_ecn_max_filter(); + ret |= test_mb_ecn_idle_clears_backlogged(); + ret |= test_mb_ecn_first_send_warmup(); + + return ret; +} diff --git a/src/ipcpd/unicast/cap.c b/src/ipcpd/unicast/cap.c new file mode 100644 index 00000000..0d823dc6 --- /dev/null +++ b/src/ipcpd/unicast/cap.c @@ -0,0 +1,291 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Link capacity estimation + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#define _DEFAULT_SOURCE +#else +#define _POSIX_C_SOURCE 200809L +#endif + +#include "config.h" + +#include <ouroboros/atomics.h> +#include <ouroboros/time.h> + +#include "cap.h" + +#include <string.h> + +/* + * Link-capacity estimation by watching the egress queue drain. + * + * A saturated link drains its queue at exactly its capacity, so we + * estimate capacity by measuring the drain rate of the ring buffer + * toward an n-1 flow (the flow to the layer below) while that ring + * is backlogged. + * + * Sampling is lock-free and off the fast path: the ring depth is + * read only at enqueue time, concurrently by many sender threads. + * Each enqueue bumps relaxed counters (packets, bytes, empty-ring + * hits). At most once per CAP_T_MIN, one thread wins a try-lock and + * closes a measurement window. + * + * Over a window, packet conservation gives the slots that drained: + * drained = queue at start (q0) + enqueued - queue now (q1) + * A window stays open until CAP_N_MIN slots have drained, so its + * length self-scales with the link rate (~1 ms at 1 Gbit, ~19 ms at + * 10 Mbit). CAP_T_MAX discards a window that spanned a traffic gap. + * + * Only a backlogged link measures its own capacity, so a window + * whose ring ran mostly idle is discarded (a few empty samples, as + * from a token-bucket shaper, are tolerated). The drain rate feeds a + * max filter that jumps up at once but decays slowly, converging on + * the capacity from below. A window that touched an empty ring at + * either edge may have drained into downstream buffers faster than + * the wire, so it may only lower the estimate, never raise it. + * + * The estimate is published as a quarter-log2 code: capacity is only + * ever needed to order-of-magnitude accuracy. + */ + +#define CAP_T_MIN (BILLION / 1000) /* min fold spacing ~1 ms */ +#define CAP_T_MAX (1ULL << 27) /* stale window cap ~134 ms */ +#define CAP_N_MIN 16 /* drained slots to close */ +#define CAP_DEC_SHFT 4 /* max-filter decay 1/16 */ +#define CAP_IDL_SHFT 3 /* idle tolerance 1/8 */ + +/* Try-lock on the busy flag: test-and-set acquire, store release. */ +#define CAP_TRY(p) (__atomic_exchange_n(p, 1, __ATOMIC_ACQUIRE) == 0) +#define CAP_REL(p) (__atomic_store_n(p, 0, __ATOMIC_RELEASE)) + +struct cap_est { + uint64_t c_pkt; /* total packets enqueued (relaxed) */ + uint64_t c_byt; /* total bytes enqueued (relaxed) */ + uint64_t c_idl; /* times ring seen empty (relaxed) */ + + uint64_t t_gate; /* last fold timestamp (ns) */ + uint8_t busy; /* fold in progress (try-lock) */ + + uint64_t t0; /* window start (ns), 0 = no window */ + uint64_t q0; /* ring occupancy at window start */ + uint64_t pkt0; /* c_pkt snapshot at window start */ + uint64_t byt0; /* c_byt snapshot at window start */ + uint64_t idl0; /* c_idl snapshot at window start */ + uint64_t rate; /* filtered drain rate (bytes/s) */ + + uint8_t cap; /* published capacity code (0=none) */ +}; + +struct { + struct cap_est est[PROC_MAX_FLOWS]; +} cap; + +int cap_init(void) +{ + memset(&cap, 0, sizeof(cap)); + + return 0; +} + +void cap_fini(void) +{ +} + +void cap_reset(int fd) +{ + /* A racing update seeds one bogus window; the filter absorbs. */ + memset(&cap.est[fd], 0, sizeof(cap.est[fd])); +} + +/* + * Rate <-> 8-bit code (cap_enc / cap_dec). The code is a tiny float: + * the high 6 bits are a band e = floor(log2 rate), the low 2 bits a + * quarter k that splits each band [2^e, 2^(e+1)) into four, so + * code = 4 * e + k. Each step is ~19% in rate; that coarseness is + * deliberate, capacity only needs order-of-magnitude accuracy. + * + * The quarter cut points are 256 * 2^(k/4) rounded to an integer: + * {256, 304, 362, 431} over the normalized range [256, 512). + */ +uint8_t cap_enc(uint64_t rate) +{ + static const uint16_t thr[3] = {304, 362, 431}; + uint64_t r = rate; /* copy halved to find band */ + unsigned e = 0; /* band: floor log2 rate */ + unsigned k = 0; /* quarter within band 0..3 */ + unsigned c; /* code = 4 * band + quarter */ + uint16_t top; /* rate scaled to [256, 512) */ + + if (rate == 0) + return 0; + + while (r > 1) { + r >>= 1; + e++; + } + + /* Top 9 bits: rate normalized to [256, 512). */ + top = e >= 8 ? (uint16_t) (rate >> (e - 8)) + : (uint16_t) (rate << (8 - e)); + + while (k < 3 && top >= thr[k]) + k++; + + c = 4 * e + k; + if (c == 0) + c = 1; /* 0 means unknown */ + + return (uint8_t) c; +} + +uint64_t cap_dec(uint8_t c) +{ + static const uint16_t m[4] = {256, 304, 362, 431}; + unsigned e = c >> 2; /* band = c >> 2 */ + unsigned k = c & 3; /* quarter = c & 3 */ + + if (c == 0) + return 0; + + if (e >= 8) + return (uint64_t) m[k] << (e - 8); + + return ((uint64_t) m[k] << e) >> 8; +} + +uint8_t cap_min(uint8_t a, + uint8_t b) +{ + if (a == 0) + return b; + + if (b == 0) + return a; + + return a < b ? a : b; +} + +void cap_stamp(uint8_t * pci, + uint8_t own) +{ + if (own == 0) + return; + + if (*pci == 0 || own < *pci) + *pci = own; +} + +/* Fold flag held; q1 is the caller's pre-write ring sample. */ +static void cap_fold(struct cap_est * e, + uint64_t q1, + uint64_t now) +{ + uint64_t pkt; /* current c_pkt snapshot */ + uint64_t byt; /* current c_byt snapshot */ + uint64_t idl; /* current c_idl snapshot */ + uint64_t dt; /* window duration (ns) */ + uint64_t enq; /* packets enqueued in window */ + uint64_t avg; /* mean packet size (bytes) */ + uint64_t r; /* window drain rate (bytes/s) */ + int64_t drained; /* slots drained over window */ + + pkt = LOAD_RELAXED(&e->c_pkt); + byt = LOAD_RELAXED(&e->c_byt); + idl = LOAD_RELAXED(&e->c_idl); + + dt = now - e->t0; + enq = pkt - e->pkt0; + + drained = (int64_t) (e->q0 + enq - q1); + + if (e->t0 == 0 || dt > CAP_T_MAX || enq == 0) + goto reopen; + + if (drained < (int64_t) CAP_N_MIN) + return; /* extend the window until enough drains */ + + if ((idl - e->idl0) << CAP_IDL_SHFT > enq) + goto reopen; /* mostly idle ring: not saturated */ + + avg = (byt - e->byt0) / enq; + r = (uint64_t) drained * avg * MILLION / (dt / 1000); + + if (r >= e->rate) { + /* Empty-edged windows drain into buffers below. */ + if (e->q0 > 0 && q1 > 0) + e->rate = r; + } else { + e->rate -= (e->rate - r) >> CAP_DEC_SHFT; + } + + STORE_RELAXED(&e->cap, cap_enc(e->rate)); + reopen: + e->t0 = now; + e->q0 = q1; + e->pkt0 = pkt; + e->byt0 = byt; + e->idl0 = idl; +} + +/* Internal, timestamped entry point; tests drive this directly. */ +static void cap_update_at(int fd, + size_t qlen, + size_t len, + uint64_t now) +{ + struct cap_est * e = &cap.est[fd]; /* this flow's estimator */ + + FETCH_ADD_RELAXED(&e->c_pkt, 1); + FETCH_ADD_RELAXED(&e->c_byt, len); + + if (qlen == 0) + FETCH_ADD_RELAXED(&e->c_idl, 1); + + if (now - LOAD_RELAXED(&e->t_gate) < CAP_T_MIN) + return; + + if (!CAP_TRY(&e->busy)) + return; + + if (now - e->t_gate >= CAP_T_MIN) { + cap_fold(e, qlen, now); + STORE_RELAXED(&e->t_gate, now); + } + + CAP_REL(&e->busy); +} + +void cap_update(int fd, + size_t qlen, + size_t len) +{ + struct timespec now; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + cap_update_at(fd, qlen, len, TS_TO_UINT64(now)); +} + +uint8_t cap_get(int fd) +{ + return LOAD_RELAXED(&cap.est[fd].cap); +} diff --git a/src/ipcpd/unicast/cap.h b/src/ipcpd/unicast/cap.h new file mode 100644 index 00000000..df8c2ec1 --- /dev/null +++ b/src/ipcpd/unicast/cap.h @@ -0,0 +1,54 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Link capacity estimation + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#ifndef OUROBOROS_IPCPD_UNICAST_CAP_H +#define OUROBOROS_IPCPD_UNICAST_CAP_H + +#include <stddef.h> +#include <stdint.h> + +int cap_init(void); + +void cap_fini(void); + +/* Account an egress packet; qlen is sampled before the write. */ +void cap_update(int fd, + size_t qlen, + size_t len); + +uint8_t cap_get(int fd); + +void cap_reset(int fd); + +/* Quarter-log2 capacity code: ~2^(c / 4) bytes/s, 0 = unknown. */ +uint8_t cap_enc(uint64_t rate); + +uint64_t cap_dec(uint8_t c); + +uint8_t cap_min(uint8_t a, + uint8_t b); + +/* MIN-combine the own link code into the PCI byte. */ +void cap_stamp(uint8_t * pci, + uint8_t own); + +#endif /* OUROBOROS_IPCPD_UNICAST_CAP_H */ diff --git a/src/ipcpd/unicast/connmgr.c b/src/ipcpd/unicast/connmgr.c index 07568fb5..f60f6fca 100644 --- a/src/ipcpd/unicast/connmgr.c +++ b/src/ipcpd/unicast/connmgr.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Handles connections between components * diff --git a/src/ipcpd/unicast/dir.c b/src/ipcpd/unicast/dir.c index 2b305626..a31a562f 100644 --- a/src/ipcpd/unicast/dir.c +++ b/src/ipcpd/unicast/dir.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Directory Management * diff --git a/src/ipcpd/unicast/dir.h b/src/ipcpd/unicast/dir.h index dbfde19f..2be7b10f 100644 --- a/src/ipcpd/unicast/dir.h +++ b/src/ipcpd/unicast/dir.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Directory * diff --git a/src/ipcpd/unicast/dir/dht.c b/src/ipcpd/unicast/dir/dht.c index 6b06def9..c2cd33aa 100644 --- a/src/ipcpd/unicast/dir/dht.c +++ b/src/ipcpd/unicast/dir/dht.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Distributed Hash Table based on Kademlia * @@ -148,15 +148,8 @@ struct dht_entry { uint8_t * key; - struct { - struct list_head list; - size_t len; - } vals; /* We don't own these, only replicate */ - - struct { - struct list_head list; - size_t len; - } lvals; /* We own these, must be republished */ + struct llist vals; /* We don't own these, only replicate */ + struct llist lvals; /* We own these, must be republished */ }; struct contact { @@ -183,38 +176,24 @@ struct peer_entry { struct dht_req { struct list_head next; - uint8_t * key; - time_t t_exp; + uint8_t * key; + time_t t_exp; - struct { - struct list_head list; - size_t len; - } peers; - - struct { - struct list_head list; - size_t len; - } cache; + struct llist peers; + struct llist cache; }; struct bucket { - struct { - struct list_head list; - size_t len; - } contacts; - - struct { - struct list_head list; - size_t len; - } alts; + struct llist contacts; + struct llist alts; - time_t t_refr; + time_t t_refr; - size_t depth; - uint8_t mask; + size_t depth; + uint8_t mask; - struct bucket * parent; - struct bucket * children[1L << DHT_BETA]; + struct bucket * parent; + struct bucket * children[1L << DHT_BETA]; }; struct cmd { @@ -236,8 +215,8 @@ struct { struct { /* Kademlia parameters */ uint32_t alpha; /* Number of concurrent requests */ size_t k; /* Number of replicas to store */ - time_t t_expire; /* Expiry time for values (s) */ - time_t t_refresh; /* Refresh time for contacts (s) */ + time_t t_exp; /* Expiry time for values (s) */ + time_t t_refr; /* Refresh time for contacts (s) */ time_t t_repl; /* Replication time for values (s) */ }; @@ -261,8 +240,7 @@ struct { } contacts; struct { - struct list_head list; - size_t len; + struct llist ll; size_t vals; size_t lvals; } kv; @@ -271,10 +249,9 @@ struct { } db; struct { - struct list_head list; - size_t len; - pthread_cond_t cond; - pthread_mutex_t mtx; + struct llist ll; + pthread_cond_t cond; + pthread_mutex_t mtx; } reqs; struct { @@ -321,7 +298,7 @@ static int dht_rib_statfile(char * buf, pthread_rwlock_rdlock(&dht.db.lock); - keys = dht.db.kv.len; + keys = dht.db.kv.ll.len; lvals = dht.db.kv.lvals; vals = dht.db.kv.vals; @@ -335,7 +312,7 @@ static int dht_rib_statfile(char * buf, tmstr, ADDR_VAL32(&dht.addr), dht.alpha, dht.k, - dht.t_expire, dht.t_refresh, dht.t_repl, + dht.t_exp, dht.t_refr, dht.t_repl, keys, vals, lvals); return strlen(buf); @@ -350,14 +327,14 @@ static size_t dht_db_file_len(void) pthread_rwlock_rdlock(&dht.db.lock); - if (dht.db.kv.len == 0) { + if (llist_is_empty(&dht.db.kv.ll)) { pthread_rwlock_unlock(&dht.db.lock); sz += 14; /* No entries */ return sz; } sz += 39 * 3 + 1; /* tally + extra newline */ - sz += dht.db.kv.len * (25 + 19 + 23 + 1); + sz += dht.db.kv.ll.len * (25 + 19 + 23 + 1); vals = dht.db.kv.vals + dht.db.kv.lvals; @@ -382,7 +359,7 @@ static int dht_rib_dbfile(char * buf, pthread_rwlock_rdlock(&dht.db.lock); - if (dht.db.kv.len == 0) { + if (llist_is_empty(&dht.db.kv.ll)) { i += snprintf(buf, len, " No entries.\n"); pthread_rwlock_unlock(&dht.db.lock); return i; @@ -393,9 +370,9 @@ static int dht_rib_dbfile(char * buf, "Number of keys: %10zu\n" "Number of local values: %10zu\n" "Number of non-local values: %10zu\n\n", - dht.db.kv.len, dht.db.kv.vals, dht.db.kv.lvals); + dht.db.kv.ll.len, dht.db.kv.vals, dht.db.kv.lvals); - list_for_each(p, &dht.db.kv.list) { + llist_for_each(p, &dht.db.kv.ll) { struct dht_entry * e = list_entry(p, struct dht_entry, next); struct list_head * h; @@ -403,7 +380,7 @@ static int dht_rib_dbfile(char * buf, KEY_VAL(e->key)); i += snprintf(buf + i, len - i, " Local entries:\n"); - list_for_each(h, &e->vals.list) { + llist_for_each(h, &e->vals) { struct val_entry * v; v = list_entry(h, struct val_entry, next); @@ -416,7 +393,7 @@ static int dht_rib_dbfile(char * buf, i += snprintf(buf + i, len - i, " " VAL_FMT - ", t_replicated=%.*s, t_expire=%.*s\n", + ", t_replicated=%.*s, t_exp=%.*s\n", VAL_VAL(v->val), RIB_TM_STRLEN, tmstr, RIB_TM_STRLEN, exstr); @@ -426,7 +403,7 @@ static int dht_rib_dbfile(char * buf, i += snprintf(buf + i, len - i, " Non-local entries:\n"); - list_for_each(h, &e->lvals.list) { + llist_for_each(h, &e->lvals) { struct val_entry * v; v= list_entry(h, struct val_entry, next); @@ -439,7 +416,7 @@ static int dht_rib_dbfile(char * buf, i += snprintf(buf + i, len - i, " " VAL_FMT - ", t_replicated=%.*s, t_expire=%.*s\n", + ", t_replicated=%.*s, t_exp=%.*s\n", VAL_VAL(v->val), RIB_TM_STRLEN, tmstr, RIB_TM_STRLEN, exstr); @@ -694,11 +671,8 @@ static struct dht_entry * dht_entry_create(const uint8_t * key) goto fail_entry; list_head_init(&e->next); - list_head_init(&e->vals.list); - list_head_init(&e->lvals.list); - - e->vals.len = 0; - e->lvals.len = 0; + llist_init(&e->vals); + llist_init(&e->lvals); e->key = dht_dup_key(key); if (e->key == NULL) @@ -718,25 +692,23 @@ static void dht_entry_destroy(struct dht_entry * e) assert(e != NULL); - list_for_each_safe(p, h, &e->vals.list) { + llist_for_each_safe(p, h, &e->vals) { struct val_entry * v = list_entry(p, struct val_entry, next); - list_del(&v->next); + llist_del(&v->next, &e->vals); val_entry_destroy(v); - --e->vals.len; --dht.db.kv.vals; } - list_for_each_safe(p, h, &e->lvals.list) { + llist_for_each_safe(p, h, &e->lvals) { struct val_entry * v = list_entry(p, struct val_entry, next); - list_del(&v->next); + llist_del(&v->next, &e->lvals); val_entry_destroy(v); - --e->lvals.len; --dht.db.kv.lvals; } free(e->key); - assert(e->vals.len == 0 && e->lvals.len == 0); + assert(llist_is_empty(&e->vals) && llist_is_empty(&e->lvals)); free(e); } @@ -750,7 +722,7 @@ static struct val_entry * dht_entry_get_lval(const struct dht_entry * e, assert(val.data != NULL); assert(val.len > 0); - list_for_each(p, &e->lvals.list) { + llist_for_each(p, &e->lvals) { struct val_entry * v = list_entry(p, struct val_entry, next); if (bufcmp(&v->val, &val) == 0) return v; @@ -768,7 +740,7 @@ static struct val_entry * dht_entry_get_val(const struct dht_entry * e, assert(val.data != NULL); assert(val.len > 0); - list_for_each(p, &e->vals.list) { + llist_for_each(p, &e->vals) { struct val_entry * v = list_entry(p, struct val_entry, next); if (bufcmp(&v->val, &val) == 0) return v; @@ -805,8 +777,7 @@ static int dht_entry_update_val(struct dht_entry * e, if (v == NULL) return -ENOMEM; - list_add_tail(&v->next, &e->vals.list); - ++e->vals.len; + llist_add_tail(&v->next, &e->vals); ++dht.db.kv.vals; return 0; @@ -833,12 +804,11 @@ static int dht_entry_update_lval(struct dht_entry * e, v = dht_entry_get_lval(e, val); if (v == NULL) { log_dbg(KV_FMT " Adding lval.", KV_VAL(e->key, val)); - v = val_entry_create(val, now.tv_sec + dht.t_expire); + v = val_entry_create(val, now.tv_sec + dht.t_exp); if (v == NULL) return -ENOMEM; - list_add_tail(&v->next, &e->lvals.list); - ++e->lvals.len; + llist_add_tail(&v->next, &e->lvals); ++dht.db.kv.lvals; return 0; @@ -862,9 +832,8 @@ static int dht_entry_remove_lval(struct dht_entry * e, log_dbg(KV_FMT " Removing lval.", KV_VAL(e->key, val)); - list_del(&v->next); + llist_del(&v->next, &e->lvals); val_entry_destroy(v); - --e->lvals.len; --dht.db.kv.lvals; return 0; @@ -881,15 +850,14 @@ static void dht_entry_remove_expired_vals(struct dht_entry * e) clock_gettime(CLOCK_REALTIME_COARSE, &now); - list_for_each_safe(p, h, &e->vals.list) { + llist_for_each_safe(p, h, &e->vals) { struct val_entry * v = list_entry(p, struct val_entry, next); if (!IS_EXPIRED(v, &now)) continue; log_dbg(KV_FMT " Value expired." , KV_VAL(e->key, v->val)); - list_del(&v->next); + llist_del(&v->next, &e->vals); val_entry_destroy(v); - --e->vals.len; --dht.db.kv.vals; } } @@ -900,7 +868,7 @@ static struct dht_entry * __dht_kv_find_entry(const uint8_t * key) assert(key != NULL); - list_for_each(p, &dht.db.kv.list) { + llist_for_each(p, &dht.db.kv.ll) { struct dht_entry * e = list_entry(p, struct dht_entry, next); if (!memcmp(key, e->key, dht.id.len)) return e; @@ -919,16 +887,15 @@ static void dht_kv_remove_expired_entries(void) pthread_rwlock_wrlock(&dht.db.lock); - list_for_each_safe(p, h, &dht.db.kv.list) { + llist_for_each_safe(p, h, &dht.db.kv.ll) { struct dht_entry * e = list_entry(p, struct dht_entry, next); dht_entry_remove_expired_vals(e); if (e->lvals.len > 0 || e->vals.len > 0) continue; log_dbg(KEY_FMT " Entry removed. ", KEY_VAL(e->key)); - list_del(&e->next); + llist_del(&e->next, &dht.db.kv.ll); dht_entry_destroy(e); - --dht.db.kv.len; } pthread_rwlock_unlock(&dht.db.lock); @@ -987,15 +954,13 @@ static struct dht_req * dht_req_create(const uint8_t * key) req->t_exp = now.tv_sec + DHT_T_RESP; - list_head_init(&req->peers.list); - req->peers.len = 0; + llist_init(&req->peers); req->key = dht_dup_key(key); if (req->key == NULL) goto fail_dup_key; - list_head_init(&req->cache.list); - req->cache.len = 0; + llist_init(&req->cache); return req; @@ -1013,34 +978,32 @@ static void dht_req_destroy(struct dht_req * req) assert(req); assert(req->key); - list_for_each_safe(p, h, &req->peers.list) { + llist_for_each_safe(p, h, &req->peers) { struct peer_entry * e = list_entry(p, struct peer_entry, next); - list_del(&e->next); + llist_del(&e->next, &req->peers); free(e->id); free(e); - --req->peers.len; } - list_for_each_safe(p, h, &req->cache.list) { + llist_for_each_safe(p, h, &req->cache) { struct val_entry * e = list_entry(p, struct val_entry, next); - list_del(&e->next); + llist_del(&e->next, &req->cache); val_entry_destroy(e); - --req->cache.len; } free(req->key); - assert(req->peers.len == 0); + assert(llist_is_empty(&req->peers)); free(req); } -static struct peer_entry * dht_req_get_peer(struct dht_req * req, - struct peer_entry * e) +static struct peer_entry * dht_req_get_peer(struct dht_req * req, + const struct peer_entry * e) { struct list_head * p; - list_for_each(p, &req->peers.list) { + llist_for_each(p, &req->peers) { struct peer_entry * x = list_entry(p, struct peer_entry, next); if (x->addr == e->addr) return x; @@ -1050,8 +1013,8 @@ static struct peer_entry * dht_req_get_peer(struct dht_req * req, } #define IS_MAGIC(peer) ((peer)->cookie == dht.magic) -void dht_req_add_peer(struct dht_req * req, - struct peer_entry * e) +static int dht_req_add_peer(struct dht_req * req, + const struct peer_entry * e) { struct peer_entry * x; /* existing */ struct list_head * p; /* iterator */ @@ -1063,16 +1026,17 @@ void dht_req_add_peer(struct dht_req * req, /* * Dedupe messages to the same peer, unless - * 1) The previous request was FIND_NODE and now it's FIND_VALUE - * 2) We urgently need contacts from emergency peer (magic cookie) + * 1) The previous was FIND_NODE and now it's FIND_VALUE + * 2) We urgently need contacts (magic cookie) */ x = dht_req_get_peer(req, e); if (x != NULL && x->code >= e->code && !IS_MAGIC(e)) - goto skip; + return -1; /* Find how this contact ranks in distance to the key */ - list_for_each(p, &req->peers.list) { - struct peer_entry * y = list_entry(p, struct peer_entry, next); + llist_for_each(p, &req->peers) { + struct peer_entry * y; + y = list_entry(p, struct peer_entry, next); if (IS_CLOSER(y->id, e->id)) { pos++; continue; @@ -1080,36 +1044,32 @@ void dht_req_add_peer(struct dht_req * req, break; } - /* Add a new peer to this request if we need to */ - if (pos < dht.alpha || !IS_MAGIC(e)) { - x = malloc(sizeof(*x)); - if (x == NULL) { - log_err("Failed to malloc peer entry."); - goto skip; - } + if (pos >= dht.alpha && IS_MAGIC(e)) + return -1; - x->cookie = e->cookie; - x->addr = e->addr; - x->code = e->code; - x->t_sent = e->t_sent; - x->id = dht_dup_key(e->id); - if (x->id == NULL) { - log_err("Failed to dup peer ID."); - free(x); - goto skip; - } + x = malloc(sizeof(*x)); + if (x == NULL) { + log_err("Failed to malloc peer entry."); + return -1; + } - if (IS_MAGIC(e)) - list_add(&x->next, p); - else - list_add_tail(&x->next, p); - ++req->peers.len; - return; + x->cookie = e->cookie; + x->addr = e->addr; + x->code = e->code; + x->t_sent = e->t_sent; + x->id = dht_dup_key(e->id); + if (x->id == NULL) { + log_err("Failed to dup peer ID."); + free(x); + return -1; } - skip: - list_del(&e->next); - free(e->id); - free(e); + + if (IS_MAGIC(e)) + llist_add_at(&x->next, p, &req->peers); + else + llist_add_tail_at(&x->next, p, &req->peers); + + return 0; } static size_t dht_req_add_peers(struct dht_req * req, @@ -1123,8 +1083,13 @@ static size_t dht_req_add_peers(struct dht_req * req, assert(pl != NULL); list_for_each_safe(p, h, pl) { - struct peer_entry * e = list_entry(p, struct peer_entry, next); - dht_req_add_peer(req, e); + struct peer_entry * e; + e = list_entry(p, struct peer_entry, next); + if (dht_req_add_peer(req, e) < 0) { + list_del(&e->next); + free(e->id); + free(e); + } } return n; @@ -1137,7 +1102,7 @@ static bool dht_req_has_peer(struct dht_req * req, assert(req != NULL); - list_for_each(p, &req->peers.list) { + llist_for_each(p, &req->peers) { struct peer_entry * e = list_entry(p, struct peer_entry, next); if (e->cookie == cookie) return true; @@ -1209,7 +1174,7 @@ static struct dht_req * __dht_kv_req_get_req(const uint8_t * key) { struct list_head * p; - list_for_each(p, &dht.reqs.list) { + llist_for_each(p, &dht.reqs.ll) { struct dht_req * r = list_entry(p, struct dht_req, next); if (memcmp(r->key, key, dht.id.len) == 0) return r; @@ -1228,7 +1193,7 @@ static struct dht_req * __dht_kv_get_req_cache(const uint8_t * key) if (req == NULL) return NULL; - if (req->cache.len == 0) + if (llist_is_empty(&req->cache)) return NULL; return req; @@ -1244,8 +1209,7 @@ static void __dht_kv_req_remove(const uint8_t * key) if (req == NULL) return; - list_del(&req->next); - --dht.reqs.len; + llist_del(&req->next, &dht.reqs.ll); dht_req_destroy(req); } @@ -1301,9 +1265,9 @@ static int dht_kv_update_req(const uint8_t * key, req = __dht_kv_req_get_req(key); if (req == NULL) { - if (dht.reqs.len == DHT_MAX_REQS) { + if (dht.reqs.ll.len == DHT_MAX_REQS) { log_err(KEY_FMT " Max reqs reached (%zu).", - KEY_VAL(key), dht.reqs.len); + KEY_VAL(key), dht.reqs.ll.len); peer_list_destroy(pl); goto fail_req; } @@ -1312,8 +1276,7 @@ static int dht_kv_update_req(const uint8_t * key, log_err(KEY_FMT "Failed to create req.", KEY_VAL(key)); goto fail_req; } - list_add_tail(&req->next, &dht.reqs.list); - ++dht.reqs.len; + llist_add_tail(&req->next, &dht.reqs.ll); } if (req->cache.len > 0) /* Already have values */ @@ -1322,9 +1285,9 @@ static int dht_kv_update_req(const uint8_t * key, dht_req_add_peers(req, pl); req->t_exp = now.tv_sec + DHT_T_RESP; - if (dht.reqs.len > DHT_WARN_REQS) { + if (dht.reqs.ll.len > DHT_WARN_REQS) { log_warn("Number of outstanding requests (%zu) exceeds %u.", - dht.reqs.len, DHT_WARN_REQS); + dht.reqs.ll.len, DHT_WARN_REQS); } pthread_mutex_unlock(&dht.reqs.mtx); @@ -1368,8 +1331,7 @@ static int dht_kv_respond_req(uint8_t * key, continue; } - list_add_tail(&e->next, &req->cache.list); - ++req->cache.len; + llist_add_tail(&e->next, &req->cache); } pthread_cond_broadcast(&dht.reqs.cond); @@ -1434,7 +1396,7 @@ static ssize_t dht_kv_wait_req(const uint8_t * key, memset(*vals, 0, max * sizeof(**vals)); - list_for_each(p, &req->cache.list) { + llist_for_each(p, &req->cache) { struct val_entry * v; if (i == max) break; /* We have enough values */ @@ -1535,10 +1497,10 @@ static ssize_t dht_kv_contact_list(const uint8_t * key, goto fail_bucket; } - b->t_refr = t.tv_sec + dht.t_refresh; + b->t_refr = t.tv_sec + dht.t_refr; if (b->contacts.len == dht.k || b->parent == NULL) { - list_for_each(p, &b->contacts.list) { + llist_for_each(p, &b->contacts) { struct contact * c; struct contact * d; c = list_entry(p, struct contact, next); @@ -1554,7 +1516,7 @@ static ssize_t dht_kv_contact_list(const uint8_t * key, } else { struct bucket * d = b->parent; for (i = 0; i < (1L << DHT_BETA) && len < dht.k; ++i) { - list_for_each(p, &d->children[i]->contacts.list) { + llist_for_each(p, &d->children[i]->contacts) { struct contact * c; struct contact * d; c = list_entry(p, struct contact, next); @@ -1635,6 +1597,7 @@ static ssize_t dht_kv_get_contacts(const uint8_t * key, fail_contact: while (i-- > 0) dht_contact_msg__free_unpacked((*msgs)[i], NULL); + free(*msgs); *msgs = NULL; fail_msgs: @@ -1661,11 +1624,11 @@ static void __dht_kv_bucket_refresh_list(struct bucket * b, __dht_kv_bucket_refresh_list(b->children[i], t, r); } - if (b->contacts.len == 0) + if (llist_is_empty(&b->contacts)) return; - c = list_first_entry(&b->contacts.list, struct contact, next); - if (t > c->t_seen + dht.t_refresh) { + c = llist_first_entry(&b->contacts, struct contact, next); + if (t > c->t_seen + dht.t_refr) { d = contact_create(c->id, c->addr); if (d != NULL) list_add(&d->next, r); @@ -1682,14 +1645,12 @@ static struct bucket * bucket_create(void) if (b == NULL) return NULL; - list_head_init(&b->contacts.list); - b->contacts.len = 0; + llist_init(&b->contacts); - list_head_init(&b->alts.list); - b->alts.len = 0; + llist_init(&b->alts); clock_gettime(CLOCK_REALTIME_COARSE, &t); - b->t_refr = t.tv_sec + dht.t_refresh; + b->t_refr = t.tv_sec + dht.t_refr; for (i = 0; i < (1L << DHT_BETA); ++i) b->children[i] = NULL; @@ -1713,18 +1674,16 @@ static void bucket_destroy(struct bucket * b) if (b->children[i] != NULL) bucket_destroy(b->children[i]); - list_for_each_safe(p, h, &b->contacts.list) { + llist_for_each_safe(p, h, &b->contacts) { struct contact * c = list_entry(p, struct contact, next); - list_del(&c->next); + llist_del(&c->next, &b->contacts); contact_destroy(c); - --b->contacts.len; } - list_for_each_safe(p, h, &b->alts.list) { + llist_for_each_safe(p, h, &b->alts) { struct contact * c = list_entry(p, struct contact, next); - list_del(&c->next); + llist_del(&c->next, &b->alts); contact_destroy(c); - --b->alts.len; } free(b); @@ -1759,13 +1718,11 @@ static int move_contacts(struct bucket * b, assert(b != NULL); assert(c != NULL); - list_for_each_safe(p, h, &b->contacts.list) { + llist_for_each_safe(p, h, &b->contacts) { d = list_entry(p, struct contact, next); if (bucket_has_id(c, d->id)) { - list_del(&d->next); - --b->contacts.len; - list_add_tail(&d->next, &c->contacts.list); - ++c->contacts.len; + llist_del(&d->next, &b->contacts); + llist_add_tail(&d->next, &c->contacts); } } @@ -1779,8 +1736,8 @@ static int split_bucket(struct bucket * b) size_t b_len; assert(b); - assert(b->alts.len == 0); - assert(b->contacts.len != 0); + assert(llist_is_empty(&b->alts)); + assert(!llist_is_empty(&b->contacts)); assert(b->children[0] == NULL); b_len = b->contacts.len; @@ -1807,6 +1764,7 @@ static int split_bucket(struct bucket * b) fail_child: while (i-- > 0) bucket_destroy(b->children[i]); + return -1; } @@ -1836,39 +1794,33 @@ static int dht_kv_update_contacts(const uint8_t * id, goto fail_update; } - list_for_each_safe(p, h, &b->contacts.list) { + llist_for_each_safe(p, h, &b->contacts) { struct contact * d = list_entry(p, struct contact, next); if (d->addr == addr) { - list_del(&d->next); + llist_del(&d->next, &b->contacts); contact_destroy(d); - --b->contacts.len; } } if (b->contacts.len == dht.k) { if (bucket_has_id(b, dht.id.data)) { - list_add_tail(&c->next, &b->contacts.list); - ++b->contacts.len; + llist_add_tail(&c->next, &b->contacts); if (split_bucket(b)) { - list_del(&c->next); + llist_del(&c->next, &b->contacts); contact_destroy(c); - --b->contacts.len; } } else if (b->alts.len == dht.k) { struct contact * d; - d = list_first_entry(&b->alts.list, + d = llist_first_entry(&b->alts, struct contact, next); - list_del(&d->next); + llist_del(&d->next, &b->alts); contact_destroy(d); - list_add_tail(&c->next, &b->alts.list); - ++b->alts.len; + llist_add_tail(&c->next, &b->alts); } else { - list_add_tail(&c->next, &b->alts.list); - ++b->alts.len; + llist_add_tail(&c->next, &b->alts); } } else { - list_add_tail(&c->next, &b->contacts.list); - ++b->contacts.len; + llist_add_tail(&c->next, &b->contacts); } pthread_rwlock_unlock(&dht.db.lock); @@ -2116,7 +2068,7 @@ static ssize_t dht_kv_retrieve(const uint8_t * key, i = 0; - list_for_each(p, &e->vals.list) { + llist_for_each(p, &e->vals) { struct val_entry * v; if (i == n) break; /* We have enough values */ @@ -2129,7 +2081,7 @@ static ssize_t dht_kv_retrieve(const uint8_t * key, memcpy((*vals)[i++].data, v->val.data, v->val.len); } - list_for_each(p, &e->lvals.list) { + llist_for_each(p, &e->lvals) { struct val_entry * v; if (i == n) break; /* We have enough values */ @@ -2266,7 +2218,7 @@ static int dht_send_msg(dht_msg_t * msg, uint64_t addr) { size_t len; - struct shm_du_buff * sdb; + struct ssm_pk_buff * spb; if (msg == NULL) return 0; @@ -2279,21 +2231,21 @@ static int dht_send_msg(dht_msg_t * msg, goto fail_msg; } - if (ipcp_sdb_reserve(&sdb, len)) { - log_warn("%s failed to get sdb.", DHT_CODE(msg)); + if (ipcp_spb_reserve(&spb, len)) { + log_warn("%s failed to get spb.", DHT_CODE(msg)); goto fail_msg; } - dht_msg__pack(msg, shm_du_buff_head(sdb)); + dht_msg__pack(msg, ssm_pk_buff_head(spb)); - if (dt_write_packet(addr, QOS_CUBE_BE, dht.eid, sdb) < 0) { + if (dt_write_packet(addr, QOS_CUBE_BE, dht.eid, spb, NULL) < 0) { log_warn("%s write failed", DHT_CODE(msg)); goto fail_send; } return 0; fail_send: - ipcp_sdb_release(sdb); + ipcp_spb_release(spb); fail_msg: return -1; } @@ -2584,15 +2536,14 @@ static void __add_dht_kv_entry(struct dht_entry * e) assert(e != NULL); - list_for_each(p, &dht.db.kv.list) { + llist_for_each(p, &dht.db.kv.ll) { struct dht_entry * d = list_entry(p, struct dht_entry, next); if (IS_CLOSER(d->key, e->key)) continue; break; } - list_add_tail(&e->next, p); - ++dht.db.kv.len; + llist_add_tail_at(&e->next, p, &dht.db.kv.ll); } /* incoming store message */ @@ -2629,9 +2580,8 @@ static int dht_kv_store(const uint8_t * key, return 0; fail_add: if (new) { - list_del(&e->next); + llist_del(&e->next, &dht.db.kv.ll); dht_entry_destroy(e); - --dht.db.kv.len; } fail: pthread_rwlock_unlock(&dht.db.lock); @@ -2669,14 +2619,13 @@ static int dht_kv_publish(const uint8_t * key, pthread_rwlock_unlock(&dht.db.lock); - dht_kv_store_remote(key, val, now.tv_sec + dht.t_expire); + dht_kv_store_remote(key, val, now.tv_sec + dht.t_exp); return 0; fail_add: if (new) { - list_del(&e->next); + llist_del(&e->next, &dht.db.kv.ll); dht_entry_destroy(e); - --dht.db.kv.len; } fail: pthread_rwlock_unlock(&dht.db.lock); @@ -2858,7 +2807,7 @@ static void do_dht_kv_store(const dht_store_msg_t * store) key = store->key.data; exp = store->exp; - if (dht_kv_store(store->key.data, val, store->exp) < 0) { + if (dht_kv_store(key, val, store->exp) < 0) { log_err(KV_FMT " Failed to store.", KV_VAL(key, val)); return; } @@ -2902,6 +2851,7 @@ static dht_msg_t * do_dht_kv_find_node_req(const dht_find_req_msg_t * req) fail_msg: while (len-- > 0) dht_contact_msg__free_unpacked(contacts[len], NULL); + free(contacts); fail_contacts: return NULL; @@ -3004,8 +2954,9 @@ static dht_msg_t * do_dht_kv_find_value_req(const dht_find_req_msg_t * req) fail_msg: freebufs(vals, n_vals); fail_vals: - while (n_contacts-- > 0) + while (contacts != NULL && n_contacts-- > 0) dht_contact_msg__free_unpacked(contacts[n_contacts], NULL); + free(contacts); fail_contacts: return NULL; @@ -3189,7 +3140,7 @@ static void * dht_handle_packet(void * o) } #ifndef __DHT_TEST__ static void dht_post_packet(void * comp, - struct shm_du_buff * sdb) + struct ssm_pk_buff * spb) { struct cmd * cmd; @@ -3201,17 +3152,17 @@ static void dht_post_packet(void * comp, goto fail_cmd; } - cmd->cbuf.data = malloc(shm_du_buff_len(sdb)); + cmd->cbuf.data = malloc(ssm_pk_buff_len(spb)); if (cmd->cbuf.data == NULL) { log_err("Command buffer malloc failed."); goto fail_buf; } - cmd->cbuf.len = shm_du_buff_len(sdb); + cmd->cbuf.len = ssm_pk_buff_len(spb); - memcpy(cmd->cbuf.data, shm_du_buff_head(sdb), cmd->cbuf.len); + memcpy(cmd->cbuf.data, ssm_pk_buff_head(spb), cmd->cbuf.len); - ipcp_sdb_release(sdb); + ipcp_spb_release(spb); pthread_mutex_lock(&dht.cmds.mtx); @@ -3226,7 +3177,7 @@ static void dht_post_packet(void * comp, fail_buf: free(cmd); fail_cmd: - ipcp_sdb_release(sdb); + ipcp_spb_release(spb); return; } #endif @@ -3365,6 +3316,42 @@ static int emergency_peer(struct list_head * pl) return -ENOMEM; } +static bool __dht_kv_bucket_has_addr(struct bucket * b, + uint64_t addr) +{ + struct list_head * p; + size_t i; + + assert(b != NULL); + + if (*b->children != NULL) + for (i = 0; i < (1L << DHT_BETA); ++i) + if (__dht_kv_bucket_has_addr(b->children[i], addr)) + return true; + + llist_for_each(p, &b->contacts) { + struct contact * c; + c = list_entry(p, struct contact, next); + if (c->addr == addr) + return true; + } + + return false; +} + +static bool dht_kv_knows_peer(void) +{ + bool found; + + pthread_rwlock_rdlock(&dht.db.lock); + + found = __dht_kv_bucket_has_addr(dht.db.contacts.root, dht.peer); + + pthread_rwlock_unlock(&dht.db.lock); + + return found; +} + static int dht_kv_seed_bootstrap_peer(void) { struct list_head pl; @@ -3376,6 +3363,9 @@ static int dht_kv_seed_bootstrap_peer(void) return 0; } + if (dht_kv_knows_peer()) + return 0; + if (emergency_peer(&pl) < 0) { log_err("Could not create emergency peer."); goto fail_peer; @@ -3391,7 +3381,8 @@ static int dht_kv_seed_bootstrap_peer(void) peer_list_destroy(&pl); - return 0; + /* Sent, but not bootstrapped until the peer is in the DHT. */ + return -EAGAIN; fail_query: peer_list_destroy(&pl); fail_peer: @@ -3449,15 +3440,14 @@ static void dht_kv_remove_expired_reqs(void) pthread_mutex_lock(&dht.reqs.mtx); - list_for_each_safe(p, h, &dht.reqs.list) { + llist_for_each_safe(p, h, &dht.reqs.ll) { struct dht_req * e; e = list_entry(p, struct dht_req, next); if (IS_EXPIRED(e, &now)) { log_dbg(KEY_FMT " Removing expired request.", KEY_VAL(e->key)); - list_del(&e->next); + llist_del(&e->next, &dht.reqs.ll); dht_req_destroy(e); - --dht.reqs.len; } } @@ -3481,6 +3471,8 @@ static void value_list_destroy(struct list_head * vl) #define MUST_REPLICATE(v, now) ((now)->tv_sec > (v)->t_repl + dht.t_repl) #define MUST_REPUBLISH(v, now) /* Close to expiry deadline */ \ (((v)->t_exp - (now)->tv_sec) < (DHT_N_REPUB * dht.t_repl)) +/* A local value must be (re)stored if near expiry or never stored. */ +#define MUST_STORE_LVAL(v, now) (MUST_REPUBLISH(v, now) || (v)->t_repl == 0) static void dht_entry_get_repl_lists(const struct dht_entry * e, struct list_head * repl, struct list_head * rebl, @@ -3489,7 +3481,7 @@ static void dht_entry_get_repl_lists(const struct dht_entry * e, struct list_head * p; struct val_entry * n; - list_for_each(p, &e->vals.list) { + llist_for_each(p, &e->vals) { struct val_entry * v = list_entry(p, struct val_entry, next); if (MUST_REPLICATE(v, now) && !IS_EXPIRED(v, now)) { n = val_entry_create(v->val, v->t_exp); @@ -3500,11 +3492,11 @@ static void dht_entry_get_repl_lists(const struct dht_entry * e, } } - list_for_each(p, &e->lvals.list) { + llist_for_each(p, &e->lvals) { struct val_entry * v = list_entry(p, struct val_entry, next); - if (MUST_REPLICATE(v, now) && MUST_REPUBLISH(v, now)) { + if (MUST_REPLICATE(v, now) && MUST_STORE_LVAL(v, now)) { /* Add expire time here, to allow creating val_entry */ - n = val_entry_create(v->val, now->tv_sec + dht.t_expire); + n = val_entry_create(v->val, now->tv_sec + dht.t_exp); if (n == NULL) continue; @@ -3533,10 +3525,10 @@ static int dht_kv_next_values(uint8_t * key, pthread_rwlock_rdlock(&dht.db.lock); - if (dht.db.kv.len == 0) + if (llist_is_empty(&dht.db.kv.ll)) goto no_entries; - list_for_each_safe(p, h, &dht.db.kv.list) { + llist_for_each_safe(p, h, &dht.db.kv.ll) { e = list_entry(p, struct dht_entry, next); if (IS_CLOSER(e->key, key)) continue; /* Already processed */ @@ -3578,7 +3570,7 @@ static void dht_kv_republish_value(const uint8_t * key, assert(MUST_REPLICATE(v, now)); if (MUST_REPUBLISH(v, now)) - assert(v->t_exp >= now->tv_sec + dht.t_expire); + assert(v->t_exp >= now->tv_sec + dht.t_exp); if (dht_kv_store_remote(key, v->val, v->t_exp) == 0) { log_dbg(KV_FMT " Republished.", KV_VAL(key, v->val)); @@ -3786,12 +3778,15 @@ static void * work(void * o) nanosleep(&now, NULL); } - intv = gcd(dht.t_expire, (dht.t_expire - DHT_N_REPUB * dht.t_repl)); - intv = gcd(intv, gcd(dht.t_repl, dht.t_refresh)) / 2; + intv = gcd(dht.t_exp, (dht.t_exp - DHT_N_REPUB * dht.t_repl)); + intv = gcd(intv, gcd(dht.t_repl, dht.t_refr)) / 2; intv = MAX(1, intv / n); log_dbg("DHT worker starting %ld seconds interval.", intv * n); + /* Flush names registered before we had peers to store them. */ + dht_kv_replicate(); + while (true) { int i = 0; while (tasks[i] != NULL) { @@ -3868,13 +3863,13 @@ int dht_init(struct dir_dht_config * conf) dht.id.len = DHT_TEST_KEY_LEN; dht.addr = DHT_TEST_ADDR; #endif - dht.t0 = now.tv_sec; - dht.alpha = conf->params.alpha; - dht.k = conf->params.k; - dht.t_expire = conf->params.t_expire; - dht.t_refresh = conf->params.t_refresh; - dht.t_repl = conf->params.t_replicate; - dht.peer = conf->peer; + dht.t0 = now.tv_sec; + dht.alpha = conf->params.alpha; + dht.k = conf->params.k; + dht.t_exp = conf->params.t_expire; + dht.t_refr = conf->params.t_refresh; + dht.t_repl = conf->params.t_replicate; + dht.peer = conf->peer; dht.magic = generate_cookie(); @@ -3899,8 +3894,7 @@ int dht_init(struct dir_dht_config * conf) goto fail_cmds_cond; } - list_head_init(&dht.reqs.list); - dht.reqs.len = 0; + llist_init(&dht.reqs.ll); if (pthread_mutex_init(&dht.reqs.mtx, NULL)) { log_err("Failed to initialize request mutex."); @@ -3922,8 +3916,7 @@ int dht_init(struct dir_dht_config * conf) goto fail_reqs_cond; } - list_head_init(&dht.db.kv.list); - dht.db.kv.len = 0; + llist_init(&dht.db.kv.ll); dht.db.kv.vals = 0; dht.db.kv.lvals = 0; @@ -3960,9 +3953,9 @@ int dht_init(struct dir_dht_config * conf) log_dbg(" address: " ADDR_FMT32 ".", ADDR_VAL32(&dht.addr)); log_dbg(" peer: " ADDR_FMT32 ".", ADDR_VAL32(&dht.peer)); log_dbg(" magic cookie: " HASH_FMT64 ".", HASH_VAL64(&dht.magic)); - log_info(" parameters: alpha=%u, k=%zu, t_expire=%ld, " - "t_refresh=%ld, t_replicate=%ld.", - dht.alpha, dht.k, dht.t_expire, dht.t_refresh, dht.t_repl); + log_info(" parameters: alpha=%u, k=%zu, t_exp=%ld, " + "t_refr=%ld, t_replicate=%ld.", + dht.alpha, dht.k, dht.t_exp, dht.t_refr, dht.t_repl); #endif dht.state = DHT_INIT; @@ -4015,11 +4008,10 @@ void dht_fini(void) pthread_mutex_lock(&dht.reqs.mtx); - list_for_each_safe(p, h, &dht.reqs.list) { + llist_for_each_safe(p, h, &dht.reqs.ll) { struct dht_req * r = list_entry(p, struct dht_req, next); - list_del(&r->next); + llist_del(&r->next, &dht.reqs.ll); dht_req_destroy(r); - dht.reqs.len--; } pthread_mutex_unlock(&dht.reqs.mtx); @@ -4029,11 +4021,10 @@ void dht_fini(void) pthread_rwlock_wrlock(&dht.db.lock); - list_for_each_safe(p, h, &dht.db.kv.list) { + llist_for_each_safe(p, h, &dht.db.kv.ll) { struct dht_entry * e = list_entry(p, struct dht_entry, next); - list_del(&e->next); + llist_del(&e->next, &dht.db.kv.ll); dht_entry_destroy(e); - dht.db.kv.len--; } if (dht.db.contacts.root != NULL) @@ -4043,10 +4034,10 @@ void dht_fini(void) pthread_rwlock_destroy(&dht.db.lock); - assert(dht.db.kv.len == 0); + assert(llist_is_empty(&dht.db.kv.ll)); assert(dht.db.kv.vals == 0); assert(dht.db.kv.lvals == 0); - assert(dht.reqs.len == 0); + assert(llist_is_empty(&dht.reqs.ll)); freebuf(dht.id); } diff --git a/src/ipcpd/unicast/dir/dht.h b/src/ipcpd/unicast/dir/dht.h index 852a5130..ed3bb9f0 100644 --- a/src/ipcpd/unicast/dir/dht.h +++ b/src/ipcpd/unicast/dir/dht.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Distributed Hash Table based on Kademlia * diff --git a/src/ipcpd/unicast/dir/dht.proto b/src/ipcpd/unicast/dir/dht.proto index ea74805f..02b6b341 100644 --- a/src/ipcpd/unicast/dir/dht.proto +++ b/src/ipcpd/unicast/dir/dht.proto @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * DHT protocol, based on Kademlia * diff --git a/src/ipcpd/unicast/dir/ops.h b/src/ipcpd/unicast/dir/ops.h index 8c6e5eb5..6c336ee0 100644 --- a/src/ipcpd/unicast/dir/ops.h +++ b/src/ipcpd/unicast/dir/ops.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Directory policy ops * diff --git a/src/ipcpd/unicast/dir/pol.h b/src/ipcpd/unicast/dir/pol.h index eae4b2e7..8ccf4f95 100644 --- a/src/ipcpd/unicast/dir/pol.h +++ b/src/ipcpd/unicast/dir/pol.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Directory policies * diff --git a/src/ipcpd/unicast/dir/tests/CMakeLists.txt b/src/ipcpd/unicast/dir/tests/CMakeLists.txt index 897f1ec2..eded823f 100644 --- a/src/ipcpd/unicast/dir/tests/CMakeLists.txt +++ b/src/ipcpd/unicast/dir/tests/CMakeLists.txt @@ -3,38 +3,36 @@ get_filename_component(CURRENT_SOURCE_PARENT_DIR get_filename_component(CURRENT_BINARY_PARENT_DIR ${CMAKE_CURRENT_BINARY_DIR} DIRECTORY) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -include_directories(${CURRENT_SOURCE_PARENT_DIR}) -include_directories(${CURRENT_BINARY_PARENT_DIR}) - -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_BINARY_DIR}/include) - get_filename_component(PARENT_PATH ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) get_filename_component(PARENT_DIR ${PARENT_PATH} NAME) +compute_test_prefix() + create_test_sourcelist(${PARENT_DIR}_tests test_suite.c # Add new tests here dht_test.c - ) +) -protobuf_generate_c(DHT_PROTO_SRCS KAD_PROTO_HDRS ../dht.proto) +protobuf_generate_c(DHT_PROTO_SRCS KAD_PROTO_HDRS ${CURRENT_SOURCE_PARENT_DIR}/dht.proto) add_executable(${PARENT_DIR}_test ${${PARENT_DIR}_tests} ${DHT_PROTO_SRCS}) -target_link_libraries(${PARENT_DIR}_test ouroboros-common) -add_dependencies(check ${PARENT_DIR}_test) +target_include_directories(${PARENT_DIR}_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CURRENT_SOURCE_PARENT_DIR} + ${CURRENT_BINARY_PARENT_DIR} + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_BINARY_DIR}/include + ${CMAKE_SOURCE_DIR}/src/ipcpd + ${CMAKE_BINARY_DIR}/src/ipcpd + ${CMAKE_SOURCE_DIR}/src/ipcpd/unicast + ${CMAKE_BINARY_DIR}/src/ipcpd/unicast +) + +disable_test_logging_for_target(${PARENT_DIR}_test) -set(tests_to_run ${${PARENT_DIR}_tests}) -if(CMAKE_VERSION VERSION_LESS "3.29.0") - remove(tests_to_run test_suite.c) -else () - list(POP_FRONT tests_to_run) -endif() +target_link_libraries(${PARENT_DIR}_test ouroboros-common) +add_dependencies(build_tests ${PARENT_DIR}_test) -foreach (test ${tests_to_run}) - get_filename_component(test_name ${test} NAME_WE) - add_test(${test_name} ${C_TEST_PATH}/${PARENT_DIR}_test ${test_name}) -endforeach (test) +ouroboros_register_tests(TARGET ${PARENT_DIR}_test TESTS ${${PARENT_DIR}_tests}) diff --git a/src/ipcpd/unicast/dir/tests/dht_test.c b/src/ipcpd/unicast/dir/tests/dht_test.c index cb6b0f9f..1f7026b3 100644 --- a/src/ipcpd/unicast/dir/tests/dht_test.c +++ b/src/ipcpd/unicast/dir/tests/dht_test.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Unit tests of the DHT * @@ -27,7 +27,7 @@ #define _POSIX_C_SOURCE 200112L #endif -#include <ouroboros/test.h> +#include <test/test.h> #include <ouroboros/list.h> #include <ouroboros/utils.h> @@ -46,10 +46,9 @@ /* forward declare for use in the dht code */ /* Packet sink for DHT tests */ struct { - bool enabled; + bool enabled; - struct list_head list; - size_t len; + struct llist msgs; } sink; struct message { @@ -66,8 +65,6 @@ static int sink_send_msg(buffer_t * pkt, assert(pkt != NULL); assert(addr != 0); - assert(!list_is_empty(&sink.list) || sink.len == 0); - if (!sink.enabled) goto finish; @@ -83,9 +80,8 @@ static int sink_send_msg(buffer_t * pkt, m->dst = addr; - list_add_tail(&m->next, &sink.list); + llist_add_tail(&m->next, &sink.msgs); - ++sink.len; finish: freebuf(*pkt); @@ -103,8 +99,7 @@ static int sink_send_msg(buffer_t * pkt, static void sink_init(void) { - list_head_init(&sink.list); - sink.len = 0; + llist_init(&sink.msgs); sink.enabled = true; } @@ -113,22 +108,20 @@ static void sink_clear(void) struct list_head * p; struct list_head * h; - list_for_each_safe(p, h, &sink.list) { + llist_for_each_safe(p, h, &sink.msgs) { struct message * m = list_entry(p, struct message, next); - list_del(&m->next); + llist_del(&m->next, &sink.msgs); dht_msg__free_unpacked((dht_msg_t *) m->msg, NULL); free(m); - --sink.len; } - assert(list_is_empty(&sink.list)); + assert(llist_is_empty(&sink.msgs)); } static void sink_fini(void) { sink_clear(); - - assert(list_is_empty(&sink.list) || sink.len != 0); + sink.enabled = false; } static dht_msg_t * sink_read(void) @@ -136,16 +129,12 @@ static dht_msg_t * sink_read(void) struct message * m; dht_msg_t * msg; - assert(!list_is_empty(&sink.list) || sink.len == 0); - - if (list_is_empty(&sink.list)) + if (llist_is_empty(&sink.msgs)) return NULL; - m = list_first_entry(&sink.list, struct message, next); - - --sink.len; + m = llist_first_entry(&sink.msgs, struct message, next); - list_del(&m->next); + llist_del(&m->next, &sink.msgs); msg = m->msg; @@ -978,7 +967,7 @@ static int test_dht_kv_find_node_rsp_msg_contacts(void) } if ((size_t) n < dht.k) { - printf("Failed to get enough contacts (%zu < %zu).\n", n, dht.k); + printf("Failed to get all contacts (%zu < %zu).\n", n, dht.k); goto fail_fill; } @@ -1204,7 +1193,7 @@ static int test_dht_kv_find_value_rsp_msg_contacts(void) } if ((size_t) n < dht.k) { - printf("Failed to get enough contacts (%zu < %zu).\n", n, dht.k); + printf("Failed to get all contacts (%zu < %zu).\n", n, dht.k); goto fail_fill; } @@ -1591,7 +1580,7 @@ static int test_dht_reg_unreg(void) goto fail_reg; } - if (sink.len != 0) { + if (!llist_is_empty(&sink.msgs)) { printf("Packet sent without contacts!"); goto fail_msg; } @@ -1642,7 +1631,7 @@ static int test_dht_reg_unreg_contacts(void) goto fail_reg; } - if (sink.len != dht.alpha) { + if (sink.msgs.len != dht.alpha) { printf("Packet sent to too few contacts!\n"); goto fail_msg; } @@ -1784,7 +1773,7 @@ static int test_dht_query(void) goto fail_get; } - if (sink.len != 0) { + if (!llist_is_empty(&sink.msgs)) { printf("Packet sent without contacts!"); goto fail_test; } diff --git a/src/ipcpd/unicast/dt.c b/src/ipcpd/unicast/dt.c index e2679ffe..381ed815 100644 --- a/src/ipcpd/unicast/dt.c +++ b/src/ipcpd/unicast/dt.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Data Transfer Component * @@ -31,10 +31,12 @@ #define DT "dt" #define OUROBOROS_PREFIX DT +#include <ouroboros/atomics.h> #include <ouroboros/bitmap.h> #include <ouroboros/errno.h> #include <ouroboros/logs.h> #include <ouroboros/dev.h> +#include <ouroboros/ipcp-dev.h> #include <ouroboros/notifier.h> #include <ouroboros/rib.h> #ifdef IPCP_FLOW_STATS @@ -45,6 +47,7 @@ #include "common/comp.h" #include "common/connmgr.h" #include "ca.h" +#include "cap.h" #include "ipcp.h" #include "dt.h" #include "pff.h" @@ -68,7 +71,7 @@ #endif struct comp_info { - void (* post_packet)(void * comp, struct shm_du_buff * sdb); + void (* post_packet)(void * comp, struct ssm_pk_buff * spb); void * comp; char * name; }; @@ -77,12 +80,14 @@ struct comp_info { #define TTL_LEN 1 #define QOS_LEN 1 #define ECN_LEN 1 +#define CAP_LEN 1 struct dt_pci { uint64_t dst_addr; qoscube_t qc; uint8_t ttl; uint8_t ecn; + uint8_t cap; uint64_t eid; }; @@ -95,6 +100,7 @@ struct { size_t qc_o; size_t ttl_o; size_t ecn_o; + size_t cap_o; size_t eid_o; /* Initial TTL value */ @@ -114,6 +120,7 @@ static void dt_pci_ser(uint8_t * head, memcpy(head + dt_pci_info.qc_o, &dt_pci->qc, QOS_LEN); memcpy(head + dt_pci_info.ttl_o, &ttl, TTL_LEN); memcpy(head + dt_pci_info.ecn_o, &dt_pci->ecn, ECN_LEN); + memcpy(head + dt_pci_info.cap_o, &dt_pci->cap, CAP_LEN); memcpy(head + dt_pci_info.eid_o, &dt_pci->eid, dt_pci_info.eid_size); } @@ -132,14 +139,15 @@ static void dt_pci_des(uint8_t * head, memcpy(&dt_pci->qc, head + dt_pci_info.qc_o, QOS_LEN); memcpy(&dt_pci->ttl, head + dt_pci_info.ttl_o, TTL_LEN); memcpy(&dt_pci->ecn, head + dt_pci_info.ecn_o, ECN_LEN); + memcpy(&dt_pci->cap, head + dt_pci_info.cap_o, CAP_LEN); memcpy(&dt_pci->eid, head + dt_pci_info.eid_o, dt_pci_info.eid_size); } -static void dt_pci_shrink(struct shm_du_buff * sdb) +static void dt_pci_shrink(struct ssm_pk_buff * spb) { - assert(sdb); + assert(spb); - shm_du_buff_head_release(sdb, dt_pci_info.head_size); + ssm_pk_buff_pop(spb, dt_pci_info.head_size); } struct { @@ -150,6 +158,7 @@ struct { struct pff * pff[QOS_CUBE_MAX]; struct routing_i * routing[QOS_CUBE_MAX]; #ifdef IPCP_FLOW_STATS + /* Flow stats use lock-free atomics; stamp is the validity flag. */ struct { time_t stamp; uint64_t addr; @@ -167,23 +176,29 @@ struct { size_t w_drp_bytes[QOS_CUBE_MAX]; size_t f_nhp_pkt[QOS_CUBE_MAX]; size_t f_nhp_bytes[QOS_CUBE_MAX]; - pthread_mutex_t lock; - } stat[PROG_MAX_FLOWS]; + } stat[PROC_MAX_FLOWS]; size_t n_flows; #endif struct bmp * res_fds; - struct comp_info comps[PROG_RES_FDS]; + struct comp_info comps[PROC_RES_FDS]; pthread_rwlock_t lock; pthread_t listener; } dt; +#ifdef IPCP_FLOW_STATS +#define dt_stat_inc(idx, name, qc, len) \ + do { \ + FETCH_ADD_RELAXED(&dt.stat[idx].name ## _pkt[qc], 1); \ + FETCH_ADD_RELAXED(&dt.stat[idx].name ## _bytes[qc], (len)); \ + } while (0) +#define dt_stat_load(idx, field, qc) LOAD_RELAXED(&dt.stat[idx].field[qc]) + static int dt_rib_read(const char * path, char * buf, size_t len) { -#ifdef IPCP_FLOW_STATS int fd; int i; char str[QOS_BLOCK_LEN + 1]; @@ -192,6 +207,8 @@ static int dt_rib_read(const char * path, char tmstr[RIB_TM_STRLEN]; size_t rxqlen = 0; size_t txqlen = 0; + time_t stamp; + uint64_t addr; struct tm * tm; /* NOTE: we may need stronger checks. */ @@ -205,22 +222,21 @@ static int dt_rib_read(const char * path, buf[0] = '\0'; - pthread_mutex_lock(&dt.stat[fd].lock); - - if (dt.stat[fd].stamp == 0) { - pthread_mutex_unlock(&dt.stat[fd].lock); + stamp = LOAD_ACQUIRE(&dt.stat[fd].stamp); + if (stamp == 0) return 0; - } - if (dt.stat[fd].addr == dt.addr) + addr = LOAD_RELAXED(&dt.stat[fd].addr); + + if (addr == dt.addr) sprintf(addrstr, "%s", dt.comps[fd].name); else - sprintf(addrstr, ADDR_FMT32, ADDR_VAL32(&dt.stat[fd].addr)); + sprintf(addrstr, ADDR_FMT32, ADDR_VAL32(&addr)); - tm = gmtime(&dt.stat[fd].stamp); + tm = gmtime(&stamp); strftime(tmstr, sizeof(tmstr), RIB_TM_FORMAT, tm); - if (fd >= PROG_RES_FDS) { + if (fd >= PROC_RES_FDS) { fccntl(fd, FLOWGRXQLEN, &rxqlen); fccntl(fd, FLOWGTXQLEN, &txqlen); } @@ -249,38 +265,29 @@ static int dt_rib_read(const char * path, " failed nhop (packets): %20zu\n" " failed nhop (bytes): %20zu\n", i, - dt.stat[fd].snd_pkt[i], - dt.stat[fd].snd_bytes[i], - dt.stat[fd].rcv_pkt[i], - dt.stat[fd].rcv_bytes[i], - dt.stat[fd].lcl_w_pkt[i], - dt.stat[fd].lcl_w_bytes[i], - dt.stat[fd].lcl_r_pkt[i], - dt.stat[fd].lcl_r_bytes[i], - dt.stat[fd].r_drp_pkt[i], - dt.stat[fd].r_drp_bytes[i], - dt.stat[fd].w_drp_pkt[i], - dt.stat[fd].w_drp_bytes[i], - dt.stat[fd].f_nhp_pkt[i], - dt.stat[fd].f_nhp_bytes[i] + dt_stat_load(fd, snd_pkt, i), + dt_stat_load(fd, snd_bytes, i), + dt_stat_load(fd, rcv_pkt, i), + dt_stat_load(fd, rcv_bytes, i), + dt_stat_load(fd, lcl_w_pkt, i), + dt_stat_load(fd, lcl_w_bytes, i), + dt_stat_load(fd, lcl_r_pkt, i), + dt_stat_load(fd, lcl_r_bytes, i), + dt_stat_load(fd, r_drp_pkt, i), + dt_stat_load(fd, r_drp_bytes, i), + dt_stat_load(fd, w_drp_pkt, i), + dt_stat_load(fd, w_drp_bytes, i), + dt_stat_load(fd, f_nhp_pkt, i), + dt_stat_load(fd, f_nhp_bytes, i) ); strcat(buf, str); } - pthread_mutex_unlock(&dt.stat[fd].lock); - return RIB_FILE_STRLEN; -#else - (void) path; - (void) buf; - (void) len; - return 0; -#endif } static int dt_rib_readdir(char *** buf) { -#ifdef IPCP_FLOW_STATS char entry[RIB_PATH_LEN + 1]; size_t i; int idx = 0; @@ -296,15 +303,9 @@ static int dt_rib_readdir(char *** buf) if (*buf == NULL) goto fail_entries; - for (i = 0; i < PROG_MAX_FLOWS; ++i) { - pthread_mutex_lock(&dt.stat[i].lock); - - if (dt.stat[i].stamp == 0) { - pthread_mutex_unlock(&dt.stat[i].lock); - break; - } - - pthread_mutex_unlock(&dt.stat[i].lock); + for (i = 0; i < PROC_MAX_FLOWS && idx < (int) dt.n_flows; ++i) { + if (LOAD_RELAXED(&dt.stat[i].stamp) == 0) + continue; /* n-1 flows start at PROC_RES_FDS */ sprintf(entry, "%zu", i); @@ -323,43 +324,35 @@ static int dt_rib_readdir(char *** buf) fail_entry: while (idx-- > 0) free((*buf)[idx]); + free(*buf); fail_entries: pthread_rwlock_unlock(&dt.lock); return -ENOMEM; -#else - (void) buf; - return 0; -#endif } static int dt_rib_getattr(const char * path, struct rib_attr * attr) { -#ifdef IPCP_FLOW_STATS int fd; char * entry; + time_t stamp; entry = strstr(path, RIB_SEPARATOR) + 1; assert(entry); fd = atoi(entry); - pthread_mutex_lock(&dt.stat[fd].lock); + stamp = LOAD_ACQUIRE(&dt.stat[fd].stamp); - if (dt.stat[fd].stamp != -1) { + if (stamp != -1) { attr->size = RIB_FILE_STRLEN; - attr->mtime = dt.stat[fd].stamp; + attr->mtime = stamp; } else { attr->size = 0; attr->mtime = 0; } - pthread_mutex_unlock(&dt.stat[fd].lock); -#else - (void) path; - (void) attr; -#endif return 0; } @@ -369,29 +362,49 @@ static struct rib_ops r_ops = { .getattr = dt_rib_getattr }; -#ifdef IPCP_FLOW_STATS static void stat_used(int fd, uint64_t addr) { struct timespec now; + int i; clock_gettime(CLOCK_REALTIME_COARSE, &now); - pthread_mutex_lock(&dt.stat[fd].lock); - - memset(&dt.stat[fd], 0, sizeof(dt.stat[fd])); + pthread_rwlock_wrlock(&dt.lock); - dt.stat[fd].stamp = (addr != INVALID_ADDR) ? now.tv_sec : 0; - dt.stat[fd].addr = addr; + STORE_RELEASE(&dt.stat[fd].stamp, 0); - pthread_mutex_unlock(&dt.stat[fd].lock); + /* Don't memset: incremented without locks in fast path. */ + for (i = 0; i < QOS_CUBE_MAX; ++i) { + STORE_RELAXED(&dt.stat[fd].snd_pkt[i], 0); + STORE_RELAXED(&dt.stat[fd].snd_bytes[i], 0); + STORE_RELAXED(&dt.stat[fd].rcv_pkt[i], 0); + STORE_RELAXED(&dt.stat[fd].rcv_bytes[i], 0); + STORE_RELAXED(&dt.stat[fd].lcl_r_pkt[i], 0); + STORE_RELAXED(&dt.stat[fd].lcl_r_bytes[i], 0); + STORE_RELAXED(&dt.stat[fd].lcl_w_pkt[i], 0); + STORE_RELAXED(&dt.stat[fd].lcl_w_bytes[i], 0); + STORE_RELAXED(&dt.stat[fd].r_drp_pkt[i], 0); + STORE_RELAXED(&dt.stat[fd].r_drp_bytes[i], 0); + STORE_RELAXED(&dt.stat[fd].w_drp_pkt[i], 0); + STORE_RELAXED(&dt.stat[fd].w_drp_bytes[i], 0); + STORE_RELAXED(&dt.stat[fd].f_nhp_pkt[i], 0); + STORE_RELAXED(&dt.stat[fd].f_nhp_bytes[i], 0); + } - pthread_rwlock_wrlock(&dt.lock); + STORE_RELAXED(&dt.stat[fd].addr, addr); - (addr != INVALID_ADDR) ? ++dt.n_flows : --dt.n_flows; + if (addr != INVALID_ADDR) { + STORE_RELEASE(&dt.stat[fd].stamp, now.tv_sec); + ++dt.n_flows; + } else { + --dt.n_flows; + } pthread_rwlock_unlock(&dt.lock); } +#else +#define dt_stat_inc(idx, name, qc, len) ((void) 0) #endif static void handle_event(void * self, @@ -411,6 +424,7 @@ static void handle_event(void * self, #ifdef IPCP_FLOW_STATS stat_used(fd, c->conn_info.addr); #endif + cap_reset(fd); psched_add(dt.psched, fd); log_dbg("Added fd %d to packet scheduler.", fd); break; @@ -427,46 +441,36 @@ static void handle_event(void * self, } } -static void packet_handler(int fd, - qoscube_t qc, - struct shm_du_buff * sdb) +static time_t packet_handler(int fd, + qoscube_t qc, + struct ssm_pk_buff * spb) { struct dt_pci dt_pci; int ret; int ofd; uint8_t * head; size_t len; + size_t qlen; + bool marks; - len = shm_du_buff_len(sdb); + len = ssm_pk_buff_len(spb); #ifndef IPCP_FLOW_STATS - (void) fd; -#else - pthread_mutex_lock(&dt.stat[fd].lock); - - ++dt.stat[fd].rcv_pkt[qc]; - dt.stat[fd].rcv_bytes[qc] += len; - - pthread_mutex_unlock(&dt.stat[fd].lock); + (void) fd; #endif + dt_stat_inc(fd, rcv, qc, len); + memset(&dt_pci, 0, sizeof(dt_pci)); - head = shm_du_buff_head(sdb); + head = ssm_pk_buff_head(spb); dt_pci_des(head, &dt_pci); if (dt_pci.dst_addr != dt.addr) { if (dt_pci.ttl == 0) { log_dbg("TTL was zero."); - ipcp_sdb_release(sdb); -#ifdef IPCP_FLOW_STATS - pthread_mutex_lock(&dt.stat[fd].lock); - - ++dt.stat[fd].r_drp_pkt[qc]; - dt.stat[fd].r_drp_bytes[qc] += len; - - pthread_mutex_unlock(&dt.stat[fd].lock); -#endif - return; + ipcp_spb_release(spb); + dt_stat_inc(fd, r_drp, qc, len); + return 0; } /* FIXME: Use qoscube from PCI instead of incoming flow. */ @@ -474,75 +478,56 @@ static void packet_handler(int fd, if (ofd < 0) { log_dbg("No next hop for %" PRIu64 ".", dt_pci.dst_addr); - ipcp_sdb_release(sdb); -#ifdef IPCP_FLOW_STATS - pthread_mutex_lock(&dt.stat[fd].lock); + ipcp_spb_release(spb); + dt_stat_inc(fd, f_nhp, qc, len); + return 0; + } - ++dt.stat[fd].f_nhp_pkt[qc]; - dt.stat[fd].f_nhp_bytes[qc] += len; + marks = ca_marks_ecn(); + qlen = marks ? ipcp_flow_queued(ofd) : 0; - pthread_mutex_unlock(&dt.stat[fd].lock); -#endif - return; - } + (void) ca_calc_ecn(qlen, head + dt_pci_info.ecn_o, qc, len); - (void) ca_calc_ecn(ofd, head + dt_pci_info.ecn_o, qc, len); + if (marks) + cap_stamp(head + dt_pci_info.cap_o, cap_get(ofd)); - ret = ipcp_flow_write(ofd, sdb); + ret = ipcp_flow_write(ofd, spb); if (ret < 0) { log_dbg("Failed to write packet to fd %d.", ofd); if (ret == -EFLOWDOWN) notifier_event(NOTIFY_DT_FLOW_DOWN, &ofd); - ipcp_sdb_release(sdb); -#ifdef IPCP_FLOW_STATS - pthread_mutex_lock(&dt.stat[ofd].lock); - - ++dt.stat[ofd].w_drp_pkt[qc]; - dt.stat[ofd].w_drp_bytes[qc] += len; - - pthread_mutex_unlock(&dt.stat[ofd].lock); -#endif - return; + ipcp_spb_release(spb); + dt_stat_inc(ofd, w_drp, qc, len); + return 0; } -#ifdef IPCP_FLOW_STATS - pthread_mutex_lock(&dt.stat[ofd].lock); - ++dt.stat[ofd].snd_pkt[qc]; - dt.stat[ofd].snd_bytes[qc] += len; + dt_stat_inc(ofd, snd, qc, len); - pthread_mutex_unlock(&dt.stat[ofd].lock); -#endif + if (marks) + cap_update(ofd, qlen, len); } else { - dt_pci_shrink(sdb); - if (dt_pci.eid >= PROG_RES_FDS) { + dt_pci_shrink(spb); + if (dt_pci.eid >= PROC_RES_FDS) { uint8_t ecn = *(head + dt_pci_info.ecn_o); - fa_np1_rcv(dt_pci.eid, ecn, sdb); - return; + uint8_t cap = *(head + dt_pci_info.cap_o); + fa_np1_rcv(dt_pci.eid, ecn, cap, spb); + return 0; } if (dt.comps[dt_pci.eid].post_packet == NULL) { log_err("No registered component on eid %" PRIu64 ".", dt_pci.eid); - ipcp_sdb_release(sdb); - return; + ipcp_spb_release(spb); + return 0; } -#ifdef IPCP_FLOW_STATS - pthread_mutex_lock(&dt.stat[fd].lock); - - ++dt.stat[fd].lcl_r_pkt[qc]; - dt.stat[fd].lcl_r_bytes[qc] += len; + dt_stat_inc(fd, lcl_r, qc, len); + dt_stat_inc(dt_pci.eid, snd, qc, len); - pthread_mutex_unlock(&dt.stat[fd].lock); - pthread_mutex_lock(&dt.stat[dt_pci.eid].lock); - - ++dt.stat[dt_pci.eid].snd_pkt[qc]; - dt.stat[dt_pci.eid].snd_bytes[qc] += len; - - pthread_mutex_unlock(&dt.stat[dt_pci.eid].lock); -#endif dt.comps[dt_pci.eid].post_packet(dt.comps[dt_pci.eid].comp, - sdb); + spb); } + + return 0; } static void * dt_conn_handle(void * o) @@ -569,7 +554,9 @@ int dt_init(struct dt_config cfg) { int i; int j; +#ifdef IPCP_FLOW_STATS char dtstr[RIB_NAME_STRLEN + 1]; +#endif enum pol_pff pp; struct conn_info info; @@ -599,9 +586,15 @@ int dt_init(struct dt_config cfg) dt_pci_info.qc_o = dt_pci_info.addr_size; dt_pci_info.ttl_o = dt_pci_info.qc_o + QOS_LEN; dt_pci_info.ecn_o = dt_pci_info.ttl_o + TTL_LEN; - dt_pci_info.eid_o = dt_pci_info.ecn_o + ECN_LEN; + dt_pci_info.cap_o = dt_pci_info.ecn_o + ECN_LEN; + dt_pci_info.eid_o = dt_pci_info.cap_o + CAP_LEN; dt_pci_info.head_size = dt_pci_info.eid_o + dt_pci_info.eid_size; + if (cap_init() < 0) { + log_err("Failed to init capacity estimator."); + goto fail_cap; + } + if (connmgr_comp_init(COMPID_DT, &info)) { log_err("Failed to register with connmgr."); goto fail_connmgr_comp_init; @@ -636,37 +629,27 @@ int dt_init(struct dt_config cfg) goto fail_rwlock_init; } - dt.res_fds = bmp_create(PROG_RES_FDS, 0); + dt.res_fds = bmp_create(PROC_RES_FDS, 0); if (dt.res_fds == NULL) goto fail_res_fds; #ifdef IPCP_FLOW_STATS memset(dt.stat, 0, sizeof(dt.stat)); - for (i = 0; i < PROG_MAX_FLOWS; ++i) - if (pthread_mutex_init(&dt.stat[i].lock, NULL)) { - log_err("Failed to init mutex for flow %d.", i); - for (j = 0; j < i; ++j) - pthread_mutex_destroy(&dt.stat[j].lock); - goto fail_stat_lock; - } - dt.n_flows = 0; -#endif + sprintf(dtstr, "%s." ADDR_FMT32, DT, ADDR_VAL32(&dt.addr)); if (rib_reg(dtstr, &r_ops)) { log_err("Failed to register RIB."); goto fail_rib_reg; } +#endif return 0; - fail_rib_reg: #ifdef IPCP_FLOW_STATS - for (i = 0; i < PROG_MAX_FLOWS; ++i) - pthread_mutex_destroy(&dt.stat[i].lock); - fail_stat_lock: -#endif + fail_rib_reg: bmp_destroy(dt.res_fds); +#endif fail_res_fds: pthread_rwlock_destroy(&dt.lock); fail_rwlock_init: @@ -680,19 +663,21 @@ int dt_init(struct dt_config cfg) fail_routing: connmgr_comp_fini(COMPID_DT); fail_connmgr_comp_init: + cap_fini(); + fail_cap: return -1; } void dt_fini(void) { +#ifdef IPCP_FLOW_STATS char dtstr[RIB_NAME_STRLEN + 1]; +#endif int i; +#ifdef IPCP_FLOW_STATS sprintf(dtstr, "%s.%" PRIu64, DT, dt.addr); rib_unreg(dtstr); -#ifdef IPCP_FLOW_STATS - for (i = 0; i < PROG_MAX_FLOWS; ++i) - pthread_mutex_destroy(&dt.stat[i].lock); #endif bmp_destroy(dt.res_fds); @@ -707,6 +692,8 @@ void dt_fini(void) routing_fini(); connmgr_comp_fini(COMPID_DT); + + cap_fini(); } int dt_start(void) @@ -758,7 +745,7 @@ void dt_stop(void) } int dt_reg_comp(void * comp, - void (* func)(void * func, struct shm_du_buff *), + void (* func)(void * func, struct ssm_pk_buff *), char * name) { int eid; @@ -791,7 +778,7 @@ int dt_reg_comp(void * comp, void dt_unreg_comp(int eid) { - assert(eid >= 0 && eid < PROG_RES_FDS); + assert(eid >= 0 && eid < PROC_RES_FDS); pthread_rwlock_wrlock(&dt.lock); @@ -809,64 +796,65 @@ void dt_unreg_comp(int eid) int dt_write_packet(uint64_t dst_addr, qoscube_t qc, uint64_t eid, - struct shm_du_buff * sdb) + struct ssm_pk_buff * spb, + uint8_t * ecn) { struct dt_pci dt_pci; int fd; int ret; uint8_t * head; size_t len; + size_t qlen; + bool marks; - assert(sdb); + assert(spb); assert(dst_addr != dt.addr); - len = shm_du_buff_len(sdb); - #ifdef IPCP_FLOW_STATS - if (eid < PROG_RES_FDS) { - pthread_mutex_lock(&dt.stat[eid].lock); + len = ssm_pk_buff_len(spb); - ++dt.stat[eid].lcl_r_pkt[qc]; - dt.stat[eid].lcl_r_bytes[qc] += len; - - pthread_mutex_unlock(&dt.stat[eid].lock); - } + if (eid < PROC_RES_FDS) + dt_stat_inc(eid, lcl_r, qc, len); #endif fd = pff_nhop(dt.pff[qc], dst_addr); if (fd < 0) { log_dbg("Could not get nhop for " ADDR_FMT32 ".", ADDR_VAL32(&dst_addr)); #ifdef IPCP_FLOW_STATS - if (eid < PROG_RES_FDS) { - pthread_mutex_lock(&dt.stat[eid].lock); - - ++dt.stat[eid].lcl_r_pkt[qc]; - dt.stat[eid].lcl_r_bytes[qc] += len; - - pthread_mutex_unlock(&dt.stat[eid].lock); - } + if (eid < PROC_RES_FDS) + dt_stat_inc(eid, lcl_r, qc, len); #endif return -EPERM; } - head = shm_du_buff_head_alloc(sdb, dt_pci_info.head_size); + head = ssm_pk_buff_push(spb, dt_pci_info.head_size); if (head == NULL) { log_dbg("Failed to allocate DT header."); goto fail_write; } - len = shm_du_buff_len(sdb); + len = ssm_pk_buff_len(spb); dt_pci.dst_addr = dst_addr; dt_pci.qc = qc; dt_pci.eid = eid; dt_pci.ecn = 0; + dt_pci.cap = 0; + + marks = ca_marks_ecn(); + qlen = marks ? ipcp_flow_queued(fd) : 0; + + (void) ca_calc_ecn(qlen, &dt_pci.ecn, qc, len); + + if (marks) + dt_pci.cap = cap_get(fd); - (void) ca_calc_ecn(fd, &dt_pci.ecn, qc, len); + if (ecn != NULL) + *ecn = dt_pci.ecn; dt_pci_ser(head, &dt_pci); - ret = ipcp_flow_write(fd, sdb); + ret = ipcp_flow_write(fd, spb); if (ret < 0) { log_dbg("Failed to write packet to fd %d.", fd); if (ret == -EFLOWDOWN) @@ -874,31 +862,22 @@ int dt_write_packet(uint64_t dst_addr, goto fail_write; } #ifdef IPCP_FLOW_STATS - pthread_mutex_lock(&dt.stat[fd].lock); + if (dt_pci.eid < PROC_RES_FDS) + dt_stat_inc(fd, lcl_w, qc, len); - if (dt_pci.eid < PROG_RES_FDS) { - ++dt.stat[fd].lcl_w_pkt[qc]; - dt.stat[fd].lcl_w_bytes[qc] += len; - } - ++dt.stat[fd].snd_pkt[qc]; - dt.stat[fd].snd_bytes[qc] += len; - - pthread_mutex_unlock(&dt.stat[fd].lock); + dt_stat_inc(fd, snd, qc, len); #endif + if (marks) + cap_update(fd, qlen, len); + return 0; fail_write: #ifdef IPCP_FLOW_STATS - pthread_mutex_lock(&dt.stat[fd].lock); - - if (eid < PROG_RES_FDS) { - ++dt.stat[fd].lcl_w_pkt[qc]; - dt.stat[fd].lcl_w_bytes[qc] += len; - } - ++dt.stat[fd].w_drp_pkt[qc]; - dt.stat[fd].w_drp_bytes[qc] += len; + if (eid < PROC_RES_FDS) + dt_stat_inc(fd, lcl_w, qc, len); - pthread_mutex_unlock(&dt.stat[fd].lock); + dt_stat_inc(fd, w_drp, qc, len); #endif return -1; } diff --git a/src/ipcpd/unicast/dt.h b/src/ipcpd/unicast/dt.h index 2c5b7978..a055efea 100644 --- a/src/ipcpd/unicast/dt.h +++ b/src/ipcpd/unicast/dt.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Data Transfer component * @@ -25,7 +25,7 @@ #include <ouroboros/ipcp.h> #include <ouroboros/qoscube.h> -#include <ouroboros/shm_rdrbuff.h> +#include <ouroboros/ssm_pool.h> #define DT_COMP "Data Transfer" #define DT_PROTO "dtp" @@ -40,7 +40,7 @@ int dt_start(void); void dt_stop(void); int dt_reg_comp(void * comp, - void (* func)(void * comp, struct shm_du_buff * sdb), + void (* func)(void * comp, struct ssm_pk_buff * spb), char * name); void dt_unreg_comp(int eid); @@ -48,6 +48,7 @@ void dt_unreg_comp(int eid); int dt_write_packet(uint64_t dst_addr, qoscube_t qc, uint64_t eid, - struct shm_du_buff * sdb); + struct ssm_pk_buff * spb, + uint8_t * ecn); #endif /* OUROBOROS_IPCPD_UNICAST_DT_H */ diff --git a/src/ipcpd/unicast/fa.c b/src/ipcpd/unicast/fa.c index ac168bd9..ac2ecaea 100644 --- a/src/ipcpd/unicast/fa.c +++ b/src/ipcpd/unicast/fa.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Flow allocator of the IPC Process * @@ -31,12 +31,15 @@ #define FA "flow-allocator" #define OUROBOROS_PREFIX FA +#include <ouroboros/atomics.h> #include <ouroboros/endian.h> #include <ouroboros/logs.h> #include <ouroboros/fqueue.h> #include <ouroboros/errno.h> #include <ouroboros/dev.h> #include <ouroboros/ipcp-dev.h> +#include <ouroboros/np1_flow.h> +#include <ouroboros/qoscube.h> #include <ouroboros/rib.h> #include <ouroboros/random.h> #include <ouroboros/pthread.h> @@ -48,6 +51,7 @@ #include "ipcp.h" #include "dt.h" #include "ca.h" +#include "np1.h" #include <inttypes.h> #include <stdlib.h> @@ -57,12 +61,13 @@ #define CLOCK_REALTIME_COARSE CLOCK_REALTIME #endif -#define TIMEOUT 10 * MILLION /* nanoseconds */ +#define TIMEOUT 10 * MILLION /* nanoseconds */ +#define MSGBUFSZ 32768 -#define FLOW_REQ 0 -#define FLOW_REPLY 1 -#define FLOW_UPDATE 2 -#define MSGBUFSZ 2048 +#define FLOW_REQ 0 +#define FLOW_REPLY 1 +#define FLOW_UPDATE 2 +#define FLOW_IRM_UPDATE 3 #define STAT_FILE_LEN 0 @@ -78,14 +83,15 @@ struct fa_msg { uint32_t max_gap; uint32_t timeout; uint16_t ece; + uint8_t cap; uint8_t code; uint8_t availability; - uint8_t in_order; + uint8_t service; } __attribute__((packed)); struct cmd { struct list_head next; - struct shm_du_buff * sdb; + struct ssm_pk_buff * spb; }; struct fa_flow { @@ -106,11 +112,13 @@ struct fa_flow { uint64_t r_eid; /* Remote endpoint id */ uint64_t r_addr; /* Remote address */ void * ctx; /* Congestion avoidance context */ + uint64_t fair; /* SFQ virtual finish tag (bytes) */ + uint8_t l_ecn; /* Local first-hop mark (relaxed) */ }; struct { pthread_rwlock_t flows_lock; - struct fa_flow flows[PROG_MAX_FLOWS]; + struct fa_flow flows[PROC_MAX_FLOWS]; #ifdef IPCP_FLOW_STATS size_t n_flows; #endif @@ -124,11 +132,11 @@ struct { struct psched * psched; } fa; +#ifdef IPCP_FLOW_STATS static int fa_rib_read(const char * path, char * buf, size_t len) { -#ifdef IPCP_FLOW_STATS struct fa_flow * flow; int fd; char r_addrstr[21]; @@ -144,7 +152,7 @@ static int fa_rib_read(const char * path, fd = atoi(entry); - if (fd < 0 || fd >= PROG_MAX_FLOWS) + if (fd < 0 || fd >= PROC_MAX_FLOWS) return -1; if (len < 1536) @@ -198,17 +206,10 @@ static int fa_rib_read(const char * path, pthread_rwlock_unlock(&fa.flows_lock); return strlen(buf); -#else - (void) path; - (void) buf; - (void) len; - return 0; -#endif } static int fa_rib_readdir(char *** buf) { -#ifdef IPCP_FLOW_STATS char entry[RIB_PATH_LEN + 1]; size_t i; int idx = 0; @@ -224,7 +225,7 @@ static int fa_rib_readdir(char *** buf) if (*buf == NULL) goto fail_entries; - for (i = 0; i < PROG_MAX_FLOWS; ++i) { + for (i = 0; i < PROC_MAX_FLOWS; ++i) { struct fa_flow * flow; flow = &fa.flows[i]; @@ -249,20 +250,16 @@ static int fa_rib_readdir(char *** buf) fail_entry: while (idx-- > 0) free((*buf)[idx]); + free(*buf); fail_entries: pthread_rwlock_unlock(&fa.flows_lock); return -ENOMEM; -#else - (void) buf; - return 0; -#endif } static int fa_rib_getattr(const char * path, struct rib_attr * attr) { -#ifdef IPCP_FLOW_STATS int fd; char * entry; struct fa_flow * flow; @@ -285,10 +282,7 @@ static int fa_rib_getattr(const char * path, } pthread_rwlock_unlock(&fa.flows_lock); -#else - (void) path; - (void) attr; -#endif + return 0; } @@ -297,6 +291,7 @@ static struct rib_ops r_ops = { .readdir = fa_rib_readdir, .getattr = fa_rib_getattr }; +#endif /* IPCP_FLOW_STATS */ static int eid_to_fd(uint64_t eid) { @@ -305,7 +300,7 @@ static int eid_to_fd(uint64_t eid) fd = eid & 0xFFFFFFFF; - if (fd < 0 || fd >= PROG_MAX_FLOWS) + if (fd < 0 || fd >= PROC_MAX_FLOWS) return -1; flow = &fa.flows[fd]; @@ -328,37 +323,40 @@ static uint64_t gen_eid(int fd) return ((uint64_t) rnd << 32) + fd; } -static void packet_handler(int fd, - qoscube_t qc, - struct shm_du_buff * sdb) +static time_t packet_handler(int fd, + qoscube_t qc, + struct ssm_pk_buff * spb) { struct fa_flow * flow; uint64_t r_addr; uint64_t r_eid; - ca_wnd_t wnd; + time_t wait; size_t len; + uint8_t ecn; flow = &fa.flows[fd]; + ecn = 0; + pthread_rwlock_wrlock(&fa.flows_lock); - len = shm_du_buff_len(sdb); + len = ssm_pk_buff_len(spb); #ifdef IPCP_FLOW_STATS ++flow->p_snd; flow->b_snd += len; #endif - wnd = ca_ctx_update_snd(flow->ctx, len); + wait = ca_ctx_update_snd(flow->ctx, len, + LOAD_RELAXED(&flow->l_ecn), &flow->fair); r_addr = flow->r_addr; r_eid = flow->r_eid; pthread_rwlock_unlock(&fa.flows_lock); - ca_wnd_wait(wnd); - - if (dt_write_packet(r_addr, qc, r_eid, sdb)) { - ipcp_sdb_release(sdb); + if (dt_write_packet(r_addr, qc, r_eid, spb, &ecn)) { + STORE_RELAXED(&flow->l_ecn, ecn); + ipcp_spb_release(spb); log_dbg("Failed to forward packet."); #ifdef IPCP_FLOW_STATS pthread_rwlock_wrlock(&fa.flows_lock); @@ -366,8 +364,12 @@ static void packet_handler(int fd, flow->b_snd_f += len; pthread_rwlock_unlock(&fa.flows_lock); #endif - return; + return wait; } + + STORE_RELAXED(&flow->l_ecn, ecn); + + return wait; } static int fa_flow_init(struct fa_flow * flow) @@ -381,9 +383,7 @@ static int fa_flow_init(struct fa_flow * flow) flow->s_eid = -1; flow->r_addr = INVALID_ADDR; - flow->ctx = ca_ctx_create(); - if (flow->ctx == NULL) - return -1; + /* ctx is acquired once (r_addr, qc) are known (ca_ctx_get). */ #ifdef IPCP_FLOW_STATS clock_gettime(CLOCK_REALTIME_COARSE, &now); @@ -397,7 +397,8 @@ static int fa_flow_init(struct fa_flow * flow) static void fa_flow_fini(struct fa_flow * flow) { - ca_ctx_destroy(flow->ctx); + if (flow->ctx != NULL) + ca_ctx_put(flow->ctx); memset(flow, 0, sizeof(*flow)); @@ -411,7 +412,7 @@ static void fa_flow_fini(struct fa_flow * flow) } static void fa_post_packet(void * comp, - struct shm_du_buff * sdb) + struct ssm_pk_buff * spb) { struct cmd * cmd; @@ -422,11 +423,11 @@ static void fa_post_packet(void * comp, cmd = malloc(sizeof(*cmd)); if (cmd == NULL) { log_err("Command failed. Out of memory."); - ipcp_sdb_release(sdb); + ipcp_spb_release(spb); return; } - cmd->sdb = sdb; + cmd->spb = spb; pthread_mutex_lock(&fa.mtx); @@ -454,16 +455,16 @@ static size_t fa_wait_for_fa_msg(struct fa_msg * msg) pthread_cleanup_pop(true); - len = shm_du_buff_len(cmd->sdb); + len = ssm_pk_buff_len(cmd->spb); if (len > MSGBUFSZ || len < sizeof(*msg)) { log_warn("Invalid flow allocation message (len: %zd).", len); free(cmd); return 0; /* No valid message */ } - memcpy(msg, shm_du_buff_head(cmd->sdb), len); + memcpy(msg, ssm_pk_buff_head(cmd->spb), len); - ipcp_sdb_release(cmd->sdb); + ipcp_spb_release(cmd->spb); free(cmd); @@ -495,11 +496,12 @@ static int fa_handle_flow_req(struct fa_msg * msg, qs.availability = msg->availability; qs.loss = ntoh32(msg->loss); qs.ber = ntoh32(msg->ber); - qs.in_order = msg->in_order; + qs.service = msg->service; qs.max_gap = ntoh32(msg->max_gap); qs.timeout = ntoh32(msg->timeout); - fd = ipcp_wait_flow_req_arr(dst, qs, IPCP_UNICAST_MPL, &data); + fd = ipcp_wait_flow_req_arr(dst, qs, IPCP_UNICAST_MPL, + IPCP_UNICAST_MTU, &data); if (fd < 0) return fd; @@ -513,6 +515,13 @@ static int fa_handle_flow_req(struct fa_msg * msg, flow->r_eid = ntoh64(msg->s_eid); flow->r_addr = ntoh64(msg->s_addr); + flow->ctx = ca_ctx_get(flow->r_addr, qos_spec_to_cube(qs)); + if (flow->ctx == NULL) { + fa_flow_fini(flow); + pthread_rwlock_unlock(&fa.flows_lock); + return -ENOMEM; + } + pthread_rwlock_unlock(&fa.flows_lock); return fd; @@ -527,7 +536,8 @@ static int fa_handle_flow_reply(struct fa_msg * msg, time_t mpl = IPCP_UNICAST_MPL; int response; - assert(len >= sizeof(*msg)); + if (len < sizeof(*msg)) + return -EINVAL; data.data = (uint8_t *) msg + sizeof(*msg); data.len = len - sizeof(*msg); @@ -557,7 +567,8 @@ static int fa_handle_flow_reply(struct fa_msg * msg, pthread_rwlock_unlock(&fa.flows_lock); - if (ipcp_flow_alloc_reply(fd, response, mpl, &data) < 0) { + if (ipcp_flow_alloc_reply(fd, response, mpl, + IPCP_UNICAST_MTU, &data) < 0) { log_err("Failed to reply for flow allocation on fd %d.", fd); return -EIRMD; } @@ -571,8 +582,8 @@ static int fa_handle_flow_update(struct fa_msg * msg, struct fa_flow * flow; int fd; - (void) len; - assert(len >= sizeof(*msg)); + if (len < sizeof(*msg)) + return -EINVAL; pthread_rwlock_wrlock(&fa.flows_lock); @@ -588,13 +599,50 @@ static int fa_handle_flow_update(struct fa_msg * msg, #ifdef IPCP_FLOW_STATS flow->u_rcv++; #endif - ca_ctx_update_ece(flow->ctx, ntoh16(msg->ece)); + ca_ctx_update_ece(flow->ctx, ntoh16(msg->ece), msg->cap); pthread_rwlock_unlock(&fa.flows_lock); return 0; } +static int fa_handle_flow_irm_update(struct fa_msg * msg, + size_t len) +{ + buffer_t data; + int fd; + int flow_id; + + if (len < sizeof(*msg)) + return -EINVAL; + + data.data = (uint8_t *) msg + sizeof(*msg); + data.len = len - sizeof(*msg); + + pthread_rwlock_rdlock(&fa.flows_lock); + + fd = eid_to_fd(ntoh64(msg->r_eid)); + + pthread_rwlock_unlock(&fa.flows_lock); + + if (fd < 0) { + log_err("Flow update for unknown EID %" PRIu64 ".", + ntoh64(msg->r_eid)); + return -ENOTALLOC; + } + + flow_id = np1_flow_id(fd); + if (flow_id < 0) + return -ENOTALLOC; + + if (ipcp_flow_update_arr(flow_id, &data) < 0) { + log_err("Failed to relay flow update on fd %d.", fd); + return -EIRMD; + } + + return 0; +} + static void * fa_handle_packet(void * o) { (void) o; @@ -623,6 +671,10 @@ static void * fa_handle_packet(void * o) if (fa_handle_flow_update(msg, len) < 0) log_err("Error handling flow update."); break; + case FLOW_IRM_UPDATE: + if (fa_handle_flow_irm_update(msg, len) < 0) + log_err("Error handling flow update."); + break; default: log_warn("Recieved unknown flow allocation message."); break; @@ -651,8 +703,10 @@ int fa_init(void) if (pthread_cond_init(&fa.cond, &cattr)) goto fail_cond; +#ifdef IPCP_FLOW_STATS if (rib_reg(FA, &r_ops)) goto fail_rib_reg; +#endif fa.eid = dt_reg_comp(&fa, &fa_post_packet, FA); if ((int) fa.eid < 0) @@ -665,8 +719,10 @@ int fa_init(void) return 0; fail_dt_reg: +#ifdef IPCP_FLOW_STATS rib_unreg(FA); fail_rib_reg: +#endif pthread_cond_destroy(&fa.cond); fail_cond: pthread_condattr_destroy(&cattr); @@ -680,20 +736,29 @@ int fa_init(void) void fa_fini(void) { +#ifdef IPCP_FLOW_STATS rib_unreg(FA); - +#endif pthread_cond_destroy(&fa.cond);; pthread_mutex_destroy(&fa.mtx); pthread_rwlock_destroy(&fa.flows_lock); } +static int np1_flow_read_fa(int fd, + struct ssm_pk_buff ** spb) +{ + return np1_flow_read(fd, spb, NP1_GET_POOL(fd)); +} + int fa_start(void) { +#ifndef BUILD_CONTAINER struct sched_param par; int pol; int max; +#endif - fa.psched = psched_create(packet_handler, np1_flow_read); + fa.psched = psched_create(packet_handler, np1_flow_read_fa); if (fa.psched == NULL) { log_err("Failed to start packet scheduler."); goto fail_psched; @@ -704,6 +769,7 @@ int fa_start(void) goto fail_thread; } +#ifndef BUILD_CONTAINER if (pthread_getschedparam(fa.worker, &pol, &par)) { log_err("Failed to get worker thread scheduling parameters."); goto fail_sched; @@ -721,12 +787,15 @@ int fa_start(void) log_err("Failed to set scheduler priority to maximum."); goto fail_sched; } +#endif return 0; +#ifndef BUILD_CONTAINER fail_sched: pthread_cancel(fa.worker); pthread_join(fa.worker, NULL); +#endif fail_thread: psched_destroy(fa.psched); fail_psched: @@ -747,7 +816,7 @@ int fa_alloc(int fd, const buffer_t * data) { struct fa_msg * msg; - struct shm_du_buff * sdb; + struct ssm_pk_buff * spb; struct fa_flow * flow; uint64_t addr; qoscube_t qc = QOS_CUBE_BE; @@ -760,10 +829,10 @@ int fa_alloc(int fd, len = sizeof(*msg) + ipcp_dir_hash_len(); - if (ipcp_sdb_reserve(&sdb, len + data->len)) + if (ipcp_spb_reserve(&spb, len + data->len)) return -1; - msg = (struct fa_msg *) shm_du_buff_head(sdb); + msg = (struct fa_msg *) ssm_pk_buff_head(spb); memset(msg, 0, sizeof(*msg)); eid = gen_eid(fd); @@ -776,17 +845,17 @@ int fa_alloc(int fd, msg->availability = qs.availability; msg->loss = hton32(qs.loss); msg->ber = hton32(qs.ber); - msg->in_order = qs.in_order; + msg->service = qs.service; msg->max_gap = hton32(qs.max_gap); msg->timeout = hton32(qs.timeout); memcpy(msg + 1, dst, ipcp_dir_hash_len()); if (data->len > 0) - memcpy(shm_du_buff_head(sdb) + len, data->data, data->len); + memcpy(ssm_pk_buff_head(spb) + len, data->data, data->len); - if (dt_write_packet(addr, qc, fa.eid, sdb)) { + if (dt_write_packet(addr, qc, fa.eid, spb, NULL)) { log_err("Failed to send flow allocation request packet."); - ipcp_sdb_release(sdb); + ipcp_spb_release(spb); return -1; } @@ -798,6 +867,13 @@ int fa_alloc(int fd, flow->r_addr = addr; flow->s_eid = eid; + flow->ctx = ca_ctx_get(addr, qos_spec_to_cube(qs)); + if (flow->ctx == NULL) { + fa_flow_fini(flow); + pthread_rwlock_unlock(&fa.flows_lock); + return -1; + } + pthread_rwlock_unlock(&fa.flows_lock); return 0; @@ -808,7 +884,7 @@ int fa_alloc_resp(int fd, const buffer_t * data) { struct fa_msg * msg; - struct shm_du_buff * sdb; + struct ssm_pk_buff * spb; struct fa_flow * flow; qoscube_t qc = QOS_CUBE_BE; @@ -819,13 +895,13 @@ int fa_alloc_resp(int fd, goto fail_alloc_resp; } - if (ipcp_sdb_reserve(&sdb, sizeof(*msg) + data->len)) { - log_err("Failed to reserve sdb (%zu bytes).", + if (ipcp_spb_reserve(&spb, sizeof(*msg) + data->len)) { + log_err("Failed to reserve spb (%zu bytes).", sizeof(*msg) + data->len); goto fail_reserve; } - msg = (struct fa_msg *) shm_du_buff_head(sdb); + msg = (struct fa_msg *) ssm_pk_buff_head(spb); memset(msg, 0, sizeof(*msg)); msg->code = FLOW_REPLY; @@ -840,7 +916,7 @@ int fa_alloc_resp(int fd, pthread_rwlock_unlock(&fa.flows_lock); - if (dt_write_packet(flow->r_addr, qc, fa.eid, sdb)) { + if (dt_write_packet(flow->r_addr, qc, fa.eid, spb, NULL)) { log_err("Failed to send flow allocation response packet."); goto fail_packet; } @@ -856,7 +932,7 @@ int fa_alloc_resp(int fd, return 0; fail_packet: - ipcp_sdb_release(sdb); + ipcp_spb_release(spb); fail_reserve: pthread_rwlock_wrlock(&fa.flows_lock); fa_flow_fini(flow); @@ -865,6 +941,44 @@ int fa_alloc_resp(int fd, return -1; } +int fa_irm_update(int fd, + const buffer_t * data) +{ + struct fa_msg * msg; + struct ssm_pk_buff * spb; + struct fa_flow * flow; + qoscube_t qc = QOS_CUBE_BE; + uint64_t r_addr; + + flow = &fa.flows[fd]; + + if (ipcp_spb_reserve(&spb, sizeof(*msg) + data->len)) + return -1; + + msg = (struct fa_msg *) ssm_pk_buff_head(spb); + memset(msg, 0, sizeof(*msg)); + + msg->code = FLOW_IRM_UPDATE; + if (data->len > 0) + memcpy(msg + 1, data->data, data->len); + + pthread_rwlock_rdlock(&fa.flows_lock); + + msg->r_eid = hton64(flow->r_eid); + msg->s_eid = hton64(flow->s_eid); + r_addr = flow->r_addr; + + pthread_rwlock_unlock(&fa.flows_lock); + + if (dt_write_packet(r_addr, qc, fa.eid, spb, NULL)) { + log_err("Failed to send flow update packet."); + ipcp_spb_release(spb); + return -1; + } + + return 0; +} + int fa_dealloc(int fd) { if (ipcp_flow_fini(fd) < 0) @@ -884,20 +998,21 @@ int fa_dealloc(int fd) } static int fa_update_remote(int fd, - uint16_t ece) + uint16_t ece, + uint8_t cap) { struct fa_msg * msg; - struct shm_du_buff * sdb; + struct ssm_pk_buff * spb; qoscube_t qc = QOS_CUBE_BE; struct fa_flow * flow; uint64_t r_addr; - if (ipcp_sdb_reserve(&sdb, sizeof(*msg))) { - log_err("Failed to reserve sdb (%zu bytes).", sizeof(*msg)); + if (ipcp_spb_reserve(&spb, sizeof(*msg))) { + log_err("Failed to reserve spb (%zu bytes).", sizeof(*msg)); return -1; } - msg = (struct fa_msg *) shm_du_buff_head(sdb); + msg = (struct fa_msg *) ssm_pk_buff_head(spb); memset(msg, 0, sizeof(*msg)); @@ -908,6 +1023,7 @@ static int fa_update_remote(int fd, msg->code = FLOW_UPDATE; msg->r_eid = hton64(flow->r_eid); msg->ece = hton16(ece); + msg->cap = cap; r_addr = flow->r_addr; #ifdef IPCP_FLOW_STATS @@ -916,9 +1032,9 @@ static int fa_update_remote(int fd, pthread_rwlock_unlock(&fa.flows_lock); - if (dt_write_packet(r_addr, qc, fa.eid, sdb)) { + if (dt_write_packet(r_addr, qc, fa.eid, spb, NULL)) { log_err("Failed to send flow update packet."); - ipcp_sdb_release(sdb); + ipcp_spb_release(spb); return -1; } @@ -927,15 +1043,17 @@ static int fa_update_remote(int fd, void fa_np1_rcv(uint64_t eid, uint8_t ecn, - struct shm_du_buff * sdb) + uint8_t cap, + struct ssm_pk_buff * spb) { struct fa_flow * flow; bool update; uint16_t ece; + uint8_t fcap; int fd; size_t len; - len = shm_du_buff_len(sdb); + len = ssm_pk_buff_len(spb); pthread_rwlock_wrlock(&fa.flows_lock); @@ -943,7 +1061,7 @@ void fa_np1_rcv(uint64_t eid, if (fd < 0) { pthread_rwlock_unlock(&fa.flows_lock); log_dbg("Received packet for unknown EID %" PRIu64 ".", eid); - ipcp_sdb_release(sdb); + ipcp_spb_release(spb); return; } @@ -953,13 +1071,13 @@ void fa_np1_rcv(uint64_t eid, ++flow->p_rcv; flow->b_rcv += len; #endif - update = ca_ctx_update_rcv(flow->ctx, len, ecn, &ece); + update = ca_ctx_update_rcv(flow->ctx, len, ecn, cap, &ece, &fcap); pthread_rwlock_unlock(&fa.flows_lock); - if (ipcp_flow_write(fd, sdb) < 0) { + if (np1_flow_write(fd, spb, NP1_GET_POOL(fd)) < 0) { log_dbg("Failed to write to flow %d.", fd); - ipcp_sdb_release(sdb); + ipcp_spb_release(spb); #ifdef IPCP_FLOW_STATS pthread_rwlock_wrlock(&fa.flows_lock); ++flow->p_rcv_f; @@ -969,5 +1087,5 @@ void fa_np1_rcv(uint64_t eid, } if (update) - fa_update_remote(eid, ece); + fa_update_remote(eid, ece, fcap); } diff --git a/src/ipcpd/unicast/fa.h b/src/ipcpd/unicast/fa.h index 1e716966..1d0012b0 100644 --- a/src/ipcpd/unicast/fa.h +++ b/src/ipcpd/unicast/fa.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Flow allocator of the IPC Process * @@ -45,8 +45,12 @@ int fa_alloc_resp(int fd, int fa_dealloc(int fd); +int fa_irm_update(int fd, + const buffer_t * data); + void fa_np1_rcv(uint64_t eid, uint8_t ecn, - struct shm_du_buff * sdb); + uint8_t cap, + struct ssm_pk_buff * spb); #endif /* OUROBOROS_IPCPD_UNICAST_FA_H */ diff --git a/src/ipcpd/unicast/main.c b/src/ipcpd/unicast/main.c index 7989d3e1..1155b88b 100644 --- a/src/ipcpd/unicast/main.c +++ b/src/ipcpd/unicast/main.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Unicast IPC Process * @@ -273,7 +273,8 @@ static struct ipcp_ops unicast_ops = { .ipcp_flow_alloc = fa_alloc, .ipcp_flow_join = NULL, .ipcp_flow_alloc_resp = fa_alloc_resp, - .ipcp_flow_dealloc = fa_dealloc + .ipcp_flow_dealloc = fa_dealloc, + .ipcp_flow_update = fa_irm_update }; int main(int argc, @@ -307,8 +308,8 @@ int main(int argc, ipcp_sigwait(); if (ipcp_get_state() == IPCP_SHUTDOWN) { - stop_components(); ipcp_stop(); + stop_components(); finalize_components(); } else { ipcp_stop(); diff --git a/src/ipcpd/unicast/pff.c b/src/ipcpd/unicast/pff.c index 9b2aa2b4..c8c3126f 100644 --- a/src/ipcpd/unicast/pff.c +++ b/src/ipcpd/unicast/pff.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * PDU Forwarding Function * diff --git a/src/ipcpd/unicast/pff.h b/src/ipcpd/unicast/pff.h index f44e5531..2eb42ce6 100644 --- a/src/ipcpd/unicast/pff.h +++ b/src/ipcpd/unicast/pff.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * PDU Forwarding Function * diff --git a/src/ipcpd/unicast/pff/alternate.c b/src/ipcpd/unicast/pff/alternate.c index 85e85914..1c508c1b 100644 --- a/src/ipcpd/unicast/pff/alternate.c +++ b/src/ipcpd/unicast/pff/alternate.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Policy for PFF with alternate next hops * @@ -211,7 +211,7 @@ struct pff_i * alternate_pff_create(void) if (pthread_rwlock_init(&tmp->lock, NULL)) goto fail_lock; - tmp->pft = pft_create(PFT_SIZE, false); + tmp->pft = pft_create(PFT_SIZE); if (tmp->pft == NULL) goto fail_pft; diff --git a/src/ipcpd/unicast/pff/alternate.h b/src/ipcpd/unicast/pff/alternate.h index 96207e74..ae3758dc 100644 --- a/src/ipcpd/unicast/pff/alternate.h +++ b/src/ipcpd/unicast/pff/alternate.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Policy for PFF with alternate next hops * diff --git a/src/ipcpd/unicast/pff/multipath.c b/src/ipcpd/unicast/pff/multipath.c index cbab0f5f..9ba59592 100644 --- a/src/ipcpd/unicast/pff/multipath.c +++ b/src/ipcpd/unicast/pff/multipath.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Policy for PFF supporting multipath routing * @@ -63,7 +63,7 @@ struct pff_i * multipath_pff_create(void) if (pthread_rwlock_init(&tmp->lock, NULL)) goto fail_rwlock; - tmp->pft = pft_create(PFT_SIZE, false); + tmp->pft = pft_create(PFT_SIZE); if (tmp->pft == NULL) goto fail_pft; diff --git a/src/ipcpd/unicast/pff/multipath.h b/src/ipcpd/unicast/pff/multipath.h index 0eb03476..5329f7fc 100644 --- a/src/ipcpd/unicast/pff/multipath.h +++ b/src/ipcpd/unicast/pff/multipath.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Policy for PFF supporting multipath routing * diff --git a/src/ipcpd/unicast/pff/ops.h b/src/ipcpd/unicast/pff/ops.h index 16a31273..e4cabd11 100644 --- a/src/ipcpd/unicast/pff/ops.h +++ b/src/ipcpd/unicast/pff/ops.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Pff policy ops * diff --git a/src/ipcpd/unicast/pff/pft.c b/src/ipcpd/unicast/pff/pft.c index 8c436113..d0e562d6 100644 --- a/src/ipcpd/unicast/pff/pft.c +++ b/src/ipcpd/unicast/pff/pft.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Packet forwarding table (PFT) with chaining on collisions * @@ -43,12 +43,10 @@ struct pft_entry { struct pft { struct list_head * buckets; - bool hash_key; uint64_t buckets_size; }; -struct pft * pft_create(uint64_t buckets, - bool hash_key) +struct pft * pft_create(uint64_t buckets) { struct pft * tmp; unsigned int i; @@ -69,7 +67,6 @@ struct pft * pft_create(uint64_t buckets, if (tmp == NULL) return NULL; - tmp->hash_key = hash_key; tmp->buckets_size = buckets; tmp->buckets = malloc(buckets * sizeof(*tmp->buckets)); @@ -113,22 +110,10 @@ void pft_flush(struct pft * pft) } } -static uint64_t hash(uint64_t key) -{ - uint64_t res[2]; - - mem_hash(HASH_MD5, res, (uint8_t *) &key, sizeof(key)); - - return res[0]; -} - static uint64_t calc_key(struct pft * pft, uint64_t dst) { - if (pft->hash_key) - dst = hash(dst); - - return (dst & (pft->buckets_size - 1)); + return hash_mix64(dst) & (pft->buckets_size - 1); } int pft_insert(struct pft * pft, diff --git a/src/ipcpd/unicast/pff/pft.h b/src/ipcpd/unicast/pff/pft.h index 711dabcb..15bbe451 100644 --- a/src/ipcpd/unicast/pff/pft.h +++ b/src/ipcpd/unicast/pff/pft.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Packet forwarding table (PFT) with chaining on collisions * @@ -24,14 +24,12 @@ #define OUROBOROS_PFT_H #include <stdint.h> -#include <stdbool.h> #include <stdlib.h> struct pft; /* Buckets is rounded up to the nearest power of 2 */ -struct pft * pft_create(uint64_t buckets, - bool hash_key); +struct pft * pft_create(uint64_t buckets); void pft_destroy(struct pft * table); diff --git a/src/ipcpd/unicast/pff/pol.h b/src/ipcpd/unicast/pff/pol.h index 245b03c4..e31c2794 100644 --- a/src/ipcpd/unicast/pff/pol.h +++ b/src/ipcpd/unicast/pff/pol.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * PDU Forwarding Function policies * diff --git a/src/ipcpd/unicast/pff/simple.c b/src/ipcpd/unicast/pff/simple.c index 5f95e3ce..7befa42f 100644 --- a/src/ipcpd/unicast/pff/simple.c +++ b/src/ipcpd/unicast/pff/simple.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Simple PDU Forwarding Function * @@ -63,7 +63,7 @@ struct pff_i * simple_pff_create(void) return NULL; } - tmp->pft = pft_create(PFT_SIZE, false); + tmp->pft = pft_create(PFT_SIZE); if (tmp->pft == NULL) { pthread_rwlock_destroy(&tmp->lock); free(tmp); diff --git a/src/ipcpd/unicast/pff/simple.h b/src/ipcpd/unicast/pff/simple.h index 0966a186..1046e4c4 100644 --- a/src/ipcpd/unicast/pff/simple.h +++ b/src/ipcpd/unicast/pff/simple.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Simple policy for PFF * diff --git a/src/ipcpd/unicast/pff/tests/CMakeLists.txt b/src/ipcpd/unicast/pff/tests/CMakeLists.txt index 99c32e7a..8c0e3d51 100644 --- a/src/ipcpd/unicast/pff/tests/CMakeLists.txt +++ b/src/ipcpd/unicast/pff/tests/CMakeLists.txt @@ -3,36 +3,32 @@ get_filename_component(CURRENT_SOURCE_PARENT_DIR get_filename_component(CURRENT_BINARY_PARENT_DIR ${CMAKE_CURRENT_BINARY_DIR} DIRECTORY) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -include_directories(${CURRENT_SOURCE_PARENT_DIR}) -include_directories(${CURRENT_BINARY_PARENT_DIR}) - -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_BINARY_DIR}/include) - get_filename_component(PARENT_PATH ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) get_filename_component(PARENT_DIR ${PARENT_PATH} NAME) +compute_test_prefix() + create_test_sourcelist(${PARENT_DIR}_tests test_suite.c # Add new tests here pft_test.c ) add_executable(${PARENT_DIR}_test ${${PARENT_DIR}_tests}) -target_link_libraries(${PARENT_DIR}_test ouroboros-common) -add_dependencies(check ${PARENT_DIR}_test) +target_include_directories(${PARENT_DIR}_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CURRENT_SOURCE_PARENT_DIR} + ${CURRENT_BINARY_PARENT_DIR} + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_BINARY_DIR}/include + ${CMAKE_SOURCE_DIR}/src/ipcpd + ${CMAKE_BINARY_DIR}/src/ipcpd +) + +disable_test_logging_for_target(${PARENT_DIR}_test) +target_link_libraries(${PARENT_DIR}_test PRIVATE ouroboros-common) -set(tests_to_run ${${PARENT_DIR}_tests}) -if(CMAKE_VERSION VERSION_LESS "3.29.0") - remove(tests_to_run test_suite.c) -else () - list(POP_FRONT tests_to_run) -endif() +add_dependencies(build_tests ${PARENT_DIR}_test) -foreach (test ${tests_to_run}) - get_filename_component(test_name ${test} NAME_WE) - add_test(${test_name} ${C_TEST_PATH}/${PARENT_DIR}_test ${test_name}) -endforeach (test) +ouroboros_register_tests(TARGET ${PARENT_DIR}_test TESTS ${${PARENT_DIR}_tests}) diff --git a/src/ipcpd/unicast/pff/tests/pft_test.c b/src/ipcpd/unicast/pff/tests/pft_test.c index 18287fb8..20e73a94 100644 --- a/src/ipcpd/unicast/pff/tests/pft_test.c +++ b/src/ipcpd/unicast/pff/tests/pft_test.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Test of the hash table * @@ -38,15 +38,7 @@ int pft_test(int argc, (void) argc; (void) argv; - pft = pft_create(TBL_SIZE, true); - if (pft == NULL) { - printf("Failed to create.\n"); - return -1; - } - - pft_destroy(pft); - - pft = pft_create(TBL_SIZE, false); + pft = pft_create(TBL_SIZE); if (pft == NULL) { printf("Failed to create.\n"); return -1; diff --git a/src/ipcpd/unicast/psched.c b/src/ipcpd/unicast/psched.c index 7e12148b..2b535d67 100644 --- a/src/ipcpd/unicast/psched.c +++ b/src/ipcpd/unicast/psched.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Packet scheduler component * @@ -30,6 +30,7 @@ #include <ouroboros/errno.h> #include <ouroboros/notifier.h> +#include <ouroboros/time.h> #include "common/connmgr.h" #include "ipcp.h" @@ -41,14 +42,16 @@ #include <stdlib.h> #include <string.h> +#ifndef BUILD_CONTAINER static int qos_prio [] = { QOS_PRIO_BE, QOS_PRIO_VIDEO, QOS_PRIO_VOICE, }; +#endif struct psched { - fset_t * set[QOS_CUBE_MAX]; + fset_t * set[QOS_CUBE_MAX * IPCP_SCHED_THR_MUL]; next_packet_fn_t callback; read_fn_t read; pthread_t readers[QOS_CUBE_MAX * IPCP_SCHED_THR_MUL]; @@ -57,23 +60,139 @@ struct psched { struct sched_info { struct psched * sch; qoscube_t qc; + size_t idx; }; +/* Map an FD to one reader's set: one FD, one thread (no shared FDs). */ +static size_t fd_set_idx(int fd, qoscube_t qc) +{ + return qc + ((size_t) fd % IPCP_SCHED_THR_MUL) * QOS_CUBE_MAX; +} + static void cleanup_reader(void * o) { fqueue_destroy((fqueue_t *) o); } +/* + * Per-reader deadline scheduler: a paced flow is served, then deferred + * to its next-send deadline instead of blocking the thread, so it never + * stalls its thread-mates. + */ +struct dsched { + uint64_t deadline[PROC_MAX_FLOWS]; /* absolute ns, per tracked fd */ + int active[PROC_MAX_FLOWS]; /* compact list of tracked fds */ + int posn[PROC_MAX_FLOWS]; /* fd -> active index, -1 = none */ + size_t n; +}; + +static void cleanup_dsched(void * o) +{ + free(o); +} + +static void dsched_track(struct dsched * d, + int fd, + uint64_t deadline) +{ + if (d->posn[fd] >= 0) + return; + + d->deadline[fd] = deadline; + d->posn[fd] = (int) d->n; + d->active[d->n++] = fd; +} + +static void dsched_untrack(struct dsched * d, + int fd) +{ + int i = d->posn[fd]; + + if (i < 0) + return; + + d->active[i] = d->active[--d->n]; + d->posn[d->active[i]] = i; + d->posn[fd] = -1; +} + +static uint64_t dsched_serve(struct dsched * d, + struct psched * sched, + qoscube_t qc, + uint64_t now) +{ + struct ssm_pk_buff * spb; + uint64_t dmin = 0; + size_t i; + int fd; + time_t wait; + + for (i = 0; i < d->n; ) { + fd = d->active[i]; + + if (d->deadline[fd] > now) { + if (dmin == 0 || d->deadline[fd] < dmin) + dmin = d->deadline[fd]; + ++i; + continue; + } + + if (sched->read(fd, &spb) < 0) { + dsched_untrack(d, fd); + continue; + } + + wait = sched->callback(fd, qc, spb); + if (wait == 0) + continue; + + d->deadline[fd] = now + (uint64_t) wait; + if (dmin == 0 || d->deadline[fd] < dmin) + dmin = d->deadline[fd]; + ++i; + } + + return dmin; +} + +static void dsched_events(struct dsched * d, + fqueue_t * fq, + uint64_t now) +{ + int fd; + + while ((fd = fqueue_next(fq)) >= 0) { + switch (fqueue_type(fq)) { + case FLOW_DEALLOC: + dsched_untrack(d, fd); + notifier_event(NOTIFY_DT_FLOW_DEALLOC, &fd); + break; + case FLOW_DOWN: + notifier_event(NOTIFY_DT_FLOW_DOWN, &fd); + break; + case FLOW_UP: + notifier_event(NOTIFY_DT_FLOW_UP, &fd); + break; + case FLOW_PKT: + dsched_track(d, fd, now); + break; + default: + break; + } + } +} + static void * packet_reader(void * o) { - struct psched * sched; - struct shm_du_buff * sdb; - int fd; - fqueue_t * fq; - qoscube_t qc; + struct psched * sched; + struct dsched * d; + fqueue_t * fq; + qoscube_t qc; + size_t idx; sched = ((struct sched_info *) o)->sch; qc = ((struct sched_info *) o)->qc; + idx = ((struct sched_info *) o)->idx; ipcp_lock_to_core(); @@ -83,37 +202,51 @@ static void * packet_reader(void * o) if (fq == NULL) return (void *) -1; + d = malloc(sizeof(*d)); + if (d == NULL) { + fqueue_destroy(fq); + return (void *) -1; + } + + memset(d, 0, sizeof(*d)); + memset(d->posn, 0xFF, sizeof(d->posn)); /* -1: nothing tracked yet */ + + pthread_cleanup_push(cleanup_dsched, d); pthread_cleanup_push(cleanup_reader, fq); while (true) { - int ret = fevent(sched->set[qc], fq, NULL); + struct timespec now_ts; + struct timespec to; + struct timespec * timeo; + uint64_t now; + uint64_t dmin; + uint64_t delta; + int ret; + + clock_gettime(PTHREAD_COND_CLOCK, &now_ts); + + now = TS_TO_UINT64(now_ts); + + dmin = dsched_serve(d, sched, qc, now); + + if (dmin == 0) { + timeo = NULL; + } else { + delta = dmin > now ? dmin - now : 1; + to.tv_sec = (time_t) (delta / BILLION); + to.tv_nsec = (long) (delta % BILLION); + timeo = &to; + } + + ret = fevent(sched->set[idx], fq, timeo); if (ret < 0) continue; - while ((fd = fqueue_next(fq)) >= 0) { - switch (fqueue_type(fq)) { - case FLOW_DEALLOC: - notifier_event(NOTIFY_DT_FLOW_DEALLOC, &fd); - break; - case FLOW_DOWN: - notifier_event(NOTIFY_DT_FLOW_DOWN, &fd); - break; - case FLOW_UP: - notifier_event(NOTIFY_DT_FLOW_UP, &fd); - break; - case FLOW_PKT: - if (sched->read(fd, &sdb) < 0) - continue; - - sched->callback(fd, qc, sdb); - break; - default: - break; - } - } + dsched_events(d, fq, now); } pthread_cleanup_pop(true); + pthread_cleanup_pop(true); return (void *) 0; } @@ -135,7 +268,7 @@ struct psched * psched_create(next_packet_fn_t callback, psched->callback = callback; psched->read = read; - for (i = 0; i < QOS_CUBE_MAX; ++i) { + for (i = 0; i < QOS_CUBE_MAX * IPCP_SCHED_THR_MUL; ++i) { psched->set[i] = fset_create(); if (psched->set[i] == NULL) { for (j = 0; j < i; ++j) @@ -153,6 +286,7 @@ struct psched * psched_create(next_packet_fn_t callback, } infos[i]->sch = psched; infos[i]->qc = i % QOS_CUBE_MAX; + infos[i]->idx = i; } for (i = 0; i < QOS_CUBE_MAX * IPCP_SCHED_THR_MUL; ++i) { @@ -168,6 +302,7 @@ struct psched * psched_create(next_packet_fn_t callback, } } +#ifndef BUILD_CONTAINER for (i = 0; i < QOS_CUBE_MAX * IPCP_SCHED_THR_MUL; ++i) { struct sched_param par; int pol = SCHED_RR; @@ -185,16 +320,20 @@ struct psched * psched_create(next_packet_fn_t callback, if (pthread_setschedparam(psched->readers[i], pol, &par)) goto fail_sched; } +#endif return psched; +#ifndef BUILD_CONTAINER fail_sched: for (j = 0; j < QOS_CUBE_MAX * IPCP_SCHED_THR_MUL; ++j) pthread_cancel(psched->readers[j]); + for (j = 0; j < QOS_CUBE_MAX * IPCP_SCHED_THR_MUL; ++j) pthread_join(psched->readers[j], NULL); +#endif fail_infos: - for (j = 0; j < QOS_CUBE_MAX; ++j) + for (j = 0; j < QOS_CUBE_MAX * IPCP_SCHED_THR_MUL; ++j) fset_destroy(psched->set[j]); fail_flow_set: free(psched); @@ -213,7 +352,7 @@ void psched_destroy(struct psched * psched) pthread_join(psched->readers[i], NULL); } - for (i = 0; i < QOS_CUBE_MAX; ++i) + for (i = 0; i < QOS_CUBE_MAX * IPCP_SCHED_THR_MUL; ++i) fset_destroy(psched->set[i]); free(psched); @@ -227,7 +366,7 @@ void psched_add(struct psched * psched, assert(psched); ipcp_flow_get_qoscube(fd, &qc); - fset_add(psched->set[qc], fd); + fset_add(psched->set[fd_set_idx(fd, qc)], fd); } void psched_del(struct psched * psched, @@ -238,5 +377,5 @@ void psched_del(struct psched * psched, assert(psched); ipcp_flow_get_qoscube(fd, &qc); - fset_del(psched->set[qc], fd); + fset_del(psched->set[fd_set_idx(fd, qc)], fd); } diff --git a/src/ipcpd/unicast/psched.h b/src/ipcpd/unicast/psched.h index 831f8084..8c2914b3 100644 --- a/src/ipcpd/unicast/psched.h +++ b/src/ipcpd/unicast/psched.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Packet scheduler component * @@ -26,12 +26,12 @@ #include <ouroboros/ipcp-dev.h> #include <ouroboros/fqueue.h> -typedef void (* next_packet_fn_t)(int fd, - qoscube_t qc, - struct shm_du_buff * sdb); +typedef time_t (* next_packet_fn_t)(int fd, + qoscube_t qc, + struct ssm_pk_buff * spb); typedef int (* read_fn_t)(int fd, - struct shm_du_buff ** sdb); + struct ssm_pk_buff ** spb); struct psched * psched_create(next_packet_fn_t callback, read_fn_t read); diff --git a/src/ipcpd/unicast/routing.c b/src/ipcpd/unicast/routing.c index 2ad7b234..1a4e4372 100644 --- a/src/ipcpd/unicast/routing.c +++ b/src/ipcpd/unicast/routing.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Routing component of the IPCP * diff --git a/src/ipcpd/unicast/routing.h b/src/ipcpd/unicast/routing.h index e14960b5..8d721095 100644 --- a/src/ipcpd/unicast/routing.h +++ b/src/ipcpd/unicast/routing.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Routing component of the IPCP * diff --git a/src/ipcpd/unicast/routing/graph.c b/src/ipcpd/unicast/routing/graph.c index 32442dad..c168eb7d 100644 --- a/src/ipcpd/unicast/routing/graph.c +++ b/src/ipcpd/unicast/routing/graph.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Undirected graph structure * @@ -57,10 +57,7 @@ struct edge { }; struct graph { - struct { - struct list_head list; - size_t len; - } vertices; + struct llist vertices; pthread_mutex_t lock; }; @@ -88,7 +85,7 @@ static struct vertex * find_vertex_by_addr(struct graph * graph, assert(graph); - list_for_each(p, &graph->vertices.list) { + llist_for_each(p, &graph->vertices) { struct vertex * e = list_entry(p, struct vertex, next); if (e->addr == addr) return e; @@ -142,7 +139,7 @@ static struct vertex * add_vertex(struct graph * graph, vertex->addr = addr; /* Keep them ordered on address. */ - list_for_each(p, &graph->vertices.list) { + llist_for_each(p, &graph->vertices) { struct vertex * v = list_entry(p, struct vertex, next); if (v->addr > addr) break; @@ -151,7 +148,7 @@ static struct vertex * add_vertex(struct graph * graph, vertex->index = i; - list_add_tail(&vertex->next, p); + llist_add_tail_at(&vertex->next, p, &graph->vertices); /* Increase the index of the vertices to the right. */ list_for_each(p, &vertex->next) { @@ -160,37 +157,41 @@ static struct vertex * add_vertex(struct graph * graph, v->index++; } - ++graph->vertices.len; - return vertex; } +static void free_edges(struct list_head * edges) +{ + struct list_head * p; + struct list_head * h; + + list_for_each_safe(p, h, edges) { + struct edge * e = list_entry(p, struct edge, next); + list_del(&e->next); + free(e); + } +} + static void del_vertex(struct graph * graph, struct vertex * vertex) { struct list_head * p; - struct list_head * h; assert(graph != NULL); assert(vertex != NULL); - list_del(&vertex->next); + llist_del(&vertex->next, &graph->vertices); /* Decrease the index of the vertices to the right. */ - list_for_each(p, &graph->vertices.list) { + llist_for_each(p, &graph->vertices) { struct vertex * v = list_entry(p, struct vertex, next); if (v->addr > vertex->addr) v->index--; } - list_for_each_safe(p, h, &vertex->edges) { - struct edge * e = list_entry(p, struct edge, next); - del_edge(e); - } + free_edges(&vertex->edges); free(vertex); - - --graph->vertices.len; } struct graph * graph_create(void) @@ -206,8 +207,7 @@ struct graph * graph_create(void) return NULL; } - graph->vertices.len = 0; - list_head_init(&graph->vertices.list); + llist_init(&graph->vertices); return graph; } @@ -221,7 +221,7 @@ void graph_destroy(struct graph * graph) pthread_mutex_lock(&graph->lock); - list_for_each_safe(p, n, &graph->vertices.list) { + llist_for_each_safe(p, n, &graph->vertices) { struct vertex * e = list_entry(p, struct vertex, next); del_vertex(graph, e); } @@ -230,7 +230,7 @@ void graph_destroy(struct graph * graph) pthread_mutex_destroy(&graph->lock); - assert(graph->vertices.len == 0); + assert(llist_is_empty(&graph->vertices)); free(graph); } @@ -371,7 +371,7 @@ static int get_min_vertex(struct graph * graph, *v = NULL; - list_for_each(p, &graph->vertices.list) { + llist_for_each(p, &graph->vertices) { if (!used[i] && dist[i] < min) { min = dist[i]; index = i; @@ -420,7 +420,7 @@ static int dijkstra(struct graph * graph, memset(*nhops, 0, sizeof(**nhops) * graph->vertices.len); memset(*dist, 0, sizeof(**dist) * graph->vertices.len); - list_for_each(p, &graph->vertices.list) { + llist_for_each(p, &graph->vertices) { v = list_entry(p, struct vertex, next); (*dist)[i++] = (v->addr == src) ? 0 : INT_MAX; } @@ -526,7 +526,7 @@ static int graph_routing_table_simple(struct graph * graph, list_head_init(table); /* Now construct the routing table from the nhops. */ - list_for_each(p, &graph->vertices.list) { + llist_for_each(p, &graph->vertices) { v = list_entry(p, struct vertex, next); /* This is the src */ @@ -603,9 +603,9 @@ static int graph_routing_table_lfa(struct graph * graph, struct list_head * table, int ** dist) { - int * n_dist[PROG_MAX_FLOWS]; - uint64_t addrs[PROG_MAX_FLOWS]; - int n_index[PROG_MAX_FLOWS]; + int * n_dist[PROC_MAX_FLOWS]; + uint64_t addrs[PROC_MAX_FLOWS]; + int n_index[PROC_MAX_FLOWS]; struct list_head * p; struct list_head * q; struct vertex * v; @@ -618,13 +618,13 @@ static int graph_routing_table_lfa(struct graph * graph, if (graph_routing_table_simple(graph, s_addr, table, dist)) goto fail_table; - for (j = 0; j < PROG_MAX_FLOWS; j++) { + for (j = 0; j < PROC_MAX_FLOWS; j++) { n_dist[j] = NULL; n_index[j] = -1; addrs[j] = -1; } - list_for_each(p, &graph->vertices.list) { + llist_for_each(p, &graph->vertices) { v = list_entry(p, struct vertex, next); if (v->addr != s_addr) @@ -650,7 +650,7 @@ static int graph_routing_table_lfa(struct graph * graph, } /* Loop though all nodes to see if we have a LFA for them. */ - list_for_each(p, &graph->vertices.list) { + llist_for_each(p, &graph->vertices) { v = list_entry(p, struct vertex, next); if (v->addr == s_addr) @@ -695,7 +695,6 @@ static int graph_routing_table_ecmp(struct graph * graph, { struct vertex ** nhops; struct list_head * p; - struct list_head * h; size_t i; struct vertex * v; struct vertex * src_v; @@ -735,16 +734,15 @@ static int graph_routing_table_ecmp(struct graph * graph, free(nhops); - list_for_each(h, &graph->vertices.list) { - v = list_entry(h, struct vertex, next); - if (tmp_dist[v->index] + 1 == (*dist)[v->index]) { + for (i = 0; i < graph->vertices.len; ++i) { + if (tmp_dist[i] + 1 == (*dist)[i]) { n = malloc(sizeof(*n)); if (n == NULL) { free(tmp_dist); goto fail_src_v; } n->nhop = e->nb->addr; - list_add_tail(&n->next, &forwarding[v->index]); + list_add_tail(&n->next, &forwarding[i]); } } @@ -753,38 +751,34 @@ static int graph_routing_table_ecmp(struct graph * graph, list_head_init(table); i = 0; - list_for_each(p, &graph->vertices.list) { + llist_for_each(p, &graph->vertices) { v = list_entry(p, struct vertex, next); - if (v->addr == s_addr) { + if (v->addr == s_addr || list_is_empty(&forwarding[i])) { ++i; continue; } t = malloc(sizeof(*t)); if (t == NULL) - goto fail_t; + goto fail_malloc; t->dst = v->addr; list_head_init(&t->nhops); - if (&forwarding[i] != forwarding[i].nxt) { - t->nhops.nxt = forwarding[i].nxt; - t->nhops.prv = forwarding[i].prv; - forwarding[i].prv->nxt = &t->nhops; - forwarding[i].nxt->prv = &t->nhops; - } + t->nhops.nxt = forwarding[i].nxt; + t->nhops.prv = forwarding[i].prv; + forwarding[i].prv->nxt = &t->nhops; + forwarding[i].nxt->prv = &t->nhops; list_add(&t->next, table); ++i; } - free(*dist); - *dist = NULL; free(forwarding); return 0; - fail_t: + fail_malloc: free_routing_table(table); fail_src_v: free(*dist); diff --git a/src/ipcpd/unicast/routing/graph.h b/src/ipcpd/unicast/routing/graph.h index 8190cc6c..f3766771 100644 --- a/src/ipcpd/unicast/routing/graph.h +++ b/src/ipcpd/unicast/routing/graph.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Undirected graph structure * diff --git a/src/ipcpd/unicast/routing/link-state.c b/src/ipcpd/unicast/routing/link-state.c index e5edf539..c4ea9e1c 100644 --- a/src/ipcpd/unicast/routing/link-state.c +++ b/src/ipcpd/unicast/routing/link-state.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Link state routing policy * @@ -56,7 +56,7 @@ #include <string.h> #define LS_ENTRY_SIZE 104 -#define LSDB "lsdb" +#define lsdb "lsdb" #ifndef CLOCK_REALTIME_COARSE #define CLOCK_REALTIME_COARSE CLOCK_REALTIME @@ -121,16 +121,8 @@ struct { struct graph * graph; struct { - struct { - struct list_head list; - size_t len; - } nbs; - - struct { - struct list_head list; - size_t len; - } db; - + struct llist nbs; + struct llist db; pthread_rwlock_t lock; }; @@ -189,7 +181,7 @@ static struct adjacency * get_adj(const char * path) assert(path); - list_for_each(p, &ls.db.list) { + llist_for_each(p, &ls.db) { struct adjacency * a = list_entry(p, struct adjacency, next); sprintf(entry, LINK_FMT, LINK_VAL(a->src, a->dst)); if (strcmp(entry, path) == 0) @@ -245,7 +237,7 @@ static int lsdb_rib_read(const char * path, pthread_rwlock_rdlock(&ls.lock); - if (ls.db.len + ls.nbs.len == 0) + if (llist_is_empty(&ls.db) && llist_is_empty(&ls.nbs)) goto fail; a = get_adj(entry); @@ -274,7 +266,7 @@ static int lsdb_rib_readdir(char *** buf) pthread_rwlock_rdlock(&ls.lock); - if (ls.db.len + ls.nbs.len == 0) { + if (llist_is_empty(&ls.db) && llist_is_empty(&ls.nbs)) { *buf = NULL; goto no_entries; } @@ -284,7 +276,7 @@ static int lsdb_rib_readdir(char *** buf) if (*buf == NULL) goto fail_entries; - list_for_each(p, &ls.nbs.list) { + llist_for_each(p, &ls.nbs) { struct nb * nb = list_entry(p, struct nb, next); char * str = (nb->type == NB_DT ? ".dt " : ".mgmt "); sprintf(entry, "%s" ADDR_FMT32 , str, ADDR_VAL32(&nb->addr)); @@ -295,7 +287,7 @@ static int lsdb_rib_readdir(char *** buf) strcpy((*buf)[idx++], entry); } - list_for_each(p, &ls.db.list) { + llist_for_each(p, &ls.db) { struct adjacency * a = list_entry(p, struct adjacency, next); sprintf(entry, LINK_FMT, LINK_VAL(a->src, a->dst)); (*buf)[idx] = malloc(strlen(entry) + 1); @@ -333,7 +325,7 @@ static int lsdb_add_nb(uint64_t addr, pthread_rwlock_wrlock(&ls.lock); - list_for_each(p, &ls.nbs.list) { + llist_for_each(p, &ls.nbs) { struct nb * el = list_entry(p, struct nb, next); if (addr > el->addr) break; @@ -360,9 +352,7 @@ static int lsdb_add_nb(uint64_t addr, nb->fd = fd; nb->type = type; - list_add_tail(&nb->next, p); - - ++ls.nbs.len; + llist_add_tail_at(&nb->next, p, &ls.nbs); log_dbg("Type %s neighbor " ADDR_FMT32 " added.", nb->type == NB_DT ? "dt" : "mgmt", ADDR_VAL32(&addr)); @@ -380,13 +370,12 @@ static int lsdb_del_nb(uint64_t addr, pthread_rwlock_wrlock(&ls.lock); - list_for_each_safe(p, h, &ls.nbs.list) { + llist_for_each_safe(p, h, &ls.nbs) { struct nb * nb = list_entry(p, struct nb, next); if (nb->addr != addr || nb->fd != fd) continue; - list_del(&nb->next); - --ls.nbs.len; + llist_del(&nb->next, &ls.nbs); pthread_rwlock_unlock(&ls.lock); log_dbg("Type %s neighbor " ADDR_FMT32 " deleted.", nb->type == NB_DT ? "dt" : "mgmt", ADDR_VAL32(&addr)); @@ -406,7 +395,7 @@ static int nbr_to_fd(uint64_t addr) pthread_rwlock_rdlock(&ls.lock); - list_for_each(p, &ls.nbs.list) { + llist_for_each(p, &ls.nbs) { struct nb * nb = list_entry(p, struct nb, next); if (nb->addr == addr && nb->type == NB_DT) { fd = nb->fd; @@ -426,7 +415,7 @@ static void calculate_pff(struct routing_i * instance) struct list_head table; struct list_head * p; struct list_head * q; - int fds[PROG_MAX_FLOWS]; + int fds[PROC_MAX_FLOWS]; assert(instance); @@ -494,7 +483,7 @@ static int lsdb_add_link(uint64_t src, pthread_rwlock_wrlock(&ls.lock); - list_for_each(p, &ls.db.list) { + llist_for_each(p, &ls.db) { struct adjacency * a = list_entry(p, struct adjacency, next); if (a->dst == dst && a->src == src) { if (a->seqno < seqno) { @@ -521,9 +510,7 @@ static int lsdb_add_link(uint64_t src, adj->seqno = seqno; adj->stamp = now.tv_sec; - list_add_tail(&adj->next, p); - - ls.db.len++; + llist_add_tail_at(&adj->next, p, &ls.db); if (graph_update_edge(ls.graph, src, dst, *qs)) log_warn("Failed to add edge to graph."); @@ -543,15 +530,13 @@ static int lsdb_del_link(uint64_t src, pthread_rwlock_wrlock(&ls.lock); - list_for_each_safe(p, h, &ls.db.list) { + llist_for_each_safe(p, h, &ls.db) { struct adjacency * a = list_entry(p, struct adjacency, next); if (a->dst == dst && a->src == src) { - list_del(&a->next); + llist_del(&a->next, &ls.db); if (graph_del_edge(ls.graph, src, dst)) log_warn("Failed to delete edge from graph."); - ls.db.len--; - pthread_rwlock_unlock(&ls.lock); set_pff_modified(false); free(a); @@ -599,7 +584,7 @@ static void send_lsm(uint64_t src, lsm.s_addr = hton64(src); lsm.seqno = hton64(seqno); - list_for_each(p, &ls.nbs.list) { + llist_for_each(p, &ls.nbs) { struct nb * nb = list_entry(p, struct nb, next); if (nb->type != NB_MGMT) continue; @@ -628,7 +613,7 @@ static void lsdb_replicate(int fd) /* Lock the lsdb, copy the lsms and send outside of lock. */ pthread_rwlock_rdlock(&ls.lock); - list_for_each(p, &ls.db.list) { + llist_for_each(p, &ls.db) { struct adjacency * adj; struct adjacency * cpy; adj = list_entry(p, struct adjacency, next); @@ -675,11 +660,11 @@ static void * lsupdate(void * o) pthread_cleanup_push(__cleanup_rwlock_unlock, &ls.lock); - list_for_each_safe(p, h, &ls.db.list) { + llist_for_each_safe(p, h, &ls.db) { struct adjacency * adj; adj = list_entry(p, struct adjacency, next); if (now.tv_sec > adj->stamp + ls.conf.t_timeo) { - list_del(&adj->next); + llist_del(&adj->next, &ls.db); log_dbg(LINK_FMT " timed out.", LINK_VAL(adj->src, adj->dst)); if (graph_del_edge(ls.graph, adj->src, @@ -746,7 +731,7 @@ static void forward_lsm(uint8_t * buf, pthread_cleanup_push(__cleanup_rwlock_unlock, &ls.lock); - list_for_each(p, &ls.nbs.list) { + llist_for_each(p, &ls.nbs) { struct nb * nb = list_entry(p, struct nb, next); if (nb->type != NB_MGMT || nb->fd == in_fd) continue; @@ -874,19 +859,19 @@ static void handle_event(void * self, pthread_cleanup_pop(true); if (lsdb_add_nb(c->conn_info.addr, c->flow_info.fd, NB_DT)) - log_dbg("Failed to add neighbor to LSDB."); + log_dbg("Failed to add neighbor to lsdb."); if (lsdb_add_link(ls.addr, c->conn_info.addr, 0, &qs)) - log_dbg("Failed to add new adjacency to LSDB."); + log_dbg("Failed to add new adjacency to lsdb."); break; case NOTIFY_DT_CONN_DEL: flow_event(c->flow_info.fd, false); if (lsdb_del_nb(c->conn_info.addr, c->flow_info.fd)) - log_dbg("Failed to delete neighbor from LSDB."); + log_dbg("Failed to delete neighbor from lsdb."); if (lsdb_del_link(ls.addr, c->conn_info.addr)) - log_dbg("Local link was not in LSDB."); + log_dbg("Local link was not in lsdb."); break; case NOTIFY_DT_CONN_QOS: log_dbg("QoS changes currently unsupported."); @@ -902,14 +887,14 @@ static void handle_event(void * self, fccntl(c->flow_info.fd, FLOWSFLAGS, flags | FLOWFRNOPART); fset_add(ls.mgmt_set, c->flow_info.fd); if (lsdb_add_nb(c->conn_info.addr, c->flow_info.fd, NB_MGMT)) - log_warn("Failed to add mgmt neighbor to LSDB."); + log_warn("Failed to add mgmt neighbor to lsdb."); /* replicate the entire lsdb */ lsdb_replicate(c->flow_info.fd); break; case NOTIFY_MGMT_CONN_DEL: fset_del(ls.mgmt_set, c->flow_info.fd); if (lsdb_del_nb(c->conn_info.addr, c->flow_info.fd)) - log_warn("Failed to delete mgmt neighbor from LSDB."); + log_warn("Failed to delete mgmt neighbor from lsdb."); break; default: break; @@ -1090,16 +1075,13 @@ int link_state_init(struct ls_config * conf, goto fail_fset_create; } - list_head_init(&ls.db.list); - list_head_init(&ls.nbs.list); + llist_init(&ls.db); + llist_init(&ls.nbs); list_head_init(&ls.instances.list); - if (rib_reg(LSDB, &r_ops)) + if (rib_reg(lsdb, &r_ops)) goto fail_rib_reg; - ls.db.len = 0; - ls.nbs.len = 0; - return 0; fail_rib_reg: @@ -1121,7 +1103,7 @@ void link_state_fini(void) struct list_head * p; struct list_head * h; - rib_unreg(LSDB); + rib_unreg(lsdb); fset_destroy(ls.mgmt_set); @@ -1131,9 +1113,9 @@ void link_state_fini(void) pthread_rwlock_wrlock(&ls.lock); - list_for_each_safe(p, h, &ls.db.list) { + llist_for_each_safe(p, h, &ls.db) { struct adjacency * a = list_entry(p, struct adjacency, next); - list_del(&a->next); + llist_del(&a->next, &ls.db); free(a); } diff --git a/src/ipcpd/unicast/routing/link-state.h b/src/ipcpd/unicast/routing/link-state.h index 69eb6781..38e19065 100644 --- a/src/ipcpd/unicast/routing/link-state.h +++ b/src/ipcpd/unicast/routing/link-state.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Link state routing policy * diff --git a/src/ipcpd/unicast/routing/ops.h b/src/ipcpd/unicast/routing/ops.h index 4bf75c80..b19c5176 100644 --- a/src/ipcpd/unicast/routing/ops.h +++ b/src/ipcpd/unicast/routing/ops.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Routing policy ops * diff --git a/src/ipcpd/unicast/routing/pol.h b/src/ipcpd/unicast/routing/pol.h index b6a6f150..545f5df2 100644 --- a/src/ipcpd/unicast/routing/pol.h +++ b/src/ipcpd/unicast/routing/pol.h @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Routing policies * diff --git a/src/ipcpd/unicast/routing/tests/CMakeLists.txt b/src/ipcpd/unicast/routing/tests/CMakeLists.txt index b5011474..be2de72c 100644 --- a/src/ipcpd/unicast/routing/tests/CMakeLists.txt +++ b/src/ipcpd/unicast/routing/tests/CMakeLists.txt @@ -3,36 +3,32 @@ get_filename_component(CURRENT_SOURCE_PARENT_DIR get_filename_component(CURRENT_BINARY_PARENT_DIR ${CMAKE_CURRENT_BINARY_DIR} DIRECTORY) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -include_directories(${CURRENT_SOURCE_PARENT_DIR}) -include_directories(${CURRENT_BINARY_PARENT_DIR}) - -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_BINARY_DIR}/include) - get_filename_component(PARENT_PATH ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) get_filename_component(PARENT_DIR ${PARENT_PATH} NAME) +compute_test_prefix() + create_test_sourcelist(${PARENT_DIR}_tests test_suite.c # Add new tests here graph_test.c ) add_executable(${PARENT_DIR}_test ${${PARENT_DIR}_tests}) -target_link_libraries(${PARENT_DIR}_test ouroboros-common) -add_dependencies(check ${PARENT_DIR}_test) +target_include_directories(${PARENT_DIR}_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CURRENT_SOURCE_PARENT_DIR} + ${CURRENT_BINARY_PARENT_DIR} + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_BINARY_DIR}/include + ${CMAKE_SOURCE_DIR}/src/ipcpd + ${CMAKE_BINARY_DIR}/src/ipcpd +) + +disable_test_logging_for_target(${PARENT_DIR}_test) +target_link_libraries(${PARENT_DIR}_test PRIVATE ouroboros-common) -set(tests_to_run ${${PARENT_DIR}_tests}) -if(CMAKE_VERSION VERSION_LESS "3.29.0") - remove(tests_to_run test_suite.c) -else () - list(POP_FRONT tests_to_run) -endif() +add_dependencies(build_tests ${PARENT_DIR}_test) -foreach (test ${tests_to_run}) - get_filename_component(test_name ${test} NAME_WE) - add_test(${test_name} ${C_TEST_PATH}/${PARENT_DIR}_test ${test_name}) -endforeach (test) +ouroboros_register_tests(TARGET ${PARENT_DIR}_test TESTS ${${PARENT_DIR}_tests}) diff --git a/src/ipcpd/unicast/routing/tests/graph_test.c b/src/ipcpd/unicast/routing/tests/graph_test.c index d805640c..40a744ff 100644 --- a/src/ipcpd/unicast/routing/tests/graph_test.c +++ b/src/ipcpd/unicast/routing/tests/graph_test.c @@ -1,5 +1,5 @@ /* - * Ouroboros - Copyright (C) 2016 - 2024 + * Ouroboros - Copyright (C) 2016 - 2026 * * Test of the graph structure * diff --git a/src/ipcpd/unicast/tests/CMakeLists.txt b/src/ipcpd/unicast/tests/CMakeLists.txt new file mode 100644 index 00000000..2e35ed66 --- /dev/null +++ b/src/ipcpd/unicast/tests/CMakeLists.txt @@ -0,0 +1,34 @@ +get_filename_component(CURRENT_SOURCE_PARENT_DIR + ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) +get_filename_component(CURRENT_BINARY_PARENT_DIR + ${CMAKE_CURRENT_BINARY_DIR} DIRECTORY) + +get_filename_component(PARENT_PATH ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) +get_filename_component(PARENT_DIR ${PARENT_PATH} NAME) + +compute_test_prefix() + +create_test_sourcelist(${PARENT_DIR}_tests test_suite.c + # Add new tests here + cap_test.c + ) + +add_executable(${PARENT_DIR}_test ${${PARENT_DIR}_tests}) + +target_include_directories(${PARENT_DIR}_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CURRENT_SOURCE_PARENT_DIR} + ${CURRENT_BINARY_PARENT_DIR} + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_BINARY_DIR}/include + ${CMAKE_SOURCE_DIR}/src/ipcpd + ${CMAKE_BINARY_DIR}/src/ipcpd +) + +disable_test_logging_for_target(${PARENT_DIR}_test) +target_link_libraries(${PARENT_DIR}_test PRIVATE ouroboros-common) + +add_dependencies(build_tests ${PARENT_DIR}_test) + +ouroboros_register_tests(TARGET ${PARENT_DIR}_test TESTS ${${PARENT_DIR}_tests}) diff --git a/src/ipcpd/unicast/tests/cap_test.c b/src/ipcpd/unicast/tests/cap_test.c new file mode 100644 index 00000000..7867b490 --- /dev/null +++ b/src/ipcpd/unicast/tests/cap_test.c @@ -0,0 +1,593 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Unit tests for link capacity estimation + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "cap.c" + +#include <test/test.h> + +#define TICK (50 * 1000ULL) /* 50 us between packets */ +#define LEN 1000ULL /* default packet size (B) */ +#define QLEN 8 /* steady ring backlog */ +#define RATE (LEN * BILLION / TICK) /* LEN per TICK = 20 MB/s */ + +#define SHP_LEN 1250ULL /* shaped-link packet (B) */ +#define SHP_STEP 20 /* packets per shaped window */ +#define SHP_RATE (SHP_LEN * BILLION / (SHP_STEP * TICK)) + +static int test_cap_init_fini(void) +{ + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + if (cap_get(0) != 0 || cap_get(PROC_MAX_FLOWS - 1) != 0) { + printf("Fresh estimator not unknown.\n"); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Exact roundtrip holds for codes >= 32 (rates >= 256 B/s). */ +static int test_cap_codec_roundtrip(void) +{ + unsigned c; + + TEST_START(); + + for (c = 32; c <= 255; c++) { + if (cap_enc(cap_dec((uint8_t) c)) != c) { + printf("Code %u does not roundtrip.\n", c); + goto fail; + } + + if (cap_dec((uint8_t) c) <= cap_dec((uint8_t) (c - 1))) { + printf("Decode not monotone at %u.\n", c); + goto fail; + } + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_codec_bounds(void) +{ + TEST_START(); + + if (cap_enc(0) != 0 || cap_dec(0) != 0) { + printf("Zero is not unknown.\n"); + goto fail; + } + + if (cap_enc(1) != 1) { + printf("Rate 1 encoded as %u.\n", cap_enc(1)); + goto fail; + } + + if (cap_enc(UINT64_MAX) != 255) { + printf("Max rate encoded as %u.\n", cap_enc(UINT64_MAX)); + goto fail; + } + + if (cap_dec(255) <= cap_dec(254)) { + printf("Top code does not decode.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_min(void) +{ + TEST_START(); + + if (cap_min(0, 42) != 42 || cap_min(42, 0) != 42) { + printf("Unknown not skipped in min.\n"); + goto fail; + } + + if (cap_min(0, 0) != 0) { + printf("Two unknowns not unknown.\n"); + goto fail; + } + + if (cap_min(97, 42) != 42 || cap_min(42, 97) != 42) { + printf("Min not taken.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_stamp(void) +{ + uint8_t pci; + + TEST_START(); + + pci = 42; + cap_stamp(&pci, 0); + if (pci != 42) { + printf("Unknown own code overwrote the byte.\n"); + goto fail; + } + + pci = 0; + cap_stamp(&pci, 97); + if (pci != 97) { + printf("Own code not written into unknown.\n"); + goto fail; + } + + pci = 97; + cap_stamp(&pci, 42); + if (pci != 42) { + printf("Lower own code did not lower the byte.\n"); + goto fail; + } + + pci = 42; + cap_stamp(&pci, 97); + if (pci != 42) { + printf("Higher own code raised the byte.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_busy_window(void) +{ + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + /* 1000 B every 50 us, ring steady at 8: drain = 20 MB/s. */ + for (i = 1; i <= 40; i++) + cap_update_at(0, QLEN, LEN, i * TICK); + + if (cap_get(0) != cap_enc(RATE)) { + printf("Estimated code: exp %u, got %u.\n", + cap_enc(RATE), cap_get(0)); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_idle_tolerated(void) +{ + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + for (i = 1; i <= 40; i++) + cap_update_at(0, i == 21 ? 0 : QLEN, LEN, i * TICK); + + if (cap_get(0) != cap_enc(RATE)) { + printf("Grazed window: exp %u, got %u.\n", + cap_enc(RATE), cap_get(0)); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_mostly_idle_rejects(void) +{ + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + for (i = 1; i <= 100; i++) + cap_update_at(0, 0, LEN, i * TICK); + + if (cap_get(0) != 0) { + printf("Idle ring estimated %u.\n", cap_get(0)); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_slow_link_extends(void) +{ + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + /* 1000 B every 100 us: 10 slots/ms closes on a 2 ms window. */ + for (i = 1; i <= 30; i++) + cap_update_at(0, QLEN, LEN, i * 2 * TICK); + + if (cap_get(0) != cap_enc(RATE / 2)) { + printf("Slow link: exp %u, got %u.\n", + cap_enc(RATE / 2), cap_get(0)); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_shaped_link(void) +{ + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + /* 1250 B every ms; one empty observation per 20 packets. */ + for (i = 1; i <= 100; i++) + cap_update_at(0, i % SHP_STEP == 0 ? 0 : 6, SHP_LEN, + i * SHP_STEP * TICK); + + if (cap_get(0) != cap_enc(SHP_RATE)) { + printf("Shaped link: exp %u, got %u.\n", + cap_enc(SHP_RATE), cap_get(0)); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_stale_discard(void) +{ + uint64_t t; + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + /* Open a window, trickle 4 slots, then ~200 ms of silence. */ + for (i = 1; i <= 5; i++) + cap_update_at(0, QLEN, LEN, i * CAP_T_MIN); + + t = 205 * CAP_T_MIN; + + cap_update_at(0, QLEN, LEN, t); + + if (cap_get(0) != 0) { + printf("Gap window estimated %u.\n", cap_get(0)); + goto fail_init; + } + + for (i = 1; i <= 40; i++) + cap_update_at(0, QLEN, LEN, t + i * TICK); + + if (cap_get(0) != cap_enc(RATE)) { + printf("Post-gap: exp %u, got %u.\n", + cap_enc(RATE), cap_get(0)); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_est_empty_start_no_raise(void) +{ + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + cap_update_at(0, 0, LEN, CAP_T_MIN); + + for (i = 1; i <= 40; i++) + cap_update_at(0, QLEN, LEN, CAP_T_MIN + i * TICK); + + if (cap_get(0) != 0) { + printf("Empty-start window raised to %u.\n", + cap_get(0)); + goto fail_init; + } + + for (i = 41; i <= 60; i++) + cap_update_at(0, QLEN, LEN, CAP_T_MIN + i * TICK); + + if (cap_get(0) != cap_enc(RATE)) { + printf("Backlogged window: exp %u, got %u.\n", + cap_enc(RATE), cap_get(0)); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Max filter: fast attack on a high sample, slow release on lower. */ +static int test_cap_est_max_filter(void) +{ + uint8_t high; + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + for (i = 1; i <= 40; i++) + cap_update_at(0, QLEN, LEN, i * TICK); + + high = cap_get(0); + if (high != cap_enc(RATE)) { + printf("Attack missed: exp %u, got %u.\n", cap_enc(RATE), + high); + goto fail_init; + } + + /* Halved packet size: valid samples at 10 MB/s. */ + for (i = 41; i <= 80; i++) + cap_update_at(0, QLEN, LEN / 2, i * TICK); + + if (cap_get(0) >= high) { + printf("Release did not decay: %u.\n", cap_get(0)); + goto fail_init; + } + + if (cap_get(0) <= cap_enc(RATE / 2)) { + printf("Release collapsed to %u.\n", cap_get(0)); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* No fold within CAP_T_MIN of the previous one. */ +static int test_cap_est_gate(void) +{ + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + cap_update_at(0, QLEN, LEN, CAP_T_MIN); + + for (i = 0; i < 5; i++) + cap_update_at(0, QLEN, LEN, CAP_T_MIN + CAP_T_MIN / 2); + + if (cap.est[0].t_gate != CAP_T_MIN) { + printf("Fold ran inside the gate.\n"); + goto fail_init; + } + + if (LOAD_RELAXED(&cap.est[0].c_pkt) != 6) { + printf("Gated packets not counted.\n"); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_cap_reset(void) +{ + size_t i; + + TEST_START(); + + if (cap_init() < 0) { + printf("Failed to init cap.\n"); + goto fail; + } + + for (i = 1; i <= 40; i++) + cap_update_at(0, QLEN, LEN, i * TICK); + + if (cap_get(0) == 0) { + printf("No estimate to reset.\n"); + goto fail_init; + } + + cap_reset(0); + + if (cap_get(0) != 0) { + printf("Reset did not clear the estimate.\n"); + goto fail_init; + } + + cap_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + cap_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int cap_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_cap_init_fini(); + ret |= test_cap_codec_roundtrip(); + ret |= test_cap_codec_bounds(); + ret |= test_cap_min(); + ret |= test_cap_stamp(); + ret |= test_cap_est_busy_window(); + ret |= test_cap_est_idle_tolerated(); + ret |= test_cap_est_mostly_idle_rejects(); + ret |= test_cap_est_slow_link_extends(); + ret |= test_cap_est_shaped_link(); + ret |= test_cap_est_stale_discard(); + ret |= test_cap_est_empty_start_no_raise(); + ret |= test_cap_est_max_filter(); + ret |= test_cap_est_gate(); + ret |= test_cap_reset(); + + return ret; +} |