diff options
Diffstat (limited to 'src/lib')
79 files changed, 21596 insertions, 4935 deletions
diff --git a/src/lib/.gitignore b/src/lib/.gitignore index 8704469b..3ecaf66d 100644 --- a/src/lib/.gitignore +++ b/src/lib/.gitignore @@ -1 +1 @@ -*.pb-c.[ch]
\ No newline at end of file +*.pb-c.[ch] diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 76d0530d..6cd3a8a4 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -1,291 +1,164 @@ -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_BINARY_DIR}/include) - -protobuf_generate_c(IRM_PROTO_SRCS IRM_PROTO_HDRS irmd_messages.proto) -protobuf_generate_c(IPCP_PROTO_SRCS IPCP_PROTO_HDRS ipcpd_messages.proto) -protobuf_generate_c(QOSSPEC_PROTO_SRCS QOSSPEC_PROTO_HDRS - qosspec.proto) -protobuf_generate_c(LAYER_CONFIG_PROTO_SRCS LAYER_CONFIG_PROTO_HDRS - ipcp_config.proto) -protobuf_generate_c(CACEP_PROTO_SRCS CACEP_PROTO_HDRS cacep.proto) - -if (NOT APPLE) - find_library(LIBRT_LIBRARIES rt) - if (NOT LIBRT_LIBRARIES) - message(FATAL_ERROR "Could not find librt") - endif () -else () - set(LIBRT_LIBRARIES "") -endif () - -find_library(LIBPTHREAD_LIBRARIES pthread) -if (NOT LIBPTHREAD_LIBRARIES) - message(FATAL_ERROR "Could not find libpthread") -endif () - -include(CheckSymbolExists) -list(APPEND CMAKE_REQUIRED_DEFINITIONS -D_POSIX_C_SOURCE=200809L) -list(APPEND CMAKE_REQUIRED_DEFINITIONS -D__XSI_VISIBLE=500) -list(APPEND CMAKE_REQUIRED_LIBRARIES pthread) -check_symbol_exists(pthread_mutexattr_setrobust pthread.h HAVE_ROBUST_MUTEX) - -if (HAVE_ROBUST_MUTEX) - set(DISABLE_ROBUST_MUTEXES FALSE CACHE BOOL "Disable robust mutex support") - if (NOT DISABLE_ROBUST_MUTEXES) - message(STATUS "Robust mutex support enabled") - set(HAVE_ROBUST_MUTEX TRUE) - else () - message(STATUS "Robust mutex support disabled by user") - unset(HAVE_ROBUST_MUTEX) - endif () -endif () - -find_library(FUSE_LIBRARIES fuse QUIET) -if (FUSE_LIBRARIES) - #FIXME: Check for version >= 2.6 - set(DISABLE_FUSE FALSE CACHE BOOL "Disable FUSE support") - if (NOT DISABLE_FUSE) - message(STATUS "FUSE support enabled") - set(FUSE_PREFIX "/tmp/ouroboros" CACHE STRING - "Mountpoint for RIB filesystem") - set(HAVE_FUSE TRUE CACHE INTERNAL "") - else () - message(STATUS "FUSE support disabled by user") - unset(HAVE_FUSE CACHE) - endif () -else () - message(STATUS "Install FUSE version > 2.6 to enable RIB access") -endif () - -if (NOT HAVE_FUSE) - set(FUSE_LIBRARIES "") - set(FUSE_INCLUDE_DIR "") -endif () - -mark_as_advanced(FUSE_LIBRARIES) - -find_library(LIBGCRYPT_LIBRARIES gcrypt QUIET) -if (LIBGCRYPT_LIBRARIES) - find_path(LIBGCRYPT_INCLUDE_DIR gcrypt.h - HINTS /usr/include /usr/local/include) - if (LIBGCRYPT_INCLUDE_DIR) - file(STRINGS ${LIBGCRYPT_INCLUDE_DIR}/gcrypt.h GCSTR - REGEX "^#define GCRYPT_VERSION ") - string(REGEX REPLACE "^#define GCRYPT_VERSION \"(.*)\".*$" "\\1" - GCVER "${GCSTR}") - if (NOT GCVER VERSION_LESS "1.7.0") - set(DISABLE_LIBGCRYPT FALSE CACHE BOOL "Disable libgcrypt support") - if (NOT DISABLE_LIBGCRYPT) - message(STATUS "libgcrypt support enabled") - set(HAVE_LIBGCRYPT TRUE CACHE INTERNAL "") - else () - message(STATUS "libgcrypt support disabled by user") - unset(HAVE_LIBGCRYPT CACHE) - endif() - else () - message(STATUS "Install version >= \"1.7.0\" to enable libgcrypt support " - "(found version \"${GCVER}\")") - endif() - endif () -endif () - -if (NOT HAVE_LIBGCRYPT) - set(LIBGCRYPT_LIBRARIES "") - set(LIBGCRYPT_INCLUDE_DIR "") -endif () - -find_package(OpenSSL QUIET) -if (OPENSSL_FOUND) - set(HAVE_OPENSSL_RNG TRUE) - if (OPENSSL_VERSION VERSION_LESS "1.1.0") - message(STATUS "Install version >= \"1.1.0\" to enable OpenSSL support " - "(found version \"${OPENSSL_VERSION}\")") - else () - set(DISABLE_OPENSSL FALSE CACHE BOOL "Disable OpenSSL support") - if (NOT DISABLE_OPENSSL) - message(STATUS "OpenSSL support enabled") - set(HAVE_OPENSSL TRUE) - else() - message(STATUS "OpenSSL support disabled") - unset(HAVE_OPENSSL) - endif() - endif () -endif () - -if (NOT HAVE_OPENSSL_RNG) - set(OPENSSL_INCLUDE_DIR "") - set(OPENSSL_LIBRARIES "") -endif () - -if (APPLE OR CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") - set(SYS_RND_HDR "") -else () - find_path(SYS_RND_HDR sys/random.h PATH /usr/include/ /usr/local/include/) - if (SYS_RND_HDR) - message(STATUS "Found sys/random.h in ${SYS_RND_HDR}") - set(HAVE_SYS_RANDOM TRUE) - else () - set(SYS_RND_HDR "") - unset(HAVE_SYS_RANDOM) - endif () -endif() - -if (NOT ((CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") OR APPLE OR - HAVE_SYS_RANDOM OR HAVE_OPENSSL_RNG OR HAVE_LIBGCRYPT)) - message(FATAL_ERROR "No secure random generator found, " - "please install libgcrypt (> 1.7.0) or OpenSSL") -endif () - -mark_as_advanced(LIBRT_LIBRARIES LIBPTHREAD_LIBRARIES - LIBGCRYPT_LIBRARIES OPENSSL_LIBRARIES SYS_RND_INCLUDE_DIR - LIBGCRYPT_INCLUDE_DIR SYS_RND_HDR) - -set(SHM_BUFFER_SIZE 4096 CACHE STRING - "Number of blocks in packet buffer, must be a power of 2") -set(SHM_RBUFF_SIZE 1024 CACHE STRING - "Number of blocks in rbuff buffer, must be a power of 2") -set(SYS_MAX_FLOWS 10240 CACHE STRING - "Maximum number of total flows for this system") -set(PROG_MAX_FLOWS 4096 CACHE STRING - "Maximum number of flows in an application") -set(PROG_RES_FDS 64 CACHE STRING - "Number of reserved flow descriptors per application") -set(PROG_MAX_FQUEUES 32 CACHE STRING - "Maximum number of flow sets per application") -set(DU_BUFF_HEADSPACE 256 CACHE STRING - "Bytes of headspace to reserve for future headers") -set(DU_BUFF_TAILSPACE 32 CACHE STRING - "Bytes of tailspace to reserve for future tails") -if (NOT APPLE) - set(PTHREAD_COND_CLOCK "CLOCK_MONOTONIC" CACHE STRING - "Clock to use for condition variable timing") -else () - set(PTHREAD_COND_CLOCK "CLOCK_REALTIME" CACHE INTERNAL - "Clock to use for condition variable timing") -endif () -set(SOCKET_TIMEOUT 1000 CACHE STRING - "Default timeout for responses from IPCPs (ms)") -set(SHM_PREFIX "ouroboros" CACHE STRING - "String to prepend to POSIX shared memory filenames") -set(SHM_RBUFF_PREFIX "/${SHM_PREFIX}.rbuff." CACHE INTERNAL - "Prefix for rbuff POSIX shared memory filenames") -set(SHM_LOCKFILE_NAME "/${SHM_PREFIX}.lockfile" CACHE INTERNAL - "Filename for the POSIX shared memory lockfile") -set(SHM_FLOW_SET_PREFIX "/${SHM_PREFIX}.set." CACHE INTERNAL - "Prefix for the POSIX shared memory flow set") -set(SHM_RDRB_NAME "/${SHM_PREFIX}.rdrb" CACHE INTERNAL - "Name for the main POSIX shared memory buffer") -set(SHM_RDRB_BLOCK_SIZE "sysconf(_SC_PAGESIZE)" CACHE STRING - "Packet buffer block size, multiple of pagesize for performance") -set(SHM_RDRB_MULTI_BLOCK TRUE CACHE BOOL - "Packet buffer multiblock packet support") -set(SHM_RBUFF_LOCKLESS FALSE CACHE BOOL - "Enable shared memory lockless rbuff support") -set(QOS_DISABLE_CRC TRUE CACHE BOOL - "Ignores ber setting on all QoS cubes") -set(DELTA_T_MPL 60 CACHE STRING - "Maximum packet lifetime (s)") -set(DELTA_T_ACK 10 CACHE STRING - "Maximum time to acknowledge a packet (s)") -set(DELTA_T_RTX 120 CACHE STRING - "Maximum time to retransmit a packet (s)") -set(DELTA_T_ACK_DELAY 10 CACHE STRING - "Maximum time to wait before acknowledging a packet (ms)") -set(FRCT_REORDER_QUEUE_SIZE 256 CACHE STRING - "Size of the reordering queue, must be a power of 2") -set(FRCT_START_WINDOW 64 CACHE STRING - "Start window, must be a power of 2") -set(FRCT_RTO_MIN 250 CACHE STRING - "Minimum Retransmission Timeout (RTO) for FRCT (us)") -set(FRCT_TICK_TIME 5000 CACHE STRING - "Tick time for FRCT activity (retransmission, acknowledgments) (us)") -set(RXM_BUFFER_ON_HEAP FALSE CACHE BOOL - "Store packets for retransmission on the heap instead of in packet buffer") -set(RXM_BLOCKING TRUE CACHE BOOL - "Use blocking writes for retransmission") -set(RXM_MIN_RESOLUTION 20 CACHE STRING - "Minimum retransmission delay (ns), as a power to 2") -set(RXM_WHEEL_MULTIPLIER 4 CACHE STRING - "Factor for retransmission wheel levels as a power to 2") -set(RXM_WHEEL_LEVELS 3 CACHE STRING - "Number of levels in the retransmission wheel") -set(RXM_WHEEL_SLOTS_PER_LEVEL 256 CACHE STRING - "Number of slots per level in the retransmission wheel, must be a power of 2") -set(ACK_WHEEL_SLOTS 128 CACHE STRING - "Number of slots in the acknowledgment wheel, must be a power of 2") -set(ACK_WHEEL_RESOLUTION 20 CACHE STRING - "Minimum acknowledgment delay (ns), as a power to 2") - -set(SOURCE_FILES_DEV - # Add source files here - cacep.c - dev.c - ) - -set(SOURCE_FILES_IRM - irm.c -) +# Ouroboros libraries build configuration +# Configuration options are in cmake/config/lib.cmake + +protobuf_generate_c(MODEL_PROTO_SRCS MODEL_PROTO_HDRS + "${CMAKE_CURRENT_SOURCE_DIR}/pb/model.proto") +protobuf_generate_c(IPCP_CONFIG_PROTO_SRCS IPCP_CONFIG_PROTO_HDRS + "${CMAKE_CURRENT_SOURCE_DIR}/pb/ipcp_config.proto") +protobuf_generate_c(ENROLL_PROTO_SRCS ENROLL_PROTO_HDRS + "${CMAKE_CURRENT_SOURCE_DIR}/pb/enroll.proto") +protobuf_generate_c(CEP_PROTO_SRCS CEP_PROTO_HDRS + "${CMAKE_CURRENT_SOURCE_DIR}/pb/cep.proto") +protobuf_generate_c(IRM_PROTO_SRCS IRM_PROTO_HDRS + "${CMAKE_CURRENT_SOURCE_DIR}/pb/irm.proto") +protobuf_generate_c(IPCP_PROTO_SRCS IPCP_PROTO_HDRS + "${CMAKE_CURRENT_SOURCE_DIR}/pb/ipcp.proto") set(SOURCE_FILES_COMMON bitmap.c btree.c - crc32.c + crc/crc8.c + crc/crc16.c + crc/crc32.c + crc/crc64.c + crypt.c hash.c - list.c lockfile.c logs.c md5.c notifier.c + protobuf.c qoscube.c random.c rib.c + serdes-irm.c + serdes-oep.c sha3.c - shm_flow_set.c - shm_rbuff.c - shm_rdrbuff.c + ssm/flow_set.c + ssm/rbuff.c + ssm/pool.c sockets.c tpm.c + tw.c utils.c ) -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/config.h.in" - "${CMAKE_CURRENT_BINARY_DIR}/config.h" @ONLY) +if(HAVE_OPENSSL) + list(APPEND SOURCE_FILES_COMMON crypt/openssl.c) +endif() + +add_library(ouroboros-common SHARED + ${SOURCE_FILES_COMMON} + ${IRM_PROTO_SRCS} + ${IPCP_PROTO_SRCS} + ${IPCP_CONFIG_PROTO_SRCS} + ${MODEL_PROTO_SRCS} + ${ENROLL_PROTO_SRCS}) + +set_target_properties(ouroboros-common PROPERTIES + VERSION ${PACKAGE_VERSION} + SOVERSION ${PACKAGE_VERSION_MAJOR}.${PACKAGE_VERSION_MINOR}) + +ouroboros_target_debug_definitions(ouroboros-common) + +target_include_directories(ouroboros-common + PUBLIC + $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include> + $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include> + $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}> + $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}> + $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> + PRIVATE + ${SYS_RND_HDR} + ${APPLE_INCLUDE_DIRS}) + +target_link_libraries(ouroboros-common + PRIVATE + ${LIBRT_LIBRARIES} + Threads::Threads + PUBLIC + ProtobufC::ProtobufC) + +if(HAVE_OPENSSL) + target_link_libraries(ouroboros-common PUBLIC OpenSSL::Crypto) +endif() + +if(HAVE_LIBGCRYPT) + target_link_libraries(ouroboros-common PUBLIC Gcrypt::Gcrypt) +endif() + +if(HAVE_FUSE) + target_link_libraries(ouroboros-common PRIVATE Fuse::Fuse) +endif() + +install(TARGETS ouroboros-common + EXPORT OuroborosTargets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -add_library(ouroboros-common SHARED ${SOURCE_FILES_COMMON} ${IRM_PROTO_SRCS} - ${IPCP_PROTO_SRCS} ${LAYER_CONFIG_PROTO_SRCS} ${QOSSPEC_PROTO_SRCS}) +set(SOURCE_FILES_DEV + cep.c + dev.c +) + +add_library(ouroboros-dev SHARED + ${SOURCE_FILES_DEV} + ${CEP_PROTO_SRCS}) + +set_target_properties(ouroboros-dev PROPERTIES + VERSION ${PACKAGE_VERSION} + SOVERSION ${PACKAGE_VERSION_MAJOR}.${PACKAGE_VERSION_MINOR}) + +ouroboros_target_debug_definitions(ouroboros-dev) -add_library(ouroboros-dev SHARED ${SOURCE_FILES_DEV} ${CACEP_PROTO_SRCS}) +target_include_directories(ouroboros-dev + PUBLIC + $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_BINARY_DIR} + ${SYS_RND_HDR}) -add_library(ouroboros-irm SHARED ${SOURCE_FILES_IRM}) +target_link_libraries(ouroboros-dev PUBLIC ouroboros-common) -include(AddCompileFlags) -if (CMAKE_BUILD_TYPE MATCHES "Debug*") - add_compile_flags(ouroboros-common -DCONFIG_OUROBOROS_DEBUG) - add_compile_flags(ouroboros-dev -DCONFIG_OUROBOROS_DEBUG) - add_compile_flags(ouroboros-irm -DCONFIG_OUROBOROS_DEBUG) -endif () +install(TARGETS ouroboros-dev + EXPORT OuroborosTargets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -target_link_libraries(ouroboros-common ${LIBRT_LIBRARIES} - ${LIBPTHREAD_LIBRARIES} ${PROTOBUF_C_LIBRARY} ${OPENSSL_LIBRARIES} - ${LIBGCRYPT_LIBRARIES} ${FUSE_LIBRARIES}) +add_library(ouroboros-irm SHARED irm.c) -target_link_libraries(ouroboros-dev ouroboros-common) -target_link_libraries(ouroboros-irm ouroboros-common) +set_target_properties(ouroboros-irm PROPERTIES + VERSION ${PACKAGE_VERSION} + SOVERSION ${PACKAGE_VERSION_MAJOR}.${PACKAGE_VERSION_MINOR}) -install(TARGETS ouroboros-common LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -install(TARGETS ouroboros-dev LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -install(TARGETS ouroboros-irm LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) +ouroboros_target_debug_definitions(ouroboros-irm) -target_include_directories(ouroboros-common PUBLIC ${CMAKE_CURRENT_BINARY_DIR} - ${SYS_RND_HDR} ${LIBGCRYPT_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR}) +target_include_directories(ouroboros-irm + PUBLIC + $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/include> + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_BINARY_DIR} + ${SYS_RND_HDR}) -target_include_directories(ouroboros-dev PUBLIC ${CMAKE_CURRENT_BINARY_DIR} - ${SYS_RND_HDR} ${LIBGCRYPT_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR}) +target_link_libraries(ouroboros-irm PUBLIC ouroboros-common) -target_include_directories(ouroboros-irm PUBLIC ${CMAKE_CURRENT_BINARY_DIR} - ${SYS_RND_HDR} ${LIBGCRYPT_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR}) +install(TARGETS ouroboros-irm + EXPORT OuroborosTargets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -add_subdirectory(tests) +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/config.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/config.h" @ONLY) + +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/ssm/ssm.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/ssm.h" @ONLY) + +if(BUILD_TESTS) + add_subdirectory(tests) + add_subdirectory(crc/tests) + add_subdirectory(ssm/tests) +endif() diff --git a/src/lib/bitmap.c b/src/lib/bitmap.c index b76960da..05cf4030 100644 --- a/src/lib/bitmap.c +++ b/src/lib/bitmap.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Bitmap implementation * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License diff --git a/src/lib/btree.c b/src/lib/btree.c index 86d1299b..37ec5e52 100644 --- a/src/lib/btree.c +++ b/src/lib/btree.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * B-trees * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License diff --git a/src/lib/cacep.c b/src/lib/cep.c index 9a889530..e953e2d9 100644 --- a/src/lib/cacep.c +++ b/src/lib/cep.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * - * The Common Application Connection Establishment Protocol + * The Ouroboros Connection Establishment Protocol * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -22,35 +22,35 @@ #define _POSIX_C_SOURCE 199309L -#include <ouroboros/cacep.h> +#include <ouroboros/cep.h> #include <ouroboros/dev.h> #include <ouroboros/errno.h> #include <stdlib.h> #include <string.h> -#include "cacep.pb-c.h" -typedef CacepMsg cacep_msg_t; +#include "cep.pb-c.h" +typedef CepMsg cep_msg_t; #define BUF_SIZE 128 static int read_msg(int fd, struct conn_info * info) { - uint8_t buf[BUF_SIZE]; - cacep_msg_t * msg; - ssize_t len; + uint8_t buf[BUF_SIZE]; + cep_msg_t * msg; + ssize_t len; len = flow_read(fd, buf, BUF_SIZE); if (len < 0) - return -1; + return (int) len; - msg = cacep_msg__unpack(NULL, len, buf); + msg = cep_msg__unpack(NULL, len, buf); if (msg == NULL) return -1; - if (strlen(msg->comp_name) > CACEP_BUF_STRLEN) { - cacep_msg__free_unpacked(msg, NULL); + if (strlen(msg->comp_name) > OCEP_BUF_STRLEN) { + cep_msg__free_unpacked(msg, NULL); return -1; } @@ -61,7 +61,7 @@ static int read_msg(int fd, info->pref_syntax = msg->pref_syntax; info->addr = msg->address; - cacep_msg__free_unpacked(msg, NULL); + cep_msg__free_unpacked(msg, NULL); return 0; } @@ -69,9 +69,9 @@ static int read_msg(int fd, static int send_msg(int fd, const struct conn_info * info) { - cacep_msg_t msg = CACEP_MSG__INIT; - uint8_t * data = NULL; - size_t len = 0; + cep_msg_t msg = CEP_MSG__INIT; + uint8_t * data = NULL; + size_t len = 0; msg.comp_name = (char *) info->comp_name; msg.protocol = (char *) info->protocol; @@ -81,7 +81,7 @@ static int send_msg(int fd, if (msg.pref_syntax < 0) return -1; - len = cacep_msg__get_packed_size(&msg); + len = cep_msg__get_packed_size(&msg); if (len == 0) return -1; @@ -89,7 +89,7 @@ static int send_msg(int fd, if (data == NULL) return -ENOMEM; - cacep_msg__pack(&msg, data); + cep_msg__pack(&msg, data); if (flow_write(fd, data, len) < 0) { free(data); @@ -101,26 +101,20 @@ static int send_msg(int fd, return 0; } -int cacep_snd(int fd, - const struct conn_info * in) +int cep_snd(int fd, + const struct conn_info * in) { if (in == NULL) return -EINVAL; - if (send_msg(fd, in)) - return -1; - - return 0; + return send_msg(fd, in); } -int cacep_rcv(int fd, - struct conn_info * out) +int cep_rcv(int fd, + struct conn_info * out) { if (out == NULL) return -EINVAL; - if (read_msg(fd, out)) - return -1; - - return 0; + return read_msg(fd, out); } diff --git a/src/lib/config.h.in b/src/lib/config.h.in index 36221b8c..7124a974 100644 --- a/src/lib/config.h.in +++ b/src/lib/config.h.in @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Ouroboros library configuration * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -20,28 +20,39 @@ * Foundation, Inc., http://www.fsf.org/about/contact/. */ +#ifndef MILLION +#define MILLION 1000000LL +#endif + +#ifndef BILLION +#define BILLION 1000000000LL +#endif + #cmakedefine HAVE_SYS_RANDOM +#cmakedefine HAVE_EXPLICIT_BZERO #cmakedefine HAVE_LIBGCRYPT #cmakedefine HAVE_OPENSSL - #ifdef HAVE_OPENSSL +#cmakedefine HAVE_OPENSSL_ML_KEM +#cmakedefine HAVE_OPENSSL_ML_DSA +#cmakedefine HAVE_OPENSSL_SLH_DSA #define HAVE_ENCRYPTION +#define SECMEM_GUARD @SECMEM_GUARD@ #endif +#define PROC_SECMEM_MAX @PROC_SECMEM_MAX@ + +#define SYS_MAX_FLOWS @SYS_MAX_FLOWS@ -#define SYS_MAX_FLOWS @SYS_MAX_FLOWS@ +#cmakedefine QOS_DISABLE_CRC +#cmakedefine HAVE_OPENSSL_RNG +#cmakedefine HAVE_PCLMUL +#cmakedefine HAVE_PMULL -#cmakedefine SHM_RBUFF_LOCKLESS -#cmakedefine SHM_RDRB_MULTI_BLOCK -#cmakedefine QOS_DISABLE_CRC -#cmakedefine HAVE_OPENSSL_RNG +#define SHM_LOCKFILE_NAME "@SHM_LOCKFILE_NAME@" +#define FLOW_ALLOC_TIMEOUT @FLOW_ALLOC_TIMEOUT@ -#define SHM_RBUFF_PREFIX "@SHM_RBUFF_PREFIX@" -#define SHM_LOCKFILE_NAME "@SHM_LOCKFILE_NAME@" -#define SHM_FLOW_SET_PREFIX "@SHM_FLOW_SET_PREFIX@" -#define SHM_RDRB_NAME "@SHM_RDRB_NAME@" -#define SHM_RDRB_BLOCK_SIZE @SHM_RDRB_BLOCK_SIZE@ -#define SHM_BUFFER_SIZE @SHM_BUFFER_SIZE@ -#define SHM_RBUFF_SIZE @SHM_RBUFF_SIZE@ +#define TPM_DEBUG_REPORT_INTERVAL @TPM_DEBUG_REPORT_INTERVAL@ +#define TPM_DEBUG_ABORT_TIMEOUT @TPM_DEBUG_ABORT_TIMEOUT@ #if defined(__linux__) || (defined(__MACH__) && !defined(__APPLE__)) /* Avoid a bug in robust mutex implementation of glibc 2.25 */ @@ -56,34 +67,31 @@ #cmakedefine HAVE_FUSE #ifdef HAVE_FUSE #define FUSE_PREFIX "@FUSE_PREFIX@" +#cmakedefine PROC_FLOW_STATS #endif -#define PTHREAD_COND_CLOCK @PTHREAD_COND_CLOCK@ +#cmakedefine FRCT_DEBUG_STDOUT -#define PROG_MAX_FLOWS @PROG_MAX_FLOWS@ -#define PROG_RES_FDS @PROG_RES_FDS@ -#define PROG_MAX_FQUEUES @PROG_MAX_FQUEUES@ +#define PTHREAD_COND_CLOCK @PTHREAD_COND_CLOCK@ -#define DU_BUFF_HEADSPACE @DU_BUFF_HEADSPACE@ -#define DU_BUFF_TAILSPACE @DU_BUFF_TAILSPACE@ +#define PROC_MAX_FLOWS @PROC_MAX_FLOWS@ +#define PROC_RES_FDS @PROC_RES_FDS@ +#define PROC_MAX_FQUEUES @PROC_MAX_FQUEUES@ /* Default Delta-t parameters */ -#define DELT_MPL (@DELTA_T_MPL@ * BILLION) /* ns */ -#define DELT_A (@DELTA_T_ACK@ * BILLION) /* ns */ -#define DELT_R (@DELTA_T_RTX@ * BILLION) /* ns */ - -#define DELT_ACK (@DELTA_T_ACK_DELAY@ * MILLION) /* ns */ +#cmakedefine FRCT_LINUX_RTT_ESTIMATOR +#define DELT_A (@DELTA_T_ACK@) /* ms */ +#define DELT_R (@DELTA_T_RTX@) /* ms */ #define RQ_SIZE (@FRCT_REORDER_QUEUE_SIZE@) #define START_WINDOW (@FRCT_START_WINDOW@) #define RTO_MIN (@FRCT_RTO_MIN@ * 1000) +#define RTO_DIV (@FRCT_RTO_INC_FACTOR@) +#define MDEV_MUL (@FRCT_RTO_MDEV_MULTIPLIER@) #define TICTIME (@FRCT_TICK_TIME@ * 1000) /* ns */ /* Retransmission tuning */ -#cmakedefine RXM_BUFFER_ON_HEAP -#cmakedefine RXM_BLOCKING - #define RXMQ_RES (@RXM_MIN_RESOLUTION@) /* 2^N ns */ #define RXMQ_BUMP (@RXM_WHEEL_MULTIPLIER@) #define RXMQ_LVLS (@RXM_WHEEL_LEVELS@) @@ -91,3 +99,5 @@ #define ACKQ_SLOTS (@ACK_WHEEL_SLOTS@) #define ACKQ_RES (@ACK_WHEEL_RESOLUTION@) /* 2^N ns */ + +#define KEY_ROTATION_BIT (@KEY_ROTATION_BIT@) /* Bit for key rotation */ diff --git a/src/lib/crc/crc16.c b/src/lib/crc/crc16.c new file mode 100644 index 00000000..9dc59429 --- /dev/null +++ b/src/lib/crc/crc16.c @@ -0,0 +1,61 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * 16-bit Cyclic Redundancy Check (CCITT-FALSE variant) + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +/* + * CRC-16/CCITT-FALSE (reveng catalog, alias CRC-16/IBM-3740): + * poly = 0x1021 + * init = 0xffff + * refin = false + * refout = false + * xorout = 0x0000 + * check = crc16_ccitt_false("123456789") == 0x29b1 + */ + +#include "config.h" + +#include <ouroboros/crc16.h> + +/* Bit-by-bit MSB-first CRC. */ +void crc16_ccitt_false(uint16_t * crc, + const void * buf, + size_t len) +{ + const uint8_t * p; + uint16_t c; + size_t n; + int i; + + p = (const uint8_t *) buf; + c = *crc ^ 0xffff; + + for (n = 0; n < len; n++) { + c ^= ((uint16_t) p[n]) << 8; + for (i = 0; i < 8; i++) { + if (c & 0x8000) + c = (uint16_t) ((c << 1) ^ 0x1021); + else + c = (uint16_t) (c << 1); + } + } + + *crc = c; +} diff --git a/src/lib/crc32.c b/src/lib/crc/crc32.c index 1b8a5bc8..0fdb62b1 100644 --- a/src/lib/crc32.c +++ b/src/lib/crc/crc32.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * 32-bit Cyclic Redundancy Check * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License diff --git a/src/lib/crc/crc64.c b/src/lib/crc/crc64.c new file mode 100644 index 00000000..1b6fb5f6 --- /dev/null +++ b/src/lib/crc/crc64.c @@ -0,0 +1,363 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * 64-bit Cyclic Redundancy Check (NVMe variant) + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +/* + * CRC-64/NVMe (reveng catalog): + * poly = 0xad93d23594c93659 + * init = 0xffffffffffffffff + * refin = true + * refout = true + * xorout = 0xffffffffffffffff + * check = crc64_nvme("123456789") == 0xae8b14860a799888 + */ + +#include "config.h" + +#include <ouroboros/crc64.h> + +/* + * Reflected CRC-64/NVMe table. Polynomial in reflected form: + * 0x9a6c9329ac4bc9b5 (bitrev of 0xad93d23594c93659). + */ +static const uint64_t crc64_nvme_tab[256] = { + 0x0000000000000000ULL, 0x7f6ef0c830358979ULL, + 0xfedde190606b12f2ULL, 0x81b31158505e9b8bULL, + 0xc962e5739841b68fULL, 0xb60c15bba8743ff6ULL, + 0x37bf04e3f82aa47dULL, 0x48d1f42bc81f2d04ULL, + 0xa61cecb46814fe75ULL, 0xd9721c7c5821770cULL, + 0x58c10d24087fec87ULL, 0x27affdec384a65feULL, + 0x6f7e09c7f05548faULL, 0x1010f90fc060c183ULL, + 0x91a3e857903e5a08ULL, 0xeecd189fa00bd371ULL, + 0x78e0ff3b88be6f81ULL, 0x078e0ff3b88be6f8ULL, + 0x863d1eabe8d57d73ULL, 0xf953ee63d8e0f40aULL, + 0xb1821a4810ffd90eULL, 0xceecea8020ca5077ULL, + 0x4f5ffbd87094cbfcULL, 0x30310b1040a14285ULL, + 0xdefc138fe0aa91f4ULL, 0xa192e347d09f188dULL, + 0x2021f21f80c18306ULL, 0x5f4f02d7b0f40a7fULL, + 0x179ef6fc78eb277bULL, 0x68f0063448deae02ULL, + 0xe943176c18803589ULL, 0x962de7a428b5bcf0ULL, + 0xf1c1fe77117cdf02ULL, 0x8eaf0ebf2149567bULL, + 0x0f1c1fe77117cdf0ULL, 0x7072ef2f41224489ULL, + 0x38a31b04893d698dULL, 0x47cdebccb908e0f4ULL, + 0xc67efa94e9567b7fULL, 0xb9100a5cd963f206ULL, + 0x57dd12c379682177ULL, 0x28b3e20b495da80eULL, + 0xa900f35319033385ULL, 0xd66e039b2936bafcULL, + 0x9ebff7b0e12997f8ULL, 0xe1d10778d11c1e81ULL, + 0x606216208142850aULL, 0x1f0ce6e8b1770c73ULL, + 0x8921014c99c2b083ULL, 0xf64ff184a9f739faULL, + 0x77fce0dcf9a9a271ULL, 0x08921014c99c2b08ULL, + 0x4043e43f0183060cULL, 0x3f2d14f731b68f75ULL, + 0xbe9e05af61e814feULL, 0xc1f0f56751dd9d87ULL, + 0x2f3dedf8f1d64ef6ULL, 0x50531d30c1e3c78fULL, + 0xd1e00c6891bd5c04ULL, 0xae8efca0a188d57dULL, + 0xe65f088b6997f879ULL, 0x9931f84359a27100ULL, + 0x1882e91b09fcea8bULL, 0x67ec19d339c963f2ULL, + 0xd75adabd7a6e2d6fULL, 0xa8342a754a5ba416ULL, + 0x29873b2d1a053f9dULL, 0x56e9cbe52a30b6e4ULL, + 0x1e383fcee22f9be0ULL, 0x6156cf06d21a1299ULL, + 0xe0e5de5e82448912ULL, 0x9f8b2e96b271006bULL, + 0x71463609127ad31aULL, 0x0e28c6c1224f5a63ULL, + 0x8f9bd7997211c1e8ULL, 0xf0f5275142244891ULL, + 0xb824d37a8a3b6595ULL, 0xc74a23b2ba0eececULL, + 0x46f932eaea507767ULL, 0x3997c222da65fe1eULL, + 0xafba2586f2d042eeULL, 0xd0d4d54ec2e5cb97ULL, + 0x5167c41692bb501cULL, 0x2e0934dea28ed965ULL, + 0x66d8c0f56a91f461ULL, 0x19b6303d5aa47d18ULL, + 0x980521650afae693ULL, 0xe76bd1ad3acf6feaULL, + 0x09a6c9329ac4bc9bULL, 0x76c839faaaf135e2ULL, + 0xf77b28a2faafae69ULL, 0x8815d86aca9a2710ULL, + 0xc0c42c4102850a14ULL, 0xbfaadc8932b0836dULL, + 0x3e19cdd162ee18e6ULL, 0x41773d1952db919fULL, + 0x269b24ca6b12f26dULL, 0x59f5d4025b277b14ULL, + 0xd846c55a0b79e09fULL, 0xa72835923b4c69e6ULL, + 0xeff9c1b9f35344e2ULL, 0x90973171c366cd9bULL, + 0x1124202993385610ULL, 0x6e4ad0e1a30ddf69ULL, + 0x8087c87e03060c18ULL, 0xffe938b633338561ULL, + 0x7e5a29ee636d1eeaULL, 0x0134d92653589793ULL, + 0x49e52d0d9b47ba97ULL, 0x368bddc5ab7233eeULL, + 0xb738cc9dfb2ca865ULL, 0xc8563c55cb19211cULL, + 0x5e7bdbf1e3ac9decULL, 0x21152b39d3991495ULL, + 0xa0a63a6183c78f1eULL, 0xdfc8caa9b3f20667ULL, + 0x97193e827bed2b63ULL, 0xe877ce4a4bd8a21aULL, + 0x69c4df121b863991ULL, 0x16aa2fda2bb3b0e8ULL, + 0xf86737458bb86399ULL, 0x8709c78dbb8deae0ULL, + 0x06bad6d5ebd3716bULL, 0x79d4261ddbe6f812ULL, + 0x3105d23613f9d516ULL, 0x4e6b22fe23cc5c6fULL, + 0xcfd833a67392c7e4ULL, 0xb0b6c36e43a74e9dULL, + 0x9a6c9329ac4bc9b5ULL, 0xe50263e19c7e40ccULL, + 0x64b172b9cc20db47ULL, 0x1bdf8271fc15523eULL, + 0x530e765a340a7f3aULL, 0x2c608692043ff643ULL, + 0xadd397ca54616dc8ULL, 0xd2bd67026454e4b1ULL, + 0x3c707f9dc45f37c0ULL, 0x431e8f55f46abeb9ULL, + 0xc2ad9e0da4342532ULL, 0xbdc36ec59401ac4bULL, + 0xf5129aee5c1e814fULL, 0x8a7c6a266c2b0836ULL, + 0x0bcf7b7e3c7593bdULL, 0x74a18bb60c401ac4ULL, + 0xe28c6c1224f5a634ULL, 0x9de29cda14c02f4dULL, + 0x1c518d82449eb4c6ULL, 0x633f7d4a74ab3dbfULL, + 0x2bee8961bcb410bbULL, 0x548079a98c8199c2ULL, + 0xd53368f1dcdf0249ULL, 0xaa5d9839ecea8b30ULL, + 0x449080a64ce15841ULL, 0x3bfe706e7cd4d138ULL, + 0xba4d61362c8a4ab3ULL, 0xc52391fe1cbfc3caULL, + 0x8df265d5d4a0eeceULL, 0xf29c951de49567b7ULL, + 0x732f8445b4cbfc3cULL, 0x0c41748d84fe7545ULL, + 0x6bad6d5ebd3716b7ULL, 0x14c39d968d029fceULL, + 0x95708ccedd5c0445ULL, 0xea1e7c06ed698d3cULL, + 0xa2cf882d2576a038ULL, 0xdda178e515432941ULL, + 0x5c1269bd451db2caULL, 0x237c997575283bb3ULL, + 0xcdb181ead523e8c2ULL, 0xb2df7122e51661bbULL, + 0x336c607ab548fa30ULL, 0x4c0290b2857d7349ULL, + 0x04d364994d625e4dULL, 0x7bbd94517d57d734ULL, + 0xfa0e85092d094cbfULL, 0x856075c11d3cc5c6ULL, + 0x134d926535897936ULL, 0x6c2362ad05bcf04fULL, + 0xed9073f555e26bc4ULL, 0x92fe833d65d7e2bdULL, + 0xda2f7716adc8cfb9ULL, 0xa54187de9dfd46c0ULL, + 0x24f29686cda3dd4bULL, 0x5b9c664efd965432ULL, + 0xb5517ed15d9d8743ULL, 0xca3f8e196da80e3aULL, + 0x4b8c9f413df695b1ULL, 0x34e26f890dc31cc8ULL, + 0x7c339ba2c5dc31ccULL, 0x035d6b6af5e9b8b5ULL, + 0x82ee7a32a5b7233eULL, 0xfd808afa9582aa47ULL, + 0x4d364994d625e4daULL, 0x3258b95ce6106da3ULL, + 0xb3eba804b64ef628ULL, 0xcc8558cc867b7f51ULL, + 0x8454ace74e645255ULL, 0xfb3a5c2f7e51db2cULL, + 0x7a894d772e0f40a7ULL, 0x05e7bdbf1e3ac9deULL, + 0xeb2aa520be311aafULL, 0x944455e88e0493d6ULL, + 0x15f744b0de5a085dULL, 0x6a99b478ee6f8124ULL, + 0x224840532670ac20ULL, 0x5d26b09b16452559ULL, + 0xdc95a1c3461bbed2ULL, 0xa3fb510b762e37abULL, + 0x35d6b6af5e9b8b5bULL, 0x4ab846676eae0222ULL, + 0xcb0b573f3ef099a9ULL, 0xb465a7f70ec510d0ULL, + 0xfcb453dcc6da3dd4ULL, 0x83daa314f6efb4adULL, + 0x0269b24ca6b12f26ULL, 0x7d0742849684a65fULL, + 0x93ca5a1b368f752eULL, 0xeca4aad306bafc57ULL, + 0x6d17bb8b56e467dcULL, 0x12794b4366d1eea5ULL, + 0x5aa8bf68aecec3a1ULL, 0x25c64fa09efb4ad8ULL, + 0xa4755ef8cea5d153ULL, 0xdb1bae30fe90582aULL, + 0xbcf7b7e3c7593bd8ULL, 0xc399472bf76cb2a1ULL, + 0x422a5673a732292aULL, 0x3d44a6bb9707a053ULL, + 0x759552905f188d57ULL, 0x0afba2586f2d042eULL, + 0x8b48b3003f739fa5ULL, 0xf42643c80f4616dcULL, + 0x1aeb5b57af4dc5adULL, 0x6585ab9f9f784cd4ULL, + 0xe436bac7cf26d75fULL, 0x9b584a0fff135e26ULL, + 0xd389be24370c7322ULL, 0xace74eec0739fa5bULL, + 0x2d545fb4576761d0ULL, 0x523aaf7c6752e8a9ULL, + 0xc41748d84fe75459ULL, 0xbb79b8107fd2dd20ULL, + 0x3acaa9482f8c46abULL, 0x45a459801fb9cfd2ULL, + 0x0d75adabd7a6e2d6ULL, 0x721b5d63e7936bafULL, + 0xf3a84c3bb7cdf024ULL, 0x8cc6bcf387f8795dULL, + 0x620ba46c27f3aa2cULL, 0x1d6554a417c62355ULL, + 0x9cd645fc4798b8deULL, 0xe3b8b53477ad31a7ULL, + 0xab69411fbfb21ca3ULL, 0xd407b1d78f8795daULL, + 0x55b4a08fdfd90e51ULL, 0x2ada5047efec8728ULL +}; + +static __inline__ uint64_t crc64_nvme_step(uint64_t c, + const uint8_t * p, + size_t len) +{ + size_t n; + + for (n = 0; n < len; n++) + c = crc64_nvme_tab[(c ^ p[n]) & 0xff] ^ (c >> 8); + + return c; +} + +void crc64_nvme_table(uint64_t * crc, + const void * buf, + size_t len) +{ + uint64_t c; + + c = crc64_nvme_step(*crc ^ UINT64_MAX, + (const uint8_t *) buf, len); + + *crc = c ^ UINT64_MAX; +} + +#ifdef HAVE_PCLMUL + +#include <smmintrin.h> +#include <wmmintrin.h> + +/* + * Fold-by-16 constants for reflected CRC-64/NVMe. Properties of the + * polynomial; identical between the PCLMUL and PMULL backends. + * k3 = bitrev64(x^(128+64) mod P) << 1 + * k4 = bitrev64(x^(128+0) mod P) << 1 + */ +static const uint64_t k3_clmul = 0xeadc41fd2ba3d420ULL; +static const uint64_t k4_clmul = 0x21e9761e252621acULL; + +__attribute__((target("pclmul,sse4.1"))) +static __m128i fold16(__m128i x, + __m128i k) +{ + __m128i lo; + __m128i hi; + + lo = _mm_clmulepi64_si128(x, k, 0x00); + hi = _mm_clmulepi64_si128(x, k, 0x11); + return _mm_xor_si128(lo, hi); +} + +/* + * Fold-by-16 over 16-byte chunks; the 128-bit folded state is then + * emitted as 16 little-endian bytes and run through the byte-table + * loop together with any tail (<=15 bytes). The 16-byte minimum on + * the bulk loop is why the short-input path uses the table directly. + */ +__attribute__((target("pclmul,sse4.1"))) +static void crc64_nvme_clmul(uint64_t * crc, + const void * buf, + size_t len) +{ + const uint8_t * p; + uint64_t seed; + uint64_t c; + size_t off; + __m128i x; + __m128i k; + uint8_t post[16]; + + p = (const uint8_t *) buf; + seed = *crc; + + if (len < 16) { + c = crc64_nvme_step(seed ^ UINT64_MAX, p, len); + *crc = c ^ UINT64_MAX; + return; + } + + x = _mm_loadu_si128((const __m128i *) p); + x = _mm_xor_si128(x, _mm_cvtsi64_si128((int64_t) + (seed ^ UINT64_MAX))); + + k = _mm_set_epi64x((int64_t) k4_clmul, (int64_t) k3_clmul); + + off = 16; + while (off + 16 <= len) { + __m128i d; + + d = _mm_loadu_si128((const __m128i *) (p + off)); + x = _mm_xor_si128(fold16(x, k), d); + off += 16; + } + + _mm_storeu_si128((__m128i *) post, x); + + c = crc64_nvme_step(0, post, 16); + c = crc64_nvme_step(c, p + off, len - off); + + *crc = c ^ UINT64_MAX; +} + +#endif /* HAVE_PCLMUL */ + +#ifdef HAVE_PMULL + +#include <arm_neon.h> + +/* Same fold-by-16 constants as the PCLMUL path (poly properties). */ +static const uint64_t k3_pmull = 0xeadc41fd2ba3d420ULL; +static const uint64_t k4_pmull = 0x21e9761e252621acULL; + +__attribute__((target("+crypto"))) +static uint64x2_t fold16_pmull(uint64x2_t x, + uint64x2_t k) +{ + poly64x2_t xp; + poly64x2_t kp; + uint64x2_t lo; + uint64x2_t hi; + + xp = vreinterpretq_p64_u64(x); + kp = vreinterpretq_p64_u64(k); + lo = vreinterpretq_u64_p128( + vmull_p64((poly64_t) vgetq_lane_u64(x, 0), + (poly64_t) vgetq_lane_u64(k, 0))); + hi = vreinterpretq_u64_p128(vmull_high_p64(xp, kp)); + return veorq_u64(lo, hi); +} + +__attribute__((target("+crypto"))) +static void crc64_nvme_pmull(uint64_t * crc, + const void * buf, + size_t len) +{ + const uint8_t * p; + uint64_t seed; + uint64_t c; + size_t off; + uint64x2_t x; + uint64x2_t k; + uint64_t seed_lane[2]; + uint64_t k_lanes[2]; + uint8_t post[16]; + + p = (const uint8_t *) buf; + seed = *crc; + + if (len < 16) { + c = crc64_nvme_step(seed ^ UINT64_MAX, p, len); + *crc = c ^ UINT64_MAX; + return; + } + + x = vld1q_u64((const uint64_t *) p); + seed_lane[0] = seed ^ UINT64_MAX; + seed_lane[1] = 0; + x = veorq_u64(x, vld1q_u64(seed_lane)); + + k_lanes[0] = k3_pmull; + k_lanes[1] = k4_pmull; + k = vld1q_u64(k_lanes); + + off = 16; + while (off + 16 <= len) { + uint64x2_t d; + + d = vld1q_u64((const uint64_t *) (p + off)); + x = veorq_u64(fold16_pmull(x, k), d); + off += 16; + } + + vst1q_u8(post, vreinterpretq_u8_u64(x)); + + c = crc64_nvme_step(0, post, 16); + c = crc64_nvme_step(c, p + off, len - off); + + *crc = c ^ UINT64_MAX; +} +#endif /* HAVE_PMULL */ + +void crc64_nvme(uint64_t * crc, + const void * buf, + size_t len) +{ +#ifdef HAVE_PCLMUL + crc64_nvme_clmul(crc, buf, len); +#elif defined(HAVE_PMULL) + crc64_nvme_pmull(crc, buf, len); +#else + crc64_nvme_table(crc, buf, len); +#endif +} diff --git a/src/lib/crc/crc8.c b/src/lib/crc/crc8.c new file mode 100644 index 00000000..20976b29 --- /dev/null +++ b/src/lib/crc/crc8.c @@ -0,0 +1,62 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * 8-bit Cyclic Redundancy Check (AUTOSAR variant) + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +/* + * CRC-8/AUTOSAR (reveng catalog): + * poly = 0x2f + * init = 0xff + * refin = false + * refout = false + * xorout = 0xff + * check = crc8_autosar("123456789") == 0xdf + */ + +#include "config.h" + +#include <ouroboros/crc8.h> + + + /* Bit-by-bit MSB-first CRC. */ +void crc8_autosar(uint8_t * crc, + const void * buf, + size_t len) +{ + const uint8_t * p; + uint8_t c; + size_t n; + int i; + + p = (const uint8_t *) buf; + c = *crc ^ 0xff; + + for (n = 0; n < len; n++) { + c ^= p[n]; + for (i = 0; i < 8; i++) { + if (c & 0x80) + c = (uint8_t) ((c << 1) ^ 0x2f); + else + c = (uint8_t) (c << 1); + } + } + + *crc = c ^ 0xff; +} diff --git a/src/lib/crc/tests/CMakeLists.txt b/src/lib/crc/tests/CMakeLists.txt new file mode 100644 index 00000000..11daca5a --- /dev/null +++ b/src/lib/crc/tests/CMakeLists.txt @@ -0,0 +1,21 @@ +get_filename_component(PARENT_PATH ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) +get_filename_component(PARENT_DIR ${PARENT_PATH} NAME) + +compute_test_prefix() + +create_test_sourcelist(${PARENT_DIR}_tests test_suite.c + # Add new tests here + crc8_test.c + crc16_test.c + crc32_test.c + crc64_test.c + ) + +add_executable(${PARENT_DIR}_test ${${PARENT_DIR}_tests}) + +disable_test_logging_for_target(${PARENT_DIR}_test) +target_link_libraries(${PARENT_DIR}_test ouroboros-common) + +add_dependencies(build_tests ${PARENT_DIR}_test) + +ouroboros_register_tests(TARGET ${PARENT_DIR}_test TESTS ${${PARENT_DIR}_tests}) diff --git a/src/lib/crc/tests/crc16_test.c b/src/lib/crc/tests/crc16_test.c new file mode 100644 index 00000000..03a5b504 --- /dev/null +++ b/src/lib/crc/tests/crc16_test.c @@ -0,0 +1,67 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the CRC-16/CCITT-FALSE function + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "config.h" + +#include <ouroboros/crc16.h> + +#include <test/test.h> + +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> + +/* reveng-catalog smoke vectors. */ +static int test_crc16_ccitt_false_basic(void) +{ + uint16_t crc; + + TEST_START(); + + crc = 0; + crc16_ccitt_false(&crc, "", 0); + if (crc != 0xffff) + goto fail; + + crc = 0; + crc16_ccitt_false(&crc, "123456789", 9); + if (crc != 0x29b1) + goto fail; + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int crc16_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_crc16_ccitt_false_basic(); + return ret; +} diff --git a/src/lib/tests/crc32_test.c b/src/lib/crc/tests/crc32_test.c index 35f8a1cd..5a1ddd87 100644 --- a/src/lib/tests/crc32_test.c +++ b/src/lib/crc/tests/crc32_test.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Test of the CRC32 function * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/src/lib/crc/tests/crc64_test.c b/src/lib/crc/tests/crc64_test.c new file mode 100644 index 00000000..cf3f5ca3 --- /dev/null +++ b/src/lib/crc/tests/crc64_test.c @@ -0,0 +1,126 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the CRC-64/NVMe function + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "config.h" + +#include <ouroboros/crc64.h> +#include <ouroboros/random.h> + +#include <test/test.h> + +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> + +/* Reference impl, internal to libouroboros-common. */ +extern void crc64_nvme_table(uint64_t * crc, + const void * buf, + size_t len); + +/* reveng-catalog smoke vectors plus a 16-byte fold-boundary check. */ +static int test_crc64_nvme_basic(void) +{ + uint64_t crc; + + TEST_START(); + + crc = 0; + crc64_nvme(&crc, "", 0); + if (crc != 0x0000000000000000ULL) + goto fail; + + crc = 0; + crc64_nvme(&crc, "123456789", 9); + if (crc != 0xae8b14860a799888ULL) + goto fail; + + crc = 0; + crc64_nvme(&crc, "0123456789abcdef", 16); + if (crc != 0x091485ca7018730eULL) + goto fail; + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +#if defined(HAVE_PCLMUL) || defined(HAVE_PMULL) +/* Cross-check the accelerated dispatcher path against the byte-table. */ +static int test_crc64_nvme_random(void) +{ + static const size_t lens[] = { + 0, 1, 7, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, + 129, 255, 256, 257, 1023, 1024, 1025, 4096 + }; + uint8_t buf[4096]; + size_t i; + uint64_t ref; + uint64_t got; + + TEST_START(); + + if (random_buffer(buf, sizeof(buf)) < 0) { + printf("Failed to generate random data.\n"); + goto fail; + } + + for (i = 0; i < sizeof(lens) / sizeof(lens[0]); i++) { + ref = 0; + crc64_nvme_table(&ref, buf, lens[i]); + + got = 0; + crc64_nvme(&got, buf, lens[i]); + + if (ref == got) + continue; + + printf("Mismatch at len=%zu: table=0x%016lx disp=0x%016lx\n", + lens[i], + (unsigned long) ref, + (unsigned long) got); + goto fail; + } + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +#endif +} + +int crc64_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_crc64_nvme_basic(); +#if defined(HAVE_PCLMUL) || defined(HAVE_PMULL) + ret |= test_crc64_nvme_random(); +#endif + return ret; +} diff --git a/src/lib/crc/tests/crc8_test.c b/src/lib/crc/tests/crc8_test.c new file mode 100644 index 00000000..f7bb33b8 --- /dev/null +++ b/src/lib/crc/tests/crc8_test.c @@ -0,0 +1,67 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the CRC-8/AUTOSAR function + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "config.h" + +#include <ouroboros/crc8.h> + +#include <test/test.h> + +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> + +/* reveng-catalog smoke vectors. */ +static int test_crc8_autosar_basic(void) +{ + uint8_t crc; + + TEST_START(); + + crc = 0; + crc8_autosar(&crc, "", 0); + if (crc != 0x00) + goto fail; + + crc = 0; + crc8_autosar(&crc, "123456789", 9); + if (crc != 0xdf) + goto fail; + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int crc8_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_crc8_autosar_basic(); + return ret; +} diff --git a/src/lib/crypt.c b/src/lib/crypt.c index 1fa12bb8..71197f6e 100644 --- a/src/lib/crypt.c +++ b/src/lib/crypt.c @@ -1,8 +1,7 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * - * Elliptic curve Diffie-Hellman key exchange and - * AES encryption for flows using OpenSSL + * Cryptographic operations * * Dimitri Staessens <dimitri@ouroboros.rocks> * Sander Vrijders <sander@ouroboros.rocks> @@ -21,427 +20,1127 @@ * Foundation, Inc., http://www.fsf.org/about/contact/. */ -#ifdef HAVE_OPENSSL +#if defined(__linux__) || defined(__CYGWIN__) +#define _DEFAULT_SOURCE +#endif + +#include <config.h> + +#include <ouroboros/errno.h> +#include <ouroboros/random.h> +#include <ouroboros/crypt.h> +#ifdef HAVE_OPENSSL #include <openssl/evp.h> -#include <openssl/ec.h> -#include <openssl/pem.h> +#include "crypt/openssl.h" +#endif + +#include <assert.h> +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> + +struct nid_map { + uint16_t nid; + const char * name; +}; + +static const struct nid_map cipher_nid_map[] = { + {NID_aes_128_gcm, "aes-128-gcm"}, + {NID_aes_192_gcm, "aes-192-gcm"}, + {NID_aes_256_gcm, "aes-256-gcm"}, + {NID_chacha20_poly1305, "chacha20-poly1305"}, + {NID_aes_128_ctr, "aes-128-ctr"}, + {NID_aes_192_ctr, "aes-192-ctr"}, + {NID_aes_256_ctr, "aes-256-ctr"}, + {NID_undef, NULL} +}; + +/* Ordered in strength preference, lowest first */ +const uint16_t crypt_supported_nids[] = { +#ifdef HAVE_OPENSSL + NID_aes_128_ctr, + NID_aes_192_ctr, + NID_aes_256_ctr, + NID_aes_128_gcm, + NID_aes_192_gcm, + NID_aes_256_gcm, + NID_chacha20_poly1305, +#endif + NID_undef +}; + +static const struct nid_map kex_nid_map[] = { + {NID_X9_62_prime256v1, "prime256v1"}, + {NID_secp384r1, "secp384r1"}, + {NID_secp521r1, "secp521r1"}, + {NID_X25519, "X25519"}, + {NID_X448, "X448"}, + {NID_ffdhe2048, "ffdhe2048"}, + {NID_ffdhe3072, "ffdhe3072"}, + {NID_ffdhe4096, "ffdhe4096"}, + {NID_MLKEM512, "ML-KEM-512"}, + {NID_MLKEM768, "ML-KEM-768"}, + {NID_MLKEM1024, "ML-KEM-1024"}, + {NID_X25519MLKEM768, "X25519MLKEM768"}, + {NID_X448MLKEM1024, "X448MLKEM1024"}, + {NID_undef, NULL} +}; + +/* Ordered in strength preference, lowest first */ +const uint16_t kex_supported_nids[] = { +#ifdef HAVE_OPENSSL + NID_ffdhe2048, + NID_X9_62_prime256v1, + NID_X25519, + NID_ffdhe3072, + NID_secp384r1, + NID_ffdhe4096, + NID_X448, + NID_secp521r1, +#ifdef HAVE_OPENSSL_ML_KEM + NID_MLKEM512, + NID_MLKEM768, + NID_MLKEM1024, + NID_X25519MLKEM768, + NID_X448MLKEM1024, +#endif +#endif + NID_undef +}; + +static const struct nid_map md_nid_map[] = { + {NID_sha256, "sha256"}, + {NID_sha384, "sha384"}, + {NID_sha512, "sha512"}, + {NID_sha3_256, "sha3-256"}, + {NID_sha3_384, "sha3-384"}, + {NID_sha3_512, "sha3-512"}, + {NID_blake2b512, "blake2b512"}, + {NID_blake2s256, "blake2s256"}, + {NID_undef, NULL} +}; + +/* Ordered in strength preference, lowest first */ +const uint16_t md_supported_nids[] = { +#ifdef HAVE_OPENSSL + NID_blake2s256, + NID_sha256, + NID_sha3_256, + NID_sha384, + NID_sha3_384, + NID_blake2b512, + NID_sha512, + NID_sha3_512, +#endif + NID_undef +}; -#include <openssl/bio.h> +struct crypt_ctx { + void * ctx; /* Encryption context */ +}; -#define IVSZ 16 -/* SYMMKEYSZ defined in dev.c */ +struct auth_ctx { + void * store; +}; -/* - * Derive the common secret from - * your public key pair (kp) - * the remote public key (pub). - * Store it in a preallocated buffer (s). - */ -static int __openssl_ecdh_derive_secret(EVP_PKEY * kp, - EVP_PKEY * pub, - uint8_t * s) +static int parse_kex_value(const char * value, + struct sec_config * cfg) { - EVP_PKEY_CTX * ctx; - int ret; - uint8_t * secret; - size_t secret_len; + SET_KEX_ALGO(cfg, value); + if (cfg->x.nid == NID_undef) + return -ENOTSUP; - ctx = EVP_PKEY_CTX_new(kp, NULL); - if (ctx == NULL) - goto fail_new; + return 0; +} - ret = EVP_PKEY_derive_init(ctx); - if (ret != 1) - goto fail_ctx; +/* not in header, but non-static for unit testing */ +int parse_sec_config(struct sec_config * cfg, + FILE * fp) +{ + char line[256]; + char * equals; + char * key; + char * value; + + assert(cfg != NULL); + assert(fp != NULL); + + /* Set defaults */ + SET_KEX_ALGO_NID(cfg, NID_X9_62_prime256v1); + cfg->x.mode = KEM_MODE_SERVER_ENCAP; + SET_KEX_KDF_NID(cfg, NID_sha256); + SET_KEX_CIPHER_NID(cfg, NID_aes_256_gcm); + SET_KEX_DIGEST_NID(cfg, NID_sha256); + + while (fgets(line, sizeof(line), fp) != NULL) { + char * trimmed; + + /* Skip comments and empty lines */ + if (line[0] == '#' || line[0] == '\n') + continue; + + /* Check for 'none' keyword */ + trimmed = trim_whitespace(line); + if (strcmp(trimmed, "none") == 0) { + memset(cfg, 0, sizeof(*cfg)); + return 0; + } + + /* Find the = separator */ + equals = strchr(line, '='); + if (equals == NULL) + continue; + + /* Split into key and value */ + *equals = '\0'; + key = trim_whitespace(line); + value = trim_whitespace(equals + 1); + + /* Parse key exchange field */ + if (strcmp(key, "kex") == 0) { + if (parse_kex_value(value, cfg) < 0) + return -EINVAL; + } else if (strcmp(key, "cipher") == 0) { + SET_KEX_CIPHER(cfg, value); + if (cfg->c.nid == NID_undef) + return -EINVAL; + } else if (strcmp(key, "kdf") == 0) { + SET_KEX_KDF(cfg, value); + if (cfg->k.nid == NID_undef) + return -EINVAL; + } else if (strcmp(key, "digest") == 0) { + SET_KEX_DIGEST(cfg, value); + if (cfg->d.nid == NID_undef) + return -EINVAL; + } else if (strcmp(key, "kem_mode") == 0) { + if (strcmp(value, "server") == 0) { + cfg->x.mode = KEM_MODE_SERVER_ENCAP; + } else if (strcmp(value, "client") == 0) { + cfg->x.mode = KEM_MODE_CLIENT_ENCAP; + } else { + return -EINVAL; + } + } + } - ret = EVP_PKEY_derive_set_peer(ctx, pub); - if (ret != 1) - goto fail_ctx; + return 0; +} - ret = EVP_PKEY_derive(ctx, NULL, &secret_len); - if (ret != 1) - goto fail_ctx; +/* Parse key exchange config from file */ +int load_sec_config_file(struct sec_config * cfg, + const char * path) +{ + FILE * fp; + int ret; - if (secret_len < SYMMKEYSZ) - goto fail_ctx; + assert(cfg != NULL); + assert(path != NULL); - secret = OPENSSL_malloc(secret_len); - if (secret == NULL) - goto fail_ctx; + fp = fopen(path, "r"); + if (fp == NULL) { + /* File doesn't exist - disable encryption */ + CLEAR_KEX_ALGO(cfg); + return 0; + } + + ret = parse_sec_config(cfg, fp); + + fclose(fp); + + return ret; +} + +int kex_pkp_create(struct sec_config * cfg, + void ** pkp, + uint8_t * pk) +{ +#ifdef HAVE_OPENSSL + assert(cfg != NULL); + assert(pkp != NULL); + + *pkp = NULL; - ret = EVP_PKEY_derive(ctx, secret, &secret_len); - if (ret != 1) - goto fail_derive; + if (cfg->x.str == NULL || kex_validate_nid(cfg->x.nid) < 0) + return -ENOTSUP; - /* Hash the secret for use as AES key. */ - mem_hash(HASH_SHA3_256, s, secret, secret_len); + return openssl_pkp_create(cfg->x.str, (EVP_PKEY **) pkp, pk); +#else + (void) cfg; + (void) pkp; + (void) pk; - OPENSSL_free(secret); - EVP_PKEY_CTX_free(ctx); + *pkp = NULL; return 0; +#endif +} + +void kex_pkp_destroy(void * pkp) +{ + if (pkp == NULL) + return; +#ifdef HAVE_OPENSSL + openssl_pkp_destroy((EVP_PKEY *) pkp); +#else + (void) pkp; + + return; +#endif +} + +int kex_dhe_derive(struct sec_config * cfg, + void * pkp, + buffer_t pk, + uint8_t * s) +{ + assert(cfg != NULL); + + if (kex_validate_nid(cfg->x.nid) < 0) + return -ENOTSUP; + +#ifdef HAVE_OPENSSL + return openssl_dhe_derive((EVP_PKEY *) pkp, pk, cfg->k.nid, s); +#else + (void) pkp; + (void) pk; + + memset(s, 0, SYMMKEYSZ); - fail_derive: - OPENSSL_free(secret); - fail_ctx: - EVP_PKEY_CTX_free(ctx); - fail_new: return -ECRYPT; +#endif } -static int __openssl_ecdh_gen_key(void ** kp) +ssize_t kex_kem_encap(buffer_t pk, + uint8_t * ct, + int kdf, + uint8_t * s) { - EVP_PKEY_CTX * ctx = NULL; - EVP_PKEY_CTX * kctx = NULL; - EVP_PKEY * params = NULL; - int ret; +#ifdef HAVE_OPENSSL + return openssl_kem_encap(pk, ct, kdf, s); +#else + (void) pk; + (void) ct; + (void) kdf; - ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_EC, NULL); - if (ctx == NULL) - goto fail_new_id; + memset(s, 0, SYMMKEYSZ); - ret = EVP_PKEY_paramgen_init(ctx); - if (ret != 1) - goto fail_paramgen; + return -ECRYPT; +#endif +} - ret = EVP_PKEY_CTX_set_ec_paramgen_curve_nid(ctx, NID_X9_62_prime256v1); - if (ret != 1) - goto fail_paramgen; +ssize_t kex_kem_encap_raw(buffer_t pk, + uint8_t * ct, + int kdf, + uint8_t * s) +{ +#ifdef HAVE_OPENSSL + return openssl_kem_encap_raw(pk, ct, kdf, s); +#else + (void) pk; + (void) ct; + (void) kdf; - ret = EVP_PKEY_paramgen(ctx, ¶ms); - if (ret != 1) - goto fail_paramgen; + memset(s, 0, SYMMKEYSZ); - kctx = EVP_PKEY_CTX_new(params, NULL); - if (kctx == NULL) - goto fail_keygen_init; + return -ECRYPT; +#endif +} - ret = EVP_PKEY_keygen_init(kctx); - if (ret != 1) - goto fail_keygen; +int kex_kem_decap(void * pkp, + buffer_t ct, + int kdf, + uint8_t * s) +{ +#ifdef HAVE_OPENSSL + return openssl_kem_decap((EVP_PKEY *) pkp, ct, kdf, s); +#else + (void) pkp; + (void) ct; + (void) kdf; - ret = EVP_PKEY_keygen(kctx, (EVP_PKEY **) kp); - if (ret != 1) - goto fail_keygen; + memset(s, 0, SYMMKEYSZ); - EVP_PKEY_free(params); - EVP_PKEY_CTX_free(kctx); - EVP_PKEY_CTX_free(ctx); + return -ECRYPT; +#endif +} - return 0; +int kex_get_algo_from_pk_der(buffer_t pk, + char * algo) +{ +#ifdef HAVE_OPENSSL + return openssl_get_algo_from_pk_der(pk, algo); +#else + (void) pk; + algo[0] = '\0'; - fail_keygen: - EVP_PKEY_CTX_free(kctx); - fail_keygen_init: - EVP_PKEY_free(params); - fail_paramgen: - EVP_PKEY_CTX_free(ctx); - fail_new_id: return -ECRYPT; +#endif } -static ssize_t openssl_ecdh_pkp_create(void ** pkp, - uint8_t * pk) +int kex_get_algo_from_pk_raw(buffer_t pk, + char * algo) { - uint8_t * pos; - ssize_t len; +#ifdef HAVE_OPENSSL + return openssl_get_algo_from_pk_raw(pk, algo); +#else + (void) pk; + algo[0] = '\0'; - assert(pkp != NULL); - assert(*pkp == NULL); - assert(pk != NULL); + return -ECRYPT; +#endif +} - if (__openssl_ecdh_gen_key(pkp) < 0) - return -ECRYPT; +int kex_validate_algo(const char * algo) +{ + if (algo == NULL) + return -EINVAL; + + /* Use NID validation instead of string array */ + return kex_validate_nid(kex_str_to_nid(algo)); +} - assert(*pkp != NULL); +int crypt_validate_nid(int nid) +{ + const struct nid_map * p; - pos = pk; /* i2d_PUBKEY increments the pointer, don't use buf! */ - len = i2d_PUBKEY(*pkp, &pos); - if (len < 0) { - EVP_PKEY_free(*pkp); - return -ECRYPT; + if (nid == NID_undef) + return -EINVAL; + + for (p = cipher_nid_map; p->name != NULL; p++) { + if (p->nid == nid) + return 0; } - return len; + return -ENOTSUP; } -static void openssl_ecdh_pkp_destroy(void * pkp) + +const char * crypt_nid_to_str(uint16_t nid) { - EVP_PKEY_free((EVP_PKEY *) pkp); + const struct nid_map * p; + + for (p = cipher_nid_map; p->name != NULL; p++) { + if (p->nid == nid) + return p->name; + } + + return NULL; } -static int openssl_ecdh_derive(void * pkp, - uint8_t * pk, - size_t len, - uint8_t * s) +uint16_t crypt_str_to_nid(const char * cipher) { - uint8_t * pos; - EVP_PKEY * pub; + const struct nid_map * p; - pos = pk; /* d2i_PUBKEY increments the pointer, don't use key ptr! */ - pub = d2i_PUBKEY(NULL, (const uint8_t **) &pos, (long) len); - if (pub == NULL) - return -ECRYPT; + if (cipher == NULL) + return NID_undef; - if (__openssl_ecdh_derive_secret(pkp, pub, s) < 0) { - EVP_PKEY_free(pub); - return -ECRYPT; + /* fast, check if cipher pointer is in the map */ + for (p = cipher_nid_map; p->name != NULL; p++) { + if (cipher == p->name) + return p->nid; } - EVP_PKEY_free(pub); + for (p = cipher_nid_map; p->name != NULL; p++) { + if (strcmp(p->name, cipher) == 0) + return p->nid; + } - return 0; + return NID_undef; } -/* - * AES encryption calls. If FRCT is disabled, we should generate a - * 128-bit random IV and append it to the packet. If the flow is - * reliable, we could initialize the context once, and consider the - * stream a single encrypted message to avoid initializing the - * encryption context for each packet. - */ +const char * kex_nid_to_str(uint16_t nid) +{ + const struct nid_map * p; + + for (p = kex_nid_map; p->name != NULL; p++) { + if (p->nid == nid) + return p->name; + } + + return NULL; +} + +uint16_t kex_str_to_nid(const char * algo) +{ + const struct nid_map * p; + + if (algo == NULL) + return NID_undef; + + /* Fast path: check if algo pointer is in the map */ + for (p = kex_nid_map; p->name != NULL; p++) { + if (algo == p->name) + return p->nid; + } + + /* Slow path: string comparison */ + for (p = kex_nid_map; p->name != NULL; p++) { + if (strcmp(p->name, algo) == 0) + return p->nid; + } + + return NID_undef; +} -static int openssl_encrypt(struct flow * f, - struct shm_du_buff * sdb) +int kex_validate_nid(int nid) { - uint8_t * out; - uint8_t * in; - uint8_t * head; - uint8_t iv[IVSZ]; - int in_sz; - int out_sz; - int tmp_sz; - int ret; + const struct nid_map * p; - in = shm_du_buff_head(sdb); - in_sz = shm_du_buff_tail(sdb) - in; + if (nid == NID_undef) + return -EINVAL; - if (random_buffer(iv, IVSZ) < 0) - goto fail_iv; + for (p = kex_nid_map; p->name != NULL; p++) { + if (p->nid == nid) + return 0; + } - out = malloc(in_sz + EVP_MAX_BLOCK_LENGTH); - if (out == NULL) - goto fail_iv; + return -ENOTSUP; +} - EVP_CIPHER_CTX_reset(f->ctx); +const char * md_nid_to_str(uint16_t nid) +{ + const struct nid_map * p; - ret = EVP_EncryptInit_ex(f->ctx, - EVP_aes_256_cbc(), - NULL, - f->key, - iv); - if (ret != 1) - goto fail_encrypt_init; + for (p = md_nid_map; p->name != NULL; p++) { + if (p->nid == nid) + return p->name; + } - ret = EVP_EncryptUpdate(f->ctx, out, &tmp_sz, in, in_sz); - if (ret != 1) - goto fail_encrypt; + return NULL; +} - out_sz = tmp_sz; - ret = EVP_EncryptFinal_ex(f->ctx, out + tmp_sz, &tmp_sz); - if (ret != 1) - goto fail_encrypt; +uint16_t md_str_to_nid(const char * kdf) +{ + const struct nid_map * p; - out_sz += tmp_sz; + if (kdf == NULL) + return NID_undef; - EVP_CIPHER_CTX_cleanup(f->ctx); + /* Fast path: check if kdf pointer is in the map */ + for (p = md_nid_map; p->name != NULL; p++) { + if (kdf == p->name) + return p->nid; + } - assert(out_sz >= in_sz); + /* Slow path: string comparison */ + for (p = md_nid_map; p->name != NULL; p++) { + if (strcmp(p->name, kdf) == 0) + return p->nid; + } - head = shm_du_buff_head_alloc(sdb, IVSZ); - if (head == NULL) - goto fail_encrypt; + return NID_undef; +} - if (shm_du_buff_tail_alloc(sdb, out_sz - in_sz) == NULL) - goto fail_tail_alloc; +int md_validate_nid(int nid) +{ + const struct nid_map * p; - memcpy(head, iv, IVSZ); - memcpy(in, out, out_sz); + if (nid == NID_undef) + return -EINVAL; - free(out); + for (p = md_nid_map; p->name != NULL; p++) { + if (p->nid == nid) + return 0; + } - return 0; + return -ENOTSUP; +} + +int crypt_cipher_rank(int nid) +{ + int i; + + if (nid == NID_undef) + return 0; + + for (i = 0; crypt_supported_nids[i] != NID_undef; i++) { + if ((int) crypt_supported_nids[i] == nid) + return i + 1; + } + + return -1; +} + +int crypt_kdf_rank(int nid) +{ + int i; + + if (nid == NID_undef) + return 0; + + for (i = 0; md_supported_nids[i] != NID_undef; i++) { + if ((int) md_supported_nids[i] == nid) + return i + 1; + } + + return -1; +} + +int crypt_kex_rank(int nid) +{ + int i; + + if (nid == NID_undef) + return 0; + + for (i = 0; kex_supported_nids[i] != NID_undef; i++) { + if ((int) kex_supported_nids[i] == nid) + return i + 1; + } + + return -1; +} + +/* Hash length now returned by md_digest() */ + +int crypt_encrypt(struct crypt_ctx * ctx, + buffer_t in, + buffer_t * out) +{ + assert(ctx != NULL); + assert(ctx->ctx != NULL); + +#ifdef HAVE_OPENSSL + return openssl_encrypt(ctx->ctx, in, out); +#else + (void) ctx; + (void) in; + (void) out; - fail_tail_alloc: - shm_du_buff_head_release(sdb, IVSZ); - fail_encrypt: - EVP_CIPHER_CTX_cleanup(f->ctx); - fail_encrypt_init: - free(out); - fail_iv: return -ECRYPT; +#endif } -static int openssl_decrypt(struct flow * f, - struct shm_du_buff * sdb) +int crypt_decrypt(struct crypt_ctx * ctx, + buffer_t in, + buffer_t * out) { - uint8_t * in; - uint8_t * out; - uint8_t iv[IVSZ]; - int ret; - int out_sz; - int in_sz; - int tmp_sz; + assert(ctx != NULL); + assert(ctx->ctx != NULL); - in = shm_du_buff_head_release(sdb, IVSZ); +#ifdef HAVE_OPENSSL + return openssl_decrypt(ctx->ctx, in, out); +#else + (void) ctx; + (void) in; + (void) out; - memcpy(iv, in, IVSZ); + return -ECRYPT; +#endif +} - in = shm_du_buff_head(sdb); +struct crypt_ctx * crypt_create_ctx(struct crypt_sk * sk) +{ + struct crypt_ctx * crypt; - in_sz = shm_du_buff_tail(sdb) - shm_du_buff_head(sdb); + if (crypt_validate_nid(sk->nid) != 0) + return NULL; - out = malloc(in_sz); - if (out == NULL) - goto fail_malloc; + crypt = malloc(sizeof(*crypt)); + if (crypt == NULL) + goto fail_crypt; - EVP_CIPHER_CTX_reset(f->ctx); + memset(crypt, 0, sizeof(*crypt)); - ret = EVP_DecryptInit_ex(f->ctx, - EVP_aes_256_cbc(), - NULL, - f->key, - iv); - if (ret != 1) - goto fail_decrypt_init; +#ifdef HAVE_OPENSSL + crypt->ctx = openssl_crypt_create_ctx(sk); + if (crypt->ctx == NULL) + goto fail_ctx; +#endif + return crypt; +#ifdef HAVE_OPENSSL + fail_ctx: + free(crypt); +#endif + fail_crypt: + return NULL; +} - ret = EVP_DecryptUpdate(f->ctx, out, &tmp_sz, in, in_sz); - if (ret != 1) - goto fail_decrypt; +void crypt_destroy_ctx(struct crypt_ctx * crypt) +{ + if (crypt == NULL) + return; - out_sz = tmp_sz; +#ifdef HAVE_OPENSSL + assert(crypt->ctx != NULL); + openssl_crypt_destroy_ctx(crypt->ctx); +#else + assert(crypt->ctx == NULL); +#endif + free(crypt); +} - ret = EVP_DecryptFinal_ex(f->ctx, out + tmp_sz, &tmp_sz); - if (ret != 1) - goto fail_decrypt; +int crypt_get_ivsz(struct crypt_ctx * ctx) +{ + if (ctx == NULL) + return -EINVAL; - out_sz += tmp_sz; +#ifdef HAVE_OPENSSL + assert(ctx->ctx != NULL); + return openssl_crypt_get_ivsz(ctx->ctx); +#else + assert(ctx->ctx == NULL); + return -ENOTSUP; +#endif +} - assert(out_sz <= in_sz); +int crypt_get_tagsz(struct crypt_ctx * ctx) +{ + if (ctx == NULL) + return -EINVAL; - shm_du_buff_tail_release(sdb, in_sz - out_sz); +#ifdef HAVE_OPENSSL + assert(ctx->ctx != NULL); + return openssl_crypt_get_tagsz(ctx->ctx); +#else + assert(ctx->ctx == NULL); + return -ENOTSUP; +#endif +} - memcpy(in, out, out_sz); +int crypt_load_privkey_file(const char * path, + void ** key) +{ + *key = NULL; - free(out); +#ifdef HAVE_OPENSSL + return openssl_load_privkey_file(path, key); +#else + (void) path; return 0; +#endif +} - fail_decrypt: - EVP_CIPHER_CTX_cleanup(f->ctx); - fail_decrypt_init: - free(out); - fail_malloc: - return -ECRYPT; +int crypt_load_privkey_str(const char * str, + void ** key) +{ + *key = NULL; + +#ifdef HAVE_OPENSSL + return openssl_load_privkey_str(str, key); +#else + (void) str; + return 0; +#endif } -static int openssl_crypt_init(void ** ctx) +int crypt_load_pubkey_str(const char * str, + void ** key) { - *ctx = EVP_CIPHER_CTX_new(); - if (*ctx == NULL) - return -ECRYPT; + *key = NULL; + +#ifdef HAVE_OPENSSL + return openssl_load_pubkey_str(str, key); +#else + (void) str; return 0; +#endif } -static void openssl_crypt_fini(void * ctx) +int crypt_load_pubkey_file(const char * path, + void ** key) { - EVP_CIPHER_CTX_free(ctx); + *key = NULL; + +#ifdef HAVE_OPENSSL + return openssl_load_pubkey_file(path, key); +#else + (void) path; + + return 0; +#endif } -#endif /* HAVE_OPENSSL */ +int crypt_load_pubkey_file_to_der(const char * path, + buffer_t * buf) +{ + assert(buf != NULL); + +#ifdef HAVE_OPENSSL + return openssl_load_pubkey_file_to_der(path, buf); +#else + (void) path; + + buf->data = NULL; + buf->len = 0; + return 0; +#endif +} -static int crypt_dh_pkp_create(void ** pkp, - uint8_t * pk) +int crypt_load_pubkey_raw_file(const char * path, + buffer_t * buf) { + assert(buf != NULL); + #ifdef HAVE_OPENSSL - assert(pkp != NULL); - *pkp = NULL; - return openssl_ecdh_pkp_create(pkp, pk); + return openssl_load_pubkey_raw_file(path, buf); #else - (void) pkp; - (void) pk; + (void) path; + + buf->data = NULL; + buf->len = 0; + return 0; +#endif +} - memset(pk, 0, MSGBUFSZ); +int crypt_load_privkey_raw_file(const char * path, + void ** key) +{ + *key = NULL; - return -ECRYPT; +#ifdef HAVE_OPENSSL + return openssl_load_privkey_raw_file(path, key); +#else + (void) path; + + return 0; #endif } -static void crypt_dh_pkp_destroy(void * pkp) +int crypt_cmp_key(const void * key1, + const void * key2) { #ifdef HAVE_OPENSSL - openssl_ecdh_pkp_destroy(pkp); + return openssl_cmp_key((const EVP_PKEY *) key1, + (const EVP_PKEY *) key2); #else - (void) pkp; + (void) key1; + (void) key2; + return 0; #endif } -static int crypt_dh_derive(void * pkp, - uint8_t * pk, - size_t len, - uint8_t * s) +void crypt_free_key(void * key) { + if (key == NULL) + return; + #ifdef HAVE_OPENSSL - return openssl_ecdh_derive(pkp, pk, len, s); + openssl_free_key((EVP_PKEY *) key); +#endif +} + +int crypt_load_crt_file(const char * path, + void ** crt) +{ + assert(crt != NULL); + + *crt = NULL; + +#ifdef HAVE_OPENSSL + return openssl_load_crt_file(path, crt); #else - (void) pkp; - (void) pk; - (void) len; + (void) path; - memset(s, 0, SYMMKEYSZ); + return 0; +#endif +} - return -ECRYPT; +int crypt_load_crt_str(const char * str, + void ** crt) +{ + assert(crt != NULL); + + *crt = NULL; + +#ifdef HAVE_OPENSSL + return openssl_load_crt_str(str, crt); +#else + (void) str; + + return 0; #endif } -static int crypt_encrypt(struct flow * f, - struct shm_du_buff * sdb) +int crypt_load_crt_der(const buffer_t buf, + void ** crt) { + assert(crt != NULL); #ifdef HAVE_OPENSSL - return openssl_encrypt(f, sdb); + return openssl_load_crt_der(buf, crt); #else - (void) f; - (void) sdb; + *crt = NULL; + + (void) buf; return 0; #endif } -static int crypt_decrypt(struct flow * f, - struct shm_du_buff * sdb) +int crypt_get_pubkey_crt(void * crt, + void ** pk) { + assert(crt != NULL); + assert(pk != NULL); + #ifdef HAVE_OPENSSL - return openssl_decrypt(f, sdb); + return openssl_get_pubkey_crt(crt, pk); #else - (void) f; - (void) sdb; + (void) crt; - return -ECRYPT; + clrbuf(*pk); + + return 0; +#endif +} + +void crypt_free_crt(void * crt) +{ + if (crt == NULL) + return; +#ifdef HAVE_OPENSSL + openssl_free_crt(crt); +#endif +} + +int crypt_crt_str(const void * crt, + char * buf) +{ +#ifdef HAVE_OPENSSL + return openssl_crt_str(crt, buf); +#else + (void) crt; + (void) buf; + + return 0; +#endif +} + +int crypt_crt_der(const void * crt, + buffer_t * buf) +{ + assert(crt != NULL); + assert(buf != NULL); + +#ifdef HAVE_OPENSSL + return openssl_crt_der(crt, buf); +#else + (void) crt; + + clrbuf(*buf); + + return 0; +#endif +} + +int crypt_check_crt_name(void * crt, + const char * name) +{ +#ifdef HAVE_OPENSSL + return openssl_check_crt_name(crt, name); +#else + (void) crt; + (void) name; + + return 0; #endif } -static int crypt_init(void ** ctx) +int crypt_get_crt_name(void * crt, + char * name) { #ifdef HAVE_OPENSSL - return openssl_crypt_init(ctx); + return openssl_get_crt_name(crt, name); #else + (void) crt; + (void) name; + + return 0; +#endif +} + +struct auth_ctx * auth_create_ctx(void) +{ + struct auth_ctx * ctx; + + ctx = malloc(sizeof(*ctx)); + if (ctx == NULL) + goto fail_malloc; + + memset(ctx, 0, sizeof(*ctx)); +#ifdef HAVE_OPENSSL + ctx->store = openssl_auth_create_store(); + if (ctx->store == NULL) + goto fail_store; +#endif + return ctx; +#ifdef HAVE_OPENSSL + fail_store: + free(ctx); +#endif + fail_malloc: + return NULL; +} + +void auth_destroy_ctx(struct auth_ctx * ctx) +{ + if (ctx == NULL) + return; +#ifdef HAVE_OPENSSL + openssl_auth_destroy_store(ctx->store); +#endif + free(ctx); +} + +int auth_add_crt_to_store(struct auth_ctx * ctx, + void * crt) +{ assert(ctx != NULL); - *ctx = NULL; + assert(crt != NULL); + +#ifdef HAVE_OPENSSL + return openssl_auth_add_crt_to_store(ctx->store, crt); +#else + (void) ctx; + (void) crt; return 0; #endif } -static void crypt_fini(void * ctx) +int auth_verify_crt(struct auth_ctx * ctx, + void * crt) { #ifdef HAVE_OPENSSL - openssl_crypt_fini(ctx); + return openssl_verify_crt(ctx->store, crt); #else - assert(ctx == NULL); (void) ctx; + (void) crt; + + return 0; +#endif +} + +int auth_sign(void * pkp, + int md_nid, + buffer_t msg, + buffer_t * sig) +{ +#ifdef HAVE_OPENSSL + return openssl_sign((EVP_PKEY *) pkp, md_nid, msg, sig); +#else + (void) pkp; + (void) md_nid; + (void) msg; + (void) sig; + + clrbuf(*sig); + + return 0; +#endif +} + +int auth_verify_sig(void * pk, + int md_nid, + buffer_t msg, + buffer_t sig) +{ +#ifdef HAVE_OPENSSL + return openssl_verify_sig((EVP_PKEY *) pk, md_nid, msg, sig); +#else + (void) pk; + (void) md_nid; + (void) msg; + (void) sig; + + return 0; +#endif +} + +ssize_t md_digest(int md_nid, + buffer_t in, + uint8_t * out) +{ +#ifdef HAVE_OPENSSL + return openssl_md_digest(md_nid, in, out); +#else + (void) md_nid; + (void) in; + (void) out; + + return -1; +#endif +} + +ssize_t md_len(int md_nid) +{ +#ifdef HAVE_OPENSSL + return openssl_md_len(md_nid); +#else + (void) md_nid; + return -1; +#endif +} + +int crypt_secure_malloc_init(size_t max) +{ +#ifdef HAVE_OPENSSL + return openssl_secure_malloc_init(max, SECMEM_GUARD); +#else + (void) max; + return 0; +#endif +} + +void crypt_secure_malloc_fini(void) +{ +#ifdef HAVE_OPENSSL + openssl_secure_malloc_fini(); +#endif +} + +void crypt_cleanup(void) +{ +#ifdef HAVE_OPENSSL + openssl_cleanup(); +#endif +} + +void * crypt_secure_malloc(size_t size) +{ +#ifdef HAVE_OPENSSL + return openssl_secure_malloc(size); +#else + return malloc(size); +#endif +} + +void crypt_secure_free(void * ptr, + size_t size) +{ + if (ptr == NULL) + return; + +#ifdef HAVE_OPENSSL + openssl_secure_free(ptr, size); +#else + memset(ptr, 0, size); + free(ptr); +#endif +} + +void crypt_secure_clear(void * ptr, + size_t size) +{ + volatile uint8_t * p; + + if (ptr == NULL) + return; + +#ifdef HAVE_OPENSSL + (void) p; + openssl_secure_clear(ptr, size); +#elif defined(HAVE_EXPLICIT_BZERO) + (void) p; + explicit_bzero(ptr, size); +#else /* best effort to avoid optimizing out */ + p = ptr; + while (size-- > 0) + *p++ = 0; #endif } diff --git a/src/lib/crypt/openssl.c b/src/lib/crypt/openssl.c new file mode 100644 index 00000000..5916e3cb --- /dev/null +++ b/src/lib/crypt/openssl.c @@ -0,0 +1,1882 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * OpenSSL based cryptographic operations + * Elliptic curve Diffie-Hellman key exchange + * AES encryption + # Authentication + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include <config.h> + +#include <ouroboros/errno.h> +#include <ouroboros/crypt.h> +#include <ouroboros/hash.h> +#include <ouroboros/random.h> +#include <ouroboros/utils.h> + +#include <openssl/evp.h> +#include <openssl/bio.h> +#include <openssl/ec.h> +#include <openssl/err.h> +#include <openssl/kdf.h> +#include <openssl/pem.h> +#include <openssl/sha.h> +#include <openssl/provider.h> +#include <openssl/x509v3.h> +#include <openssl/x509_vfy.h> + +#include <assert.h> +#include <stdio.h> + +#define IS_EC_GROUP(str) (strcmp(str, "EC") == 0) +#define IS_DH_GROUP(str) (strcmp(str, "DH") == 0) + +#define HKDF_INFO_DHE "o7s-ossl-dhe" +#define HKDF_INFO_ENCAP "o7s-ossl-encap" +#define HKDF_INFO_ROTATION "o7s-key-rotation" +#define HKDF_SALT_LEN 32 /* SHA-256 output size */ + +struct ossl_crypt_ctx { + EVP_CIPHER_CTX * evp_ctx; + const EVP_CIPHER * cipher; + int ivsz; + int tagsz; + + struct { + uint8_t * cur; /* current key */ + uint8_t * prv; /* rotated key */ + } keys; + + struct { + uint32_t cntr; /* counter */ + uint32_t mask; /* phase mask */ + uint32_t age; /* counter within epoch */ + uint8_t phase; /* current key phase */ + uint8_t salt[HKDF_SALT_LEN]; + } rot; /* rotation logic */ +}; + +struct kdf_info { + buffer_t secret; + int nid; + buffer_t salt; + buffer_t info; + buffer_t key; +}; + +/* Key rotation macros */ +#define HAS_PHASE_BIT_TOGGLED(ctx) \ + (((ctx)->rot.cntr & (ctx)->rot.mask) != \ + (((ctx)->rot.cntr - 1) & (ctx)->rot.mask)) + +#define HAS_GRACE_EXPIRED(ctx) \ + ((ctx)->rot.age >= ((ctx)->rot.mask >> 1)) + +#define ROTATION_TOO_RECENT(ctx) \ + ((ctx)->rot.age < ((ctx)->rot.mask - ((ctx)->rot.mask >> 2))) + +/* Convert hash NID to OpenSSL digest name string for HKDF */ +static const char * hash_nid_to_digest_name(int nid) +{ + const EVP_MD * md; + const char * name; + + md = EVP_get_digestbynid(nid); + if (md == NULL) + return "SHA256"; /* fallback to SHA-256 */ + + name = EVP_MD_get0_name(md); + if (name == NULL) + return "SHA256"; /* fallback to SHA-256 */ + + return name; +} + +/* Extract public key bytes from a key pair for salt derivation */ +static int get_pk_bytes_from_key(EVP_PKEY * key, + buffer_t * pk) +{ + const char * name; + int ret; + + assert(key != NULL); + assert(pk != NULL); + + name = EVP_PKEY_get0_type_name(key); + if (name == NULL) + goto fail_name; + + if (IS_HYBRID_KEM(name)) { + pk->len = EVP_PKEY_get1_encoded_public_key(key, &pk->data); + if (pk->len == 0) + goto fail_name; + } else { + /* Pure ML-KEM: use DER encoding to match encap */ + pk->data = NULL; + ret = i2d_PUBKEY(key, &pk->data); + if (ret <= 0) + goto fail_name; + pk->len = (size_t) ret; + } + + return 0; + fail_name: + return -ECRYPT; +} + +/* Derive salt from public key bytes by hashing them */ +static int derive_salt_from_pk_bytes(buffer_t pk, + uint8_t * salt, + size_t salt_len) +{ + uint8_t hash[EVP_MAX_MD_SIZE]; + unsigned hash_len; + + assert(pk.data != NULL); + assert(salt != NULL); + + if (EVP_Digest(pk.data, pk.len, hash, &hash_len, + EVP_sha256(), NULL) != 1) + goto fail_digest; + + memcpy(salt, hash, salt_len < hash_len ? salt_len : hash_len); + + return 0; + fail_digest: + return -ECRYPT; +} + +/* Derive salt from two public key byte buffers (DHE) in canonical order */ +static int derive_salt_from_pk_bytes_dhe(buffer_t local, + buffer_t remote, + uint8_t * salt, + size_t salt_len) +{ + uint8_t * concat; + size_t concat_len; + uint8_t hash[EVP_MAX_MD_SIZE]; + unsigned hash_len; + size_t min_len; + int cmp; + + assert(local.data != NULL); + assert(remote.data != NULL); + assert(salt != NULL); + + concat_len = local.len + remote.len; + concat = OPENSSL_malloc(concat_len); + if (concat == NULL) + goto fail_malloc; + + /* Canonical order: compare and concatenate smaller first */ + min_len = local.len < remote.len ? local.len : remote.len; + cmp = memcmp(local.data, remote.data, min_len); + if (cmp < 0 || (cmp == 0 && local.len < remote.len)) { + memcpy(concat, local.data, local.len); + memcpy(concat + local.len, remote.data, remote.len); + } else { + memcpy(concat, remote.data, remote.len); + memcpy(concat + remote.len, local.data, local.len); + } + + if (EVP_Digest(concat, concat_len, hash, &hash_len, + EVP_sha256(), NULL) != 1) + goto fail_digest; + + OPENSSL_free(concat); + + memcpy(salt, hash, salt_len < hash_len ? salt_len : hash_len); + + return 0; + fail_digest: + OPENSSL_free(concat); + fail_malloc: + return -ECRYPT; +} + +/* Derive key using HKDF */ +#define OPc_u_str OSSL_PARAM_construct_utf8_string +#define OPc_o_str OSSL_PARAM_construct_octet_string +static int derive_key_hkdf(struct kdf_info * ki) +{ + EVP_KDF * kdf; + EVP_KDF_CTX * kctx; + OSSL_PARAM params[5]; + const char * digest; + int idx; + + digest = hash_nid_to_digest_name(ki->nid); + + kdf = EVP_KDF_fetch(NULL, "HKDF", NULL); + if (kdf == NULL) + goto fail_fetch; + + kctx = EVP_KDF_CTX_new(kdf); + if (kctx == NULL) + goto fail_ctx; + + idx = 0; + params[idx++] = OPc_u_str("digest", (char *) digest, 0); + params[idx++] = OPc_o_str("key", ki->secret.data, ki->secret.len); + params[idx++] = OPc_o_str("salt", ki->salt.data, ki->salt.len); + params[idx++] = OPc_o_str("info", ki->info.data, ki->info.len); + + params[idx] = OSSL_PARAM_construct_end(); + + if (EVP_KDF_derive(kctx, ki->key.data, ki->key.len, params) != 1) + goto fail_derive; + + EVP_KDF_CTX_free(kctx); + EVP_KDF_free(kdf); + + return 0; + + fail_derive: + EVP_KDF_CTX_free(kctx); + fail_ctx: + EVP_KDF_free(kdf); + fail_fetch: + return -ECRYPT; +} + +/* Key rotation helper functions implementation */ +static int should_rotate_key_rx(struct ossl_crypt_ctx * ctx, + uint8_t rx_phase) +{ + assert(ctx != NULL); + + /* Phase must have changed */ + if (rx_phase == ctx->rot.phase) + return 0; + + if (ROTATION_TOO_RECENT(ctx)) + return 0; + + return 1; +} + +static int rotate_key(struct ossl_crypt_ctx * ctx) +{ + struct kdf_info ki; + uint8_t * tmp; + + assert(ctx != NULL); + + /* Swap keys - move current to prev */ + tmp = ctx->keys.prv; + ctx->keys.prv = ctx->keys.cur; + + if (tmp != NULL) { + /* Reuse old prev_key memory for new key */ + ctx->keys.cur = tmp; + } else { + /* First rotation - allocate new memory */ + ctx->keys.cur = OPENSSL_secure_malloc(SYMMKEYSZ); + if (ctx->keys.cur == NULL) + return -ECRYPT; + } + + /* Derive new key from previous key using HKDF */ + ki.secret.data = ctx->keys.prv; + ki.secret.len = SYMMKEYSZ; + ki.nid = NID_sha256; + ki.salt.data = ctx->rot.salt; + ki.salt.len = HKDF_SALT_LEN; + ki.info.data = (uint8_t *) HKDF_INFO_ROTATION; + ki.info.len = strlen(HKDF_INFO_ROTATION); + ki.key.data = ctx->keys.cur; + ki.key.len = SYMMKEYSZ; + + if (derive_key_hkdf(&ki) != 0) + return -ECRYPT; + + ctx->rot.age = 0; + ctx->rot.phase = !ctx->rot.phase; + + return 0; +} + +static void cleanup_old_key(struct ossl_crypt_ctx * ctx) +{ + assert(ctx != NULL); + + if (ctx->keys.prv == NULL) + return; + + if (!HAS_GRACE_EXPIRED(ctx)) + return; + + OPENSSL_secure_clear_free(ctx->keys.prv, SYMMKEYSZ); + ctx->keys.prv = NULL; +} + +static int try_decrypt(struct ossl_crypt_ctx * ctx, + uint8_t * key, + uint8_t * iv, + uint8_t * input, + int in_sz, + uint8_t * out, + int * out_sz) +{ + uint8_t * tag; + int tmp_sz; + int ret; + + tag = input + in_sz; + + EVP_CIPHER_CTX_reset(ctx->evp_ctx); + + ret = EVP_DecryptInit_ex(ctx->evp_ctx, ctx->cipher, NULL, key, iv); + if (ret != 1) + return -1; + + if (ctx->tagsz > 0) { + ret = EVP_CIPHER_CTX_ctrl(ctx->evp_ctx, EVP_CTRL_AEAD_SET_TAG, + ctx->tagsz, tag); + if (ret != 1) + return -1; + } + + ret = EVP_DecryptUpdate(ctx->evp_ctx, out, &tmp_sz, input, in_sz); + if (ret != 1) + return -1; + + *out_sz = tmp_sz; + + ret = EVP_DecryptFinal_ex(ctx->evp_ctx, out + tmp_sz, &tmp_sz); + if (ret != 1) + return -1; + + *out_sz += tmp_sz; + + return 0; +} + +/* + * Derive the common secret from + * - your public key pair (pkp) + * - the remote public key bytes (remote_pk). + * Store it in a preallocated buffer (s). + */ +static int __openssl_dhe_derive(EVP_PKEY * pkp, + EVP_PKEY * pub, + buffer_t remote_pk, + int kdf, + uint8_t * s) +{ + EVP_PKEY_CTX * ctx; + struct kdf_info ki; + buffer_t local_pk; + int ret; + uint8_t * secret; + size_t secret_len; + uint8_t salt_buf[HKDF_SALT_LEN]; + + /* Extract local public key bytes */ + local_pk.data = NULL; + ret = i2d_PUBKEY(pkp, &local_pk.data); + if (ret <= 0) + goto fail_local; + local_pk.len = (size_t) ret; + + /* Derive salt from both public keys */ + if (derive_salt_from_pk_bytes_dhe(local_pk, remote_pk, salt_buf, + HKDF_SALT_LEN) < 0) + goto fail_salt; + + ctx = EVP_PKEY_CTX_new(pkp, NULL); + if (ctx == NULL) + goto fail_salt; + + ret = EVP_PKEY_derive_init(ctx); + if (ret != 1) + goto fail_ctx; + + ret = EVP_PKEY_derive_set_peer(ctx, pub); + if (ret != 1) + goto fail_ctx; + + ret = EVP_PKEY_derive(ctx, NULL, &secret_len); + if (ret != 1) + goto fail_ctx; + + if (secret_len < SYMMKEYSZ) + goto fail_ctx; + + secret = OPENSSL_malloc(secret_len); + if (secret == NULL) + goto fail_ctx; + + ret = EVP_PKEY_derive(ctx, secret, &secret_len); + if (ret != 1) + goto fail_derive; + + ki.nid = kdf; + ki.secret.len = secret_len; + ki.secret.data = secret; + ki.info.len = strlen(HKDF_INFO_DHE); + ki.info.data = (uint8_t *) HKDF_INFO_DHE; + ki.key.len = SYMMKEYSZ; + ki.key.data = s; + ki.salt.len = HKDF_SALT_LEN; + ki.salt.data = salt_buf; + + /* Derive symmetric key from shared secret using HKDF */ + ret = derive_key_hkdf(&ki); + + OPENSSL_free(secret); + EVP_PKEY_CTX_free(ctx); + OPENSSL_free(local_pk.data); + + if (ret != 0) + return ret; + + return 0; + fail_derive: + OPENSSL_free(secret); + fail_ctx: + EVP_PKEY_CTX_free(ctx); + fail_salt: + OPENSSL_free(local_pk.data); + fail_local: + return -ECRYPT; +} + +static int __openssl_dhe_gen_key(const char * algo, + EVP_PKEY ** kp) +{ + EVP_PKEY_CTX * ctx = NULL; + EVP_PKEY_CTX * kctx = NULL; + EVP_PKEY * params = NULL; + int nid; + int type; + int ret; + + assert(algo != NULL); + assert(kp != NULL); + + nid = OBJ_txt2nid(algo); + if (nid == NID_undef) + return -ECRYPT; + + /* X25519 and X448: direct keygen context */ + if (nid == EVP_PKEY_X25519 || nid == EVP_PKEY_X448) { + kctx = EVP_PKEY_CTX_new_id(nid, NULL); + if (kctx == NULL) + goto fail_kctx; + + goto keygen; + } + /* EC and FFDHE: parameter generation first */ + type = (strncmp(algo, "ffdhe", 5) == 0) ? EVP_PKEY_DH : EVP_PKEY_EC; + + ctx = EVP_PKEY_CTX_new_id(type, NULL); + if (ctx == NULL) + goto fail_ctx; + + ret = EVP_PKEY_paramgen_init(ctx); + if (ret != 1) + goto fail_paramgen; + + if (type == EVP_PKEY_EC) + ret = EVP_PKEY_CTX_set_ec_paramgen_curve_nid(ctx, nid); + else /* EVP_PKEY_DH */ + ret = EVP_PKEY_CTX_set_dh_nid(ctx, nid); + + if (ret != 1) + goto fail_paramgen; + + ret = EVP_PKEY_paramgen(ctx, ¶ms); + if (ret != 1) + goto fail_paramgen; + + kctx = EVP_PKEY_CTX_new(params, NULL); + if (kctx == NULL) + goto fail_kctx; + + EVP_PKEY_free(params); + EVP_PKEY_CTX_free(ctx); + keygen: + ret = EVP_PKEY_keygen_init(kctx); + if (ret != 1) + goto fail_keygen; + + ret = EVP_PKEY_keygen(kctx, kp); + if (ret != 1) + goto fail_keygen; + + EVP_PKEY_CTX_free(kctx); + + return 0; + + fail_keygen: + EVP_PKEY_CTX_free(kctx); + return -ECRYPT; + fail_kctx: + if (params != NULL) + EVP_PKEY_free(params); + fail_paramgen: + if (ctx != NULL) + EVP_PKEY_CTX_free(ctx); + fail_ctx: + return -ECRYPT; +} + +static int __openssl_kem_gen_key(const char * algo, + EVP_PKEY ** kp) +{ + EVP_PKEY_CTX * kctx; + int ret; + + assert(algo != NULL); + assert(kp != NULL); + + /* PQC KEM (ML-KEM-512, ML-KEM-768, ML-KEM-1024) or hybrid */ + kctx = EVP_PKEY_CTX_new_from_name(NULL, algo, NULL); + if (kctx == NULL) + goto fail_kctx; + + ret = EVP_PKEY_keygen_init(kctx); + if (ret != 1) + goto fail_keygen; + + ret = EVP_PKEY_keygen(kctx, kp); + if (ret != 1) + goto fail_keygen; + + EVP_PKEY_CTX_free(kctx); + + return 0; + + fail_keygen: + EVP_PKEY_CTX_free(kctx); + fail_kctx: + return -ECRYPT; +} + +/* Determine hybrid KEM algorithm from raw key/ciphertext length */ +static const char * __openssl_hybrid_algo_from_len(size_t len) +{ + switch(len) { + case X25519MLKEM768_PKSZ: + return "X25519MLKEM768"; + case X25519MLKEM768_CTSZ: + return "X25519MLKEM768"; + case X448MLKEM1024_PKSZ: + return "X448MLKEM1024"; + default: + break; + } + + return NULL; +} + +static int __openssl_kex_gen_key(const char * algo, + EVP_PKEY ** kp) +{ + assert(algo != NULL); + assert(kp != NULL); + + /* Dispatch based on algorithm name prefix */ + if (IS_KEM_ALGORITHM(algo)) + return __openssl_kem_gen_key(algo, kp); + + return __openssl_dhe_gen_key(algo, kp); +} + +ssize_t openssl_pkp_create(const char * algo, + EVP_PKEY ** pkp, + uint8_t * pk) +{ + uint8_t * pos; + buffer_t raw; + ssize_t len; + + assert(algo != NULL); + assert(pkp != NULL); + assert(*pkp == NULL); + assert(pk != NULL); + + if (__openssl_kex_gen_key(algo, pkp) < 0) + goto fail_key; + + if (IS_HYBRID_KEM(algo)) { /* Raw encode hybrid KEM */ + raw.len = EVP_PKEY_get1_encoded_public_key(*pkp, &raw.data); + if (raw.len == 0) + goto fail_pubkey; + + memcpy(pk, raw.data, raw.len); + OPENSSL_free(raw.data); + + return (ssize_t) raw.len; + } else { /* DER encode standard algorithms */ + pos = pk; /* i2d_PUBKEY increments the ptr, don't use pk! */ + len = i2d_PUBKEY(*pkp, &pos); + if (len < 0) + goto fail_pubkey; + + return len; + } + fail_pubkey: + EVP_PKEY_free(*pkp); + fail_key: + return -ECRYPT; +} + +/* Common KEM encapsulation - pub key and salt already prepared */ +static ssize_t __openssl_kem_encap(EVP_PKEY * pub, + uint8_t * salt, + uint8_t * ct, + int kdf, + uint8_t * s) +{ + EVP_PKEY_CTX * ctx; + struct kdf_info ki; + uint8_t * secret; + size_t secret_len; + size_t ct_len; + int ret; + + ctx = EVP_PKEY_CTX_new(pub, NULL); + if (ctx == NULL) + goto fail_ctx; + + ret = EVP_PKEY_encapsulate_init(ctx, NULL); + if (ret != 1) + goto fail_encap; + + /* Get required lengths */ + ret = EVP_PKEY_encapsulate(ctx, NULL, &ct_len, NULL, &secret_len); + if (ret != 1 || ct_len > CRYPT_KEY_BUFSZ) + goto fail_encap; + + /* Allocate buffer for secret */ + secret = OPENSSL_malloc(secret_len); + if (secret == NULL) + goto fail_encap; + + /* Perform encapsulation */ + ret = EVP_PKEY_encapsulate(ctx, ct, &ct_len, secret, &secret_len); + if (ret != 1) + goto fail_secret; + + ki.secret.len = secret_len; + ki.secret.data = secret; + ki.nid = kdf; + ki.info.len = strlen(HKDF_INFO_ENCAP); + ki.info.data = (uint8_t *) HKDF_INFO_ENCAP; + ki.key.len = SYMMKEYSZ; + ki.key.data = s; + ki.salt.len = HKDF_SALT_LEN; + ki.salt.data = salt; + + /* Derive symmetric key from shared secret using HKDF */ + ret = derive_key_hkdf(&ki); + + OPENSSL_free(secret); + EVP_PKEY_CTX_free(ctx); + + if (ret != 0) + return -ECRYPT; + + return (ssize_t) ct_len; + + fail_secret: + OPENSSL_free(secret); + fail_encap: + EVP_PKEY_CTX_free(ctx); + fail_ctx: + return -ECRYPT; +} + +/* ML-KEM encapsulation - DER-encoded public key */ +ssize_t openssl_kem_encap(buffer_t pk, + uint8_t * ct, + int kdf, + uint8_t * s) +{ + EVP_PKEY * pub; + uint8_t * pos; + uint8_t salt[HKDF_SALT_LEN]; + ssize_t ret; + + assert(pk.data != NULL); + assert(ct != NULL); + assert(s != NULL); + + if (derive_salt_from_pk_bytes(pk, salt, HKDF_SALT_LEN) < 0) + goto fail_salt; + + pos = pk.data; + pub = d2i_PUBKEY(NULL, (const uint8_t **) &pos, (long) pk.len); + if (pub == NULL) + goto fail_salt; + + ret = __openssl_kem_encap(pub, salt, ct, kdf, s); + + EVP_PKEY_free(pub); + + return ret; + fail_salt: + return -ECRYPT; +} + +/* Hybrid KEM encapsulation: raw-encoded public key */ +ssize_t openssl_kem_encap_raw(buffer_t pk, + uint8_t * ct, + int kdf, + uint8_t * s) +{ + EVP_PKEY * pub; + const char * algo; + uint8_t salt[HKDF_SALT_LEN]; + ssize_t ret; + + assert(pk.data != NULL); + assert(ct != NULL); + assert(s != NULL); + + if (derive_salt_from_pk_bytes(pk, salt, HKDF_SALT_LEN) < 0) + goto fail_salt; + + algo = __openssl_hybrid_algo_from_len(pk.len); + if (algo == NULL) + goto fail_salt; + + pub = EVP_PKEY_new_raw_public_key_ex(NULL, algo, NULL, + pk.data, pk.len); + if (pub == NULL) + goto fail_salt; + + ret = __openssl_kem_encap(pub, salt, ct, kdf, s); + + EVP_PKEY_free(pub); + + return ret; + fail_salt: + return -ECRYPT; +} + +/* KEM decapsulation - used by party that generated the keypair */ +int openssl_kem_decap(EVP_PKEY * priv, + buffer_t ct, + int kdf, + uint8_t * s) +{ + EVP_PKEY_CTX * ctx; + struct kdf_info ki; + buffer_t pk; + uint8_t * secret; + size_t secret_len; + int ret; + uint8_t salt[HKDF_SALT_LEN]; + + /* Extract public key bytes from private key */ + if (get_pk_bytes_from_key(priv, &pk) < 0) + goto fail_pk; + + if (derive_salt_from_pk_bytes(pk, salt, HKDF_SALT_LEN) < 0) + goto fail_salt; + + ctx = EVP_PKEY_CTX_new(priv, NULL); + if (ctx == NULL) + goto fail_salt; + + ret = EVP_PKEY_decapsulate_init(ctx, NULL); + if (ret != 1) + goto fail_ctx; + + /* Get required secret length */ + ret = EVP_PKEY_decapsulate(ctx, NULL, &secret_len, ct.data, ct.len); + if (ret != 1) + goto fail_ctx; + + /* Allocate buffer for secret */ + secret = OPENSSL_malloc(secret_len); + if (secret == NULL) + goto fail_ctx; + + /* Perform decapsulation */ + ret = EVP_PKEY_decapsulate(ctx, secret, &secret_len, ct.data, ct.len); + if (ret != 1) + goto fail_secret; + + ki.secret.len = secret_len; + ki.secret.data = secret; + ki.nid = kdf; + ki.info.len = strlen(HKDF_INFO_ENCAP); + ki.info.data = (uint8_t *) HKDF_INFO_ENCAP; + ki.key.len = SYMMKEYSZ; + ki.key.data = s; + ki.salt.len = HKDF_SALT_LEN; + ki.salt.data = salt; + + /* Derive symmetric key from shared secret using HKDF */ + ret = derive_key_hkdf(&ki); + + OPENSSL_free(secret); + EVP_PKEY_CTX_free(ctx); + OPENSSL_free(pk.data); + + if (ret != 0) + return ret; + + return 0; + + fail_secret: + OPENSSL_free(secret); + fail_ctx: + EVP_PKEY_CTX_free(ctx); + fail_salt: + OPENSSL_free(pk.data); + fail_pk: + return -ECRYPT; +} + +void openssl_pkp_destroy(EVP_PKEY * pkp) +{ + EVP_PKEY_free(pkp); +} + +int __openssl_get_curve(EVP_PKEY * pub, + char * algo) +{ + int ret; + size_t len = KEX_ALGO_BUFSZ; + + ret = EVP_PKEY_get_utf8_string_param(pub, "group", algo, len, &len); + return ret == 1 ? 0 : -ECRYPT; +} + +int openssl_get_algo_from_pk_der(buffer_t pk, + char * algo) +{ + uint8_t * pos; + EVP_PKEY * pub; + char * type_str; + + assert(pk.data != NULL); + assert(algo != NULL); + + pos = pk.data; + pub = d2i_PUBKEY(NULL, (const uint8_t **) &pos, (long) pk.len); + if (pub == NULL) + goto fail_decode; + + type_str = (char *) EVP_PKEY_get0_type_name(pub); + if (type_str == NULL) + goto fail_pub; + + strcpy(algo, type_str); + + if ((IS_EC_GROUP(algo) || IS_DH_GROUP(algo)) && + __openssl_get_curve(pub, algo) < 0) + goto fail_pub; + + EVP_PKEY_free(pub); + return 0; + + fail_pub: + EVP_PKEY_free(pub); + fail_decode: + return -ECRYPT; +} + +int openssl_get_algo_from_pk_raw(buffer_t pk, + char * algo) +{ + const char * hybrid_algo; + + assert(pk.data != NULL); + assert(algo != NULL); + + hybrid_algo = __openssl_hybrid_algo_from_len(pk.len); + if (hybrid_algo == NULL) + return -ECRYPT; + + strcpy(algo, hybrid_algo); + + return 0; +} + +int openssl_dhe_derive(EVP_PKEY * pkp, + buffer_t pk, + int kdf, + uint8_t * s) +{ + uint8_t * pos; + EVP_PKEY * pub; + + assert(pkp != NULL); + assert(pk.data != NULL); + assert(s != NULL); + + /* X.509 DER decoding for DHE */ + pos = pk.data; /* d2i_PUBKEY increments pos, don't use key ptr! */ + pub = d2i_PUBKEY(NULL, (const uint8_t **) &pos, (long) pk.len); + if (pub == NULL) + goto fail_decode; + + if (__openssl_dhe_derive(pkp, pub, pk, kdf, s) < 0) + goto fail_derive; + + EVP_PKEY_free(pub); + + return 0; + fail_derive: + EVP_PKEY_free(pub); + fail_decode: + return -ECRYPT; +} + +int openssl_encrypt(struct ossl_crypt_ctx * ctx, + buffer_t in, + buffer_t * out) +{ + uint8_t * ptr; + uint8_t * iv; + int in_sz; + int out_sz; + int tmp_sz; + int ret; + + assert(ctx != NULL); + + in_sz = (int) in.len; + + out->data = malloc(in.len + EVP_MAX_BLOCK_LENGTH + \ + ctx->ivsz + ctx->tagsz); + if (out->data == NULL) + goto fail_malloc; + + iv = out->data; + ptr = out->data + ctx->ivsz; + + if (random_buffer(iv, ctx->ivsz) < 0) + goto fail_encrypt; + + /* Set IV bit 7 to current key phase (KEY_ROTATION_BIT of counter) */ + if (ctx->rot.cntr & ctx->rot.mask) + iv[0] |= 0x80; + else + iv[0] &= 0x7F; + + EVP_CIPHER_CTX_reset(ctx->evp_ctx); + + ret = EVP_EncryptInit_ex(ctx->evp_ctx, ctx->cipher, NULL, + ctx->keys.cur, iv); + if (ret != 1) + goto fail_encrypt; + + ret = EVP_EncryptUpdate(ctx->evp_ctx, ptr, &tmp_sz, in.data, in_sz); + if (ret != 1) + goto fail_encrypt; + + out_sz = tmp_sz; + ret = EVP_EncryptFinal_ex(ctx->evp_ctx, ptr + tmp_sz, &tmp_sz); + if (ret != 1) + goto fail_encrypt; + + out_sz += tmp_sz; + + /* For AEAD ciphers, get and append the authentication tag */ + if (ctx->tagsz > 0) { + ret = EVP_CIPHER_CTX_ctrl(ctx->evp_ctx, EVP_CTRL_AEAD_GET_TAG, + ctx->tagsz, ptr + out_sz); + if (ret != 1) + goto fail_encrypt; + out_sz += ctx->tagsz; + } + + assert(out_sz >= in_sz); + + out->len = (size_t) out_sz + ctx->ivsz; + + /* Increment packet counter and check for key rotation */ + ctx->rot.cntr++; + ctx->rot.age++; + + if (HAS_PHASE_BIT_TOGGLED(ctx)) { + if (rotate_key(ctx) != 0) + goto fail_encrypt; + } + + cleanup_old_key(ctx); + + return 0; + fail_encrypt: + free(out->data); + fail_malloc: + clrbuf(*out); + return -ECRYPT; +} + +int openssl_decrypt(struct ossl_crypt_ctx * ctx, + buffer_t in, + buffer_t * out) +{ + uint8_t * iv; + uint8_t * input; + uint8_t rx_phase; + int out_sz; + int in_sz; + + assert(ctx != NULL); + + in_sz = (int) in.len - ctx->ivsz; + if (in_sz < ctx->tagsz) + return -ECRYPT; + + in_sz -= ctx->tagsz; + + out->data = malloc(in_sz + EVP_MAX_BLOCK_LENGTH); + if (out->data == NULL) + goto fail_malloc; + + iv = in.data; + input = in.data + ctx->ivsz; + + /* Extract phase from IV bit 7 and check for key rotation */ + rx_phase = (iv[0] & 0x80) ? 1 : 0; + + if (should_rotate_key_rx(ctx, rx_phase)) { + if (rotate_key(ctx) != 0) + goto fail_decrypt; + } + + ctx->rot.cntr++; + ctx->rot.age++; + + if (try_decrypt(ctx, ctx->keys.cur, iv, input, in_sz, out->data, + &out_sz) != 0) { + if (ctx->keys.prv == NULL) + goto fail_decrypt; + if (try_decrypt(ctx, ctx->keys.prv, iv, input, in_sz, + out->data, &out_sz) != 0) + goto fail_decrypt; + } + + assert(out_sz <= in_sz); + + out->len = (size_t) out_sz; + + return 0; + fail_decrypt: + free(out->data); + fail_malloc: + clrbuf(*out); + return -ECRYPT; +} + +struct ossl_crypt_ctx * openssl_crypt_create_ctx(struct crypt_sk * sk) +{ + struct ossl_crypt_ctx * ctx; + + assert(sk != NULL); + assert(sk->key != NULL); + assert(sk->rot_bit > 0 && sk->rot_bit < 32); + + ctx = malloc(sizeof(*ctx)); + if (ctx == NULL) + goto fail_malloc; + + memset(ctx, 0, sizeof(*ctx)); + + ctx->keys.cur = OPENSSL_secure_malloc(SYMMKEYSZ); + if (ctx->keys.cur == NULL) + goto fail_key; + + memcpy(ctx->keys.cur, sk->key, SYMMKEYSZ); + + ctx->keys.prv = NULL; + + /* Derive rotation salt from initial shared secret */ + if (EVP_Digest(sk->key, SYMMKEYSZ, ctx->rot.salt, NULL, + EVP_sha256(), NULL) != 1) + goto fail_cipher; + + ctx->cipher = EVP_get_cipherbynid(sk->nid); + if (ctx->cipher == NULL) + goto fail_cipher; + + ctx->ivsz = EVP_CIPHER_iv_length(ctx->cipher); + + /* Set tag size for AEAD ciphers (GCM, CCM, OCB, ChaCha20-Poly1305) */ + if (EVP_CIPHER_flags(ctx->cipher) & EVP_CIPH_FLAG_AEAD_CIPHER) + ctx->tagsz = 16; /* Standard AEAD tag length (128 bits) */ + + ctx->rot.cntr = 0; + ctx->rot.mask = (1U << sk->rot_bit); + ctx->rot.age = 0; + ctx->rot.phase = 0; + + ctx->evp_ctx = EVP_CIPHER_CTX_new(); + if (ctx->evp_ctx == NULL) + goto fail_cipher; + + return ctx; + + fail_cipher: + OPENSSL_secure_clear_free(ctx->keys.cur, SYMMKEYSZ); + fail_key: + free(ctx); + fail_malloc: + return NULL; +} + +void openssl_crypt_destroy_ctx(struct ossl_crypt_ctx * ctx) +{ + if (ctx == NULL) + return; + + if (ctx->keys.cur != NULL) + OPENSSL_secure_clear_free(ctx->keys.cur, SYMMKEYSZ); + + if (ctx->keys.prv != NULL) + OPENSSL_secure_clear_free(ctx->keys.prv, SYMMKEYSZ); + + EVP_CIPHER_CTX_free(ctx->evp_ctx); + free(ctx); +} + +int openssl_crypt_get_ivsz(struct ossl_crypt_ctx * ctx) +{ + assert(ctx != NULL); + + return ctx->ivsz; +} + +int openssl_crypt_get_tagsz(struct ossl_crypt_ctx * ctx) +{ + assert(ctx != NULL); + + return ctx->tagsz; +} + +/* AUTHENTICATION */ + +int openssl_load_crt_file(const char * path, + void ** crt) +{ + FILE * fp; + X509 * xcrt; + + fp = fopen(path, "r"); + if (fp == NULL) + goto fail_file; + + xcrt = PEM_read_X509(fp, NULL, NULL, NULL); + if (xcrt == NULL) + goto fail_crt; + + fclose(fp); + + *crt = (void *) xcrt; + + return 0; + fail_crt: + fclose(fp); + fail_file: + *crt = NULL; + return -1; +} + +int openssl_load_crt_str(const char * str, + void ** crt) +{ + BIO * bio; + X509 * xcrt; + + bio = BIO_new(BIO_s_mem()); + if (bio == NULL) + goto fail_bio; + + if (BIO_write(bio, str, strlen(str)) < 0) + goto fail_crt; + + xcrt = PEM_read_bio_X509(bio, NULL, NULL, NULL); + if (xcrt == NULL) + goto fail_crt; + + BIO_free(bio); + + *crt = (void *) xcrt; + + return 0; + fail_crt: + BIO_free(bio); + fail_bio: + *crt = NULL; + return -1; +} + +int openssl_load_crt_der(buffer_t buf, + void ** crt) +{ + const uint8_t * p; + X509 * xcrt; + + assert(crt != NULL); + + p = buf.data; + + xcrt = d2i_X509(NULL, &p, buf.len); + if (xcrt == NULL) + goto fail_crt; + + *crt = (void *) xcrt; + + return 0; + fail_crt: + *crt = NULL; + return -1; +} + +int openssl_get_pubkey_crt(void * crt, + void ** key) +{ + EVP_PKEY * pk; + X509 * xcrt; + + assert(crt != NULL); + assert(key != NULL); + + xcrt = (X509 *) crt; + + pk = X509_get_pubkey(xcrt); + if (pk == NULL) + goto fail_key; + + *key = (void *) pk; + + return 0; + fail_key: + return -1; +} + +void openssl_free_crt(void * crt) +{ + X509_free((X509 *) crt); +} + +int openssl_load_privkey_file(const char * path, + void ** key) +{ + FILE * fp; + EVP_PKEY * pkey; + + fp = fopen(path, "r"); + if (fp == NULL) + goto fail_file; + + pkey = PEM_read_PrivateKey(fp, NULL, NULL, ""); + if (pkey == NULL) + goto fail_key; + + fclose(fp); + + *key = (void *) pkey; + + return 0; + fail_key: + fclose(fp); + fail_file: + *key = NULL; + return -1; +} + +int openssl_load_privkey_str(const char * str, + void ** key) +{ + BIO * bio; + EVP_PKEY * pkey; + + bio = BIO_new(BIO_s_mem()); + if (bio == NULL) + goto fail_bio; + + if (BIO_write(bio, str, strlen(str)) < 0) + goto fail_key; + + pkey = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL); + if (pkey == NULL) + goto fail_key; + + BIO_free(bio); + + *key = (void *) pkey; + + return 0; + fail_key: + BIO_free(bio); + fail_bio: + *key = NULL; + return -1; +} + +int openssl_load_pubkey_file(const char * path, + void ** key) +{ + FILE * fp; + EVP_PKEY * pkey; + + fp = fopen(path, "r"); + if (fp == NULL) + goto fail_file; + + pkey = PEM_read_PUBKEY(fp, NULL, NULL, NULL); + if (pkey == NULL) + goto fail_key; + + fclose(fp); + + *key = (void *) pkey; + + return 0; + fail_key: + fclose(fp); + fail_file: + *key = NULL; + return -1; +} + +int openssl_load_pubkey_file_to_der(const char * path, + buffer_t * buf) +{ + FILE * fp; + EVP_PKEY * pkey; + int ret; + + assert(path != NULL); + assert(buf != NULL); + + memset(buf, 0, sizeof(*buf)); + + fp = fopen(path, "r"); + if (fp == NULL) + goto fail_file; + + pkey = PEM_read_PUBKEY(fp, NULL, NULL, NULL); + if (pkey == NULL) + goto fail_key; + + /* Extract public key bytes in DER format */ + ret = get_pk_bytes_from_key(pkey, buf); + if (ret < 0) + goto fail_extract; + + EVP_PKEY_free(pkey); + + fclose(fp); + + return 0; + + fail_extract: + EVP_PKEY_free(pkey); + fail_key: + fclose(fp); + fail_file: + clrbuf(*buf); + return -1; +} + +int openssl_load_pubkey_str(const char * str, + void ** key) +{ + BIO * bio; + EVP_PKEY * pkey; + + bio = BIO_new(BIO_s_mem()); + if (bio == NULL) + goto fail_bio; + + if (BIO_write(bio, str, strlen(str)) < 0) + goto fail_key; + + pkey = PEM_read_bio_PUBKEY(bio, NULL, NULL, NULL); + if (pkey == NULL) + goto fail_key; + + BIO_free(bio); + + *key = (void *) pkey; + + return 0; + fail_key: + BIO_free(bio); + fail_bio: + *key = NULL; + return -1; +} + +int openssl_load_pubkey_raw_file(const char * path, + buffer_t * buf) +{ + FILE * fp; + uint8_t tmp_buf[CRYPT_KEY_BUFSZ]; + size_t bytes_read; + const char * algo; + + assert(path != NULL); + assert(buf != NULL); + + fp = fopen(path, "rb"); + if (fp == NULL) + goto fail_file; + + bytes_read = fread(tmp_buf, 1, CRYPT_KEY_BUFSZ, fp); + if (bytes_read == 0) + goto fail_read; + + /* Validate that this is a known hybrid KEM format */ + algo = __openssl_hybrid_algo_from_len(bytes_read); + if (algo == NULL) + goto fail_read; + + buf->data = malloc(bytes_read); + if (buf->data == NULL) + goto fail_malloc; + + memcpy(buf->data, tmp_buf, bytes_read); + buf->len = bytes_read; + + fclose(fp); + return 0; + + fail_malloc: + fail_read: + fclose(fp); + fail_file: + clrbuf(*buf); + return -1; +} + +/* Determine hybrid KEM algorithm from raw private key length */ +static const char * __openssl_hybrid_algo_from_sk_len(size_t len) +{ + switch(len) { + case X25519MLKEM768_SKSZ: + return "X25519MLKEM768"; + case X448MLKEM1024_SKSZ: + return "X448MLKEM1024"; + default: + break; + } + + return NULL; +} + +int openssl_load_privkey_raw_file(const char * path, + void ** key) +{ + FILE * fp; + uint8_t tmp_buf[4096]; + size_t bytes_read; + const char * algo; + EVP_PKEY * pkey; + + assert(path != NULL); + assert(key != NULL); + + fp = fopen(path, "rb"); + if (fp == NULL) + goto fail_file; + + bytes_read = fread(tmp_buf, 1, sizeof(tmp_buf), fp); + if (bytes_read == 0) + goto fail_read; + + /* Determine algorithm from key size */ + algo = __openssl_hybrid_algo_from_sk_len(bytes_read); + if (algo == NULL) + goto fail_read; + + pkey = EVP_PKEY_new_raw_private_key_ex(NULL, algo, NULL, + tmp_buf, bytes_read); + /* Clear sensitive data from stack */ + OPENSSL_cleanse(tmp_buf, bytes_read); + + if (pkey == NULL) + goto fail_read; + + fclose(fp); + + *key = (void *) pkey; + + return 0; + + fail_read: + fclose(fp); + fail_file: + *key = NULL; + return -1; +} + +int openssl_cmp_key(const EVP_PKEY * key1, + const EVP_PKEY * key2) +{ + assert(key1 != NULL); + assert(key2 != NULL); + +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + return EVP_PKEY_eq(key1, key2) == 1 ? 0 : -1; +#else + return EVP_PKEY_cmp(key1, key2) == 1 ? 0 : -1; +#endif +} + +void openssl_free_key(EVP_PKEY * key) +{ + EVP_PKEY_free(key); +} + +int openssl_check_crt_name(void * crt, + const char * name) +{ + char * subj; + char * cn; + X509 * xcrt; + + xcrt = (X509 *) crt; + + subj = X509_NAME_oneline(X509_get_subject_name(xcrt), NULL, 0); + if (subj == NULL) + goto fail_subj; + + cn = strstr(subj, "CN="); + if (cn == NULL) + goto fail_cn; + + if (strcmp(cn + 3, name) != 0) + goto fail_cn; + + free(subj); + + return 0; + fail_cn: + free(subj); + fail_subj: + return -1; +} + +int openssl_get_crt_name(void * crt, + char * name) +{ + char * subj; + char * cn; + char * end; + X509 * xcrt; + + xcrt = (X509 *) crt; + + subj = X509_NAME_oneline(X509_get_subject_name(xcrt), NULL, 0); + if (subj == NULL) + goto fail_subj; + + cn = strstr(subj, "CN="); + if (cn == NULL) + goto fail_cn; + + cn += 3; /* Skip "CN=" */ + + /* Find end of CN (comma or slash for next field) */ + end = strpbrk(cn, ",/"); + if (end != NULL) + *end = '\0'; + + strcpy(name, cn); + free(subj); + + return 0; + fail_cn: + free(subj); + fail_subj: + return -1; +} + +int openssl_crt_str(const void * crt, + char * str) +{ + BIO * bio; + X509 * xcrt; + char * p; + ssize_t len; + + xcrt = (X509 *) crt; + + bio = BIO_new(BIO_s_mem()); + if (bio == NULL) + goto fail_bio; + + X509_print(bio, xcrt); + + len = (ssize_t) BIO_get_mem_data(bio, &p); + if (len <= 0 || p == NULL) + goto fail_p; + + memcpy(str, p, len); + str[len] = '\0'; + + BIO_free(bio); + + return 0; + fail_p: + BIO_free(bio); + fail_bio: + return -1; +} + +int openssl_crt_der(const void * crt, + buffer_t * buf) +{ + uint8_t * p; + int len; + + assert(crt != NULL); + assert(buf != NULL); + + /* Get the size by encoding to NULL */ + len = i2d_X509((X509 *) crt, NULL); + if (len < 0) + goto fail_len; + + buf->data = malloc((size_t) len); + if (buf->data == NULL) + goto fail_malloc; + + p = buf->data; /* i2d_X509 increments p */ + i2d_X509((X509 *) crt, &p); + buf->len = (size_t) len; + + return 0; + + fail_malloc: + fail_len: + clrbuf(*buf); + return -1; +} + +void * openssl_auth_create_store(void) +{ + return X509_STORE_new(); +} + +void openssl_auth_destroy_store(void * ctx) +{ + X509_STORE_free((X509_STORE *) ctx); +} + +int openssl_auth_add_crt_to_store(void * store, + void * crt) +{ + int ret; + + ret = X509_STORE_add_cert((X509_STORE *) store, (X509 *) crt); + + return ret == 1 ? 0 : -1; +} + +int openssl_verify_crt(void * store, + void * crt) +{ + X509_STORE_CTX * ctx; + X509_STORE * _store; + X509* _crt; + int ret; + + _store = (X509_STORE *) store; + _crt = (X509 *) crt; + + ctx = X509_STORE_CTX_new(); + if (ctx == NULL) + goto fail_store_ctx; + + ret = X509_STORE_CTX_init(ctx, _store, _crt, NULL); + if (ret != 1) + goto fail_ca; + + ret = X509_verify_cert(ctx); + if (ret != 1) + goto fail_ca; + + X509_STORE_CTX_free(ctx); + + return 0; + fail_ca: + X509_STORE_CTX_free(ctx); + fail_store_ctx: + return -1; +} + +static const EVP_MD * select_md(EVP_PKEY * pkey, + int nid) +{ + if (EVP_PKEY_get_id(pkey) < 0) + return NULL; /* Provider-based (PQC) */ + + if (nid == NID_undef) + return NULL; /* Classical requires explicit nid */ + + return EVP_get_digestbynid(nid); +} + +int openssl_sign(EVP_PKEY * pkp, + int nid, + buffer_t msg, + buffer_t * sig) +{ + EVP_MD_CTX * mdctx; + const EVP_MD * md; + size_t required; + + assert(pkp != NULL); + assert(sig != NULL); + + mdctx = EVP_MD_CTX_new(); + if (!mdctx) + goto fail_ctx; + + md = select_md(pkp, nid); + + if (EVP_DigestSignInit(mdctx, NULL, md, NULL, pkp) != 1) + goto fail_digest; + + /* Get required signature buffer size */ + if (EVP_DigestSign(mdctx, NULL, &required, msg.data, msg.len) != 1) + goto fail_digest; + + sig->data = malloc(required); + if (sig->data == NULL) + goto fail_digest; + + if (EVP_DigestSign(mdctx, sig->data, &required, msg.data, msg.len) != 1) + goto fail_sign; + + sig->len = required; + + EVP_MD_CTX_free(mdctx); + + return 0; + fail_sign: + freebuf(*sig); + fail_digest: + EVP_MD_CTX_free(mdctx); + fail_ctx: + clrbuf(*sig); + return -1; +} + +int openssl_verify_sig(EVP_PKEY * pk, + int nid, + buffer_t msg, + buffer_t sig) +{ + EVP_MD_CTX * mdctx; + const EVP_MD * md; + int ret; + + assert(pk != NULL); + + mdctx = EVP_MD_CTX_new(); + if (!mdctx) + goto fail_ctx; + + md = select_md(pk, nid); + + if (EVP_DigestVerifyInit(mdctx, NULL, md, NULL, pk) != 1) + goto fail_digest; + + ret = EVP_DigestVerify(mdctx, sig.data, sig.len, msg.data, msg.len); + if (ret != 1) + goto fail_digest; + + EVP_MD_CTX_free(mdctx); + + return 0; + fail_digest: + EVP_MD_CTX_free(mdctx); + fail_ctx: + clrbuf(sig); + return -1; +} + +ssize_t openssl_md_digest(int nid, + buffer_t in, + uint8_t * out) +{ + const EVP_MD * md; + unsigned int len; + + assert(in.data != NULL); + assert(out != NULL); + + md = EVP_get_digestbynid(nid); + if (md == NULL) + return -1; + + if (EVP_Digest(in.data, in.len, out, &len, md, NULL) != 1) + return -1; + + return (ssize_t) len; +} + +ssize_t openssl_md_len(int nid) +{ + const EVP_MD * md; + + md = EVP_get_digestbynid(nid); + if (md == NULL) + return -1; + + return (ssize_t) EVP_MD_get_size(md); +} + +int openssl_secure_malloc_init(size_t max, + size_t guard) +{ + return CRYPTO_secure_malloc_init(max, guard) == 1 ? 0 : -1; +} + +void openssl_secure_malloc_fini(void) +{ + CRYPTO_secure_malloc_done(); +} + +void * openssl_secure_malloc(size_t size) +{ + return OPENSSL_secure_malloc(size); +} + +void openssl_secure_free(void * ptr) +{ + OPENSSL_secure_free(ptr); +} + +void openssl_secure_clear(void * ptr, + size_t size) +{ + OPENSSL_cleanse(ptr, size); +} +void openssl_cleanup(void) +{ + OPENSSL_cleanup(); +} diff --git a/src/lib/crypt/openssl.h b/src/lib/crypt/openssl.h new file mode 100644 index 00000000..af285232 --- /dev/null +++ b/src/lib/crypt/openssl.h @@ -0,0 +1,174 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * OpenSSL based cryptographic operations + * Elliptic curve Diffie-Hellman key exchange + * AES encryption + # Authentication + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#ifndef OUROBOROS_LIB_CRYPT_OPENSSL_H +#define OUROBOROS_LIB_CRYPT_OPENSSL_H + +struct ossl_crypt_ctx; + +ssize_t openssl_pkp_create(const char * algo, + EVP_PKEY ** pkp, + uint8_t * pk); + +void openssl_pkp_destroy(EVP_PKEY * pkp); + +int openssl_dhe_derive(EVP_PKEY * pkp, + buffer_t pk, + int kdf_nid, + uint8_t * s); + +ssize_t openssl_kem_encap(buffer_t pk, + uint8_t * ct, + int kdf_nid, + uint8_t * s); + +/* no X509 DER support yet for DHKEM public keys */ +ssize_t openssl_kem_encap_raw(buffer_t pk, + uint8_t * ct, + int kdf_nid, + uint8_t * s); + +int openssl_kem_decap(EVP_PKEY * priv, + buffer_t ct, + int kdf_nid, + uint8_t * s); + +int openssl_get_algo_from_pk_der(buffer_t pk, + char * algo); + +int openssl_get_algo_from_pk_raw(buffer_t pk, + char * algo); + +int openssl_encrypt(struct ossl_crypt_ctx * ctx, + buffer_t in, + buffer_t * out); + +int openssl_decrypt(struct ossl_crypt_ctx * ctx, + buffer_t in, + buffer_t * out); + +struct ossl_crypt_ctx * openssl_crypt_create_ctx(struct crypt_sk * sk); + +void openssl_crypt_destroy_ctx(struct ossl_crypt_ctx * ctx); + +int openssl_crypt_get_ivsz(struct ossl_crypt_ctx * ctx); + +int openssl_crypt_get_tagsz(struct ossl_crypt_ctx * ctx); + +/* AUTHENTICATION */ + +int openssl_load_crt_file(const char * path, + void ** crt); + +int openssl_load_crt_str(const char * str, + void ** crt); + +int openssl_load_crt_der(buffer_t buf, + void ** crt); + +int openssl_get_pubkey_crt(void * crt, + void ** pk); + +void openssl_free_crt(void * crt); + +int openssl_load_privkey_file(const char * path, + void ** key); + +int openssl_load_privkey_str(const char * str, + void ** key); + +int openssl_load_pubkey_file(const char * path, + void ** key); + +int openssl_load_pubkey_str(const char * str, + void ** key); +int openssl_load_pubkey_file_to_der(const char * path, + buffer_t * buf); +int openssl_load_pubkey_raw_file(const char * path, + buffer_t * buf); + +int openssl_load_privkey_raw_file(const char * path, + void ** key); + +int openssl_cmp_key(const EVP_PKEY * key1, + const EVP_PKEY * key2); + +void openssl_free_key(EVP_PKEY * key); + +int openssl_check_crt_name(void * crt, + const char * name); + +int openssl_get_crt_name(void * crt, + char * name); + +int openssl_crt_str(const void * crt, + char * str); + +int openssl_crt_der(const void * crt, + buffer_t * buf); + +void * openssl_auth_create_store(void); + +void openssl_auth_destroy_store(void * store); + +int openssl_auth_add_crt_to_store(void * store, + void * crt); + +int openssl_verify_crt(void * store, + void * crt); + +int openssl_sign(EVP_PKEY * pkp, + int md_nid, + buffer_t msg, + buffer_t * sig); + +int openssl_verify_sig(EVP_PKEY * pk, + int md_nid, + buffer_t msg, + buffer_t sig); + +ssize_t openssl_md_digest(int md_nid, + buffer_t in, + uint8_t * out); + +ssize_t openssl_md_len(int md_nid); + +/* Secure memory allocation */ +int openssl_secure_malloc_init(size_t max, + size_t guard); + +void openssl_secure_malloc_fini(void); + +void * openssl_secure_malloc(size_t size); + +void openssl_secure_free(void * ptr, + size_t size); + +void openssl_secure_clear(void * ptr, + size_t size); + +void openssl_cleanup(void); + +#endif /* OUROBOROS_LIB_CRYPT_OPENSSL_H */ diff --git a/src/lib/dev.c b/src/lib/dev.c index a6be762b..ae0401b7 100644 --- a/src/lib/dev.c +++ b/src/lib/dev.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * API for applications * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -27,85 +27,82 @@ #endif #include "config.h" +#include "ssm.h" -#include <ouroboros/hash.h> -#include <ouroboros/cacep.h> -#include <ouroboros/errno.h> +#include <ouroboros/atomics.h> +#include <ouroboros/bitmap.h> +#include <ouroboros/cep.h> +#include <ouroboros/crc16.h> +#include <ouroboros/crypt.h> #include <ouroboros/dev.h> +#include <ouroboros/endian.h> +#include <ouroboros/errno.h> +#include <ouroboros/fccntl.h> +#include <ouroboros/flow.h> +#include <ouroboros/fqueue.h> +#include <ouroboros/hash.h> +#include <ouroboros/ipcp.h> #include <ouroboros/ipcp-dev.h> +#include <ouroboros/list.h> #include <ouroboros/local-dev.h> -#include <ouroboros/sockets.h> -#include <ouroboros/fccntl.h> -#include <ouroboros/bitmap.h> +#include <ouroboros/np1_flow.h> +#include <ouroboros/pthread.h> #include <ouroboros/random.h> -#include <ouroboros/shm_flow_set.h> -#include <ouroboros/shm_rdrbuff.h> -#include <ouroboros/shm_rbuff.h> +#ifdef PROC_FLOW_STATS +#include <ouroboros/rib.h> +#endif +#include <ouroboros/serdes-irm.h> +#include <ouroboros/sockets.h> +#include <ouroboros/ssm_flow_set.h> +#include <ouroboros/ssm_pool.h> +#include <ouroboros/ssm_rbuff.h> +#include <ouroboros/tw.h> #include <ouroboros/utils.h> -#include <ouroboros/fqueue.h> -#include <stdlib.h> -#include <string.h> -#include <stdio.h> +#include <assert.h> +#ifdef HAVE_LIBGCRYPT +#include <gcrypt.h> +#endif #include <stdarg.h> #include <stdbool.h> +#include <inttypes.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> #include <sys/types.h> #ifndef CLOCK_REALTIME_COARSE #define CLOCK_REALTIME_COARSE CLOCK_REALTIME #endif -/* Partial read information. */ #define NO_PART -1 #define DONE_PART -2 #define CRCLEN (sizeof(uint32_t)) #define SECMEMSZ 16384 -#define SYMMKEYSZ 32 #define MSGBUFSZ 2048 -struct flow_set { - size_t idx; -}; - -struct fqueue { - int fqueue[2 * SHM_BUFFER_SIZE]; /* Safe copy from shm. */ - size_t fqsize; - size_t next; -}; - -enum port_state { - PORT_NULL = 0, - PORT_INIT, - PORT_ID_PENDING, - PORT_ID_ASSIGNED, - PORT_DESTROY -}; - -struct port { +struct fmap { int fd; - - enum port_state state; - pthread_mutex_t state_lock; - pthread_cond_t state_cond; + enum flow_state state; }; -#define frcti_to_flow(frcti) \ - ((struct flow *)((uint8_t *) frcti - offsetof(struct flow, frcti))) - struct flow { - struct shm_rbuff * rx_rb; - struct shm_rbuff * tx_rb; - struct shm_flow_set * set; - int flow_id; - int oflags; - qosspec_t qs; + struct flow_info info; + + struct ssm_rbuff * rx_rb; + struct ssm_rbuff * tx_rb; + struct ssm_flow_set * set; + + uint16_t oflags; ssize_t part_idx; - void * ctx; - uint8_t key[SYMMKEYSZ]; + struct crypt_ctx * crypt; + int headsz; /* IV */ + int tailsz; /* Tag + CRC */ - pid_t pid; + struct timespec snd_act; + struct timespec rcv_act; bool snd_timesout; bool rcv_timesout; @@ -115,232 +112,645 @@ struct flow { struct frcti * frcti; }; -struct { - char * prog; - pid_t pid; +struct flow_set { + size_t idx; + pthread_rwlock_t lock; +}; - struct shm_rdrbuff * rdrb; - struct shm_flow_set * fqset; +struct fqueue { + struct flowevent fqueue[SSM_RBUFF_SIZE]; /* Safe copy from shm. */ + size_t fqsize; + size_t next; +}; - struct timerwheel * tw; +struct { + struct ssm_pool * pool; + struct ssm_flow_set * fqset; struct bmp * fds; struct bmp * fqueues; struct flow * flows; - struct port * ports; + struct fmap * id_to_fd; - pthread_rwlock_t lock; -} ai; + pthread_mutex_t mtx; + pthread_cond_t cond; -#include "frct.c" + pthread_rwlock_t lock; +} proc; -static void port_destroy(struct port * p) +static void flow_destroy(struct fmap * p) { - pthread_mutex_lock(&p->state_lock); + pthread_mutex_lock(&proc.mtx); - if (p->state == PORT_DESTROY) { - pthread_mutex_unlock(&p->state_lock); + if (p->state == FLOW_DESTROY) { + pthread_mutex_unlock(&proc.mtx); return; } - if (p->state == PORT_ID_PENDING) - p->state = PORT_DESTROY; + if (p->state == FLOW_ALLOC_PENDING) + p->state = FLOW_DESTROY; else - p->state = PORT_NULL; + p->state = FLOW_NULL; + + pthread_cond_signal(&proc.cond); - pthread_cond_signal(&p->state_cond); + pthread_cleanup_push(__cleanup_mutex_unlock, &proc.mtx); - while (p->state != PORT_NULL) - pthread_cond_wait(&p->state_cond, &p->state_lock); + while (p->state != FLOW_NULL) + pthread_cond_wait(&proc.cond, &proc.mtx); - p->fd = -1; - p->state = PORT_INIT; + p->fd = -1; + p->state = FLOW_INIT; - pthread_mutex_unlock(&p->state_lock); + pthread_cleanup_pop(true); } -static void port_set_state(struct port * p, - enum port_state state) +static void flow_set_state(struct fmap * p, + enum flow_state state) { - pthread_mutex_lock(&p->state_lock); + pthread_mutex_lock(&proc.mtx); - if (p->state == PORT_DESTROY) { - pthread_mutex_unlock(&p->state_lock); + if (p->state == FLOW_DESTROY) { + pthread_mutex_unlock(&proc.mtx); return; } p->state = state; - pthread_cond_broadcast(&p->state_cond); + pthread_cond_broadcast(&proc.cond); - pthread_mutex_unlock(&p->state_lock); + pthread_mutex_unlock(&proc.mtx); } -static enum port_state port_wait_assign(int flow_id) +static enum flow_state flow_wait_assign(int flow_id) { - enum port_state state; - struct port * p; + enum flow_state state; + struct fmap * p; - p = &ai.ports[flow_id]; + p = &proc.id_to_fd[flow_id]; - pthread_mutex_lock(&p->state_lock); + pthread_mutex_lock(&proc.mtx); - if (p->state == PORT_ID_ASSIGNED) { - pthread_mutex_unlock(&p->state_lock); - return PORT_ID_ASSIGNED; + if (p->state == FLOW_ALLOCATED) { + pthread_mutex_unlock(&proc.mtx); + return FLOW_ALLOCATED; } - if (p->state == PORT_INIT) - p->state = PORT_ID_PENDING; + if (p->state == FLOW_INIT) + p->state = FLOW_ALLOC_PENDING; - while (p->state == PORT_ID_PENDING) - pthread_cond_wait(&p->state_cond, &p->state_lock); + pthread_cleanup_push(__cleanup_mutex_unlock, &proc.mtx); - if (p->state == PORT_DESTROY) { - p->state = PORT_NULL; - pthread_cond_broadcast(&p->state_cond); + while (p->state == FLOW_ALLOC_PENDING) + pthread_cond_wait(&proc.cond, &proc.mtx); + + if (p->state == FLOW_DESTROY) { + p->state = FLOW_NULL; + pthread_cond_broadcast(&proc.cond); } state = p->state; - assert(state != PORT_INIT); + pthread_cleanup_pop(true); - pthread_mutex_unlock(&p->state_lock); + assert(state != FLOW_INIT); return state; } -static int proc_announce(char * prog) +static int proc_announce(const struct proc_info * proc) +{ + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + int err; + + if (proc_announce__irm_req_ser(&msg, proc) < 0) + return -ENOMEM; + + err = send_recv_msg(&msg); + if (err < 0) + return err; + + return irm__irm_result_des(&msg); +} + +/* IRMd cleans up on failure. */ +static void proc_exit(void) +{ + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + + if (proc_exit__irm_req_ser(&msg) < 0) + return; + + send_recv_msg(&msg); +} + +static int spb_encrypt(struct flow * flow, + struct ssm_pk_buff * spb) +{ + buffer_t in; + buffer_t out; + uint8_t * head; + uint8_t * tail; + + if (flow->crypt == NULL) + return 0; + + in.data = ssm_pk_buff_head(spb); + in.len = ssm_pk_buff_len(spb); + + if (crypt_encrypt(flow->crypt, in, &out) < 0) + goto fail_encrypt; + + head = ssm_pk_buff_push(spb, flow->headsz); + if (head == NULL) + goto fail_alloc; + + tail = ssm_pk_buff_push_tail(spb, flow->tailsz); + if (tail == NULL) + goto fail_alloc; + + memcpy(head, out.data, out.len); + + freebuf(out); + + return 0; + fail_alloc: + freebuf(out); + fail_encrypt: + return -ECRYPT; +} + +static int spb_decrypt(struct flow * flow, + struct ssm_pk_buff * spb) +{ + buffer_t in; + buffer_t out; + uint8_t * head; + + if (flow->crypt == NULL) + return 0; + + in.data = ssm_pk_buff_head(spb); + in.len = ssm_pk_buff_len(spb); + + if (crypt_decrypt(flow->crypt, in, &out) < 0) + return -ENOMEM; + + + head = ssm_pk_buff_pop(spb, flow->headsz) + flow->headsz; + ssm_pk_buff_pop_tail(spb, flow->tailsz); + + memcpy(head, out.data, out.len); + + freebuf(out); + + return 0; +} + +/* tw_move under proc.lock rdlock; gates teardown vs in-flight fires. */ +static void tw_move_safe(void) { - irm_msg_t msg = IRM_MSG__INIT; - irm_msg_t * recv_msg; - int ret = -1; + pthread_rwlock_rdlock(&proc.lock); - msg.code = IRM_MSG_CODE__IRM_PROC_ANNOUNCE; - msg.has_pid = true; - msg.pid = ai.pid; - msg.prog = prog; + pthread_cleanup_push(__cleanup_rwlock_unlock, &proc.lock); - recv_msg = send_recv_irm_msg(&msg); - if (recv_msg == NULL) { - return -EIRMD; + tw_move(); + + pthread_cleanup_pop(1); +} + +static int crc_add(struct ssm_pk_buff * spb, + size_t head_skip) +{ + uint8_t * head; + uint8_t * tail; + + tail = ssm_pk_buff_push_tail(spb, CRCLEN); + if (tail == NULL) + return -ENOMEM; + + head = ssm_pk_buff_head(spb) + head_skip; + + mem_hash(HASH_CRC32, tail, head, tail - head); + + return 0; +} + +static int crc_check(struct ssm_pk_buff * spb, + size_t head_skip) +{ + uint32_t crc; + uint8_t * head = ssm_pk_buff_head(spb) + head_skip; + uint8_t * tail = ssm_pk_buff_pop_tail(spb, CRCLEN); + + mem_hash(HASH_CRC32, &crc, head, tail - head); + + return !(crc == *((uint32_t *) tail)); +} + +/* FRCT included here so it can use proc and dev.c statics directly. */ +#include "frct.c" + +/* + * SACK / DATA carry trailer CRC32; HCS protects the headers on every + * FRCT packet. Decrypt before any check so plaintext is authoritative. + */ +static bool invalid_pkt(struct flow * flow, + struct ssm_pk_buff * spb) +{ + const struct frct_pci * pci; + uint16_t flags; + size_t pci_total; + + if (spb == NULL || ssm_pk_buff_len(spb) == 0) + return true; + + if (spb_decrypt(flow, spb) < 0) + return true; + + if (flow->frcti == NULL) { + if (flow->info.qs.ber == 0 && crc_check(spb, 0) != 0) + return true; + return false; } - if (!recv_msg->has_result || (ret = recv_msg->result)) { - irm_msg__free_unpacked(recv_msg, NULL); - return ret; + if (ssm_pk_buff_len(spb) < FRCT_PCILEN) + return true; + + pci = (const struct frct_pci *) ssm_pk_buff_head(spb); + flags = ntoh16(pci->flags); + + /* Untrusted flag read; mismatch on HCS will drop on corrupt. */ + if (flags & FRCT_DATA) + pci_total = frcti_data_hdr_len(flow->frcti); + else + pci_total = frcti_ctrl_hdr_len(flow->frcti); + + if (ssm_pk_buff_len(spb) < pci_total) + return true; + + if (frct_hcs_check(pci, flow->frcti) != 0) + return true; + + /* HCS valid: CRC32 on SACK; or on DATA if ber = 0. */ + if (flags & FRCT_SACK) { + if (crc_check(spb, pci_total) != 0) + return true; + + } else if ((flags & FRCT_DATA) && flow->info.qs.ber == 0) { + if (crc_check(spb, pci_total) != 0) + return true; } - irm_msg__free_unpacked(recv_msg, NULL); + return false; +} - return ret; +static bool deadline_passed(const struct timespec * abs) +{ + struct timespec now; + + if (abs == NULL) + return false; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + return ts_diff_ns(&now, abs) >= 0; } -static void flow_clear(int fd) +/* Clamp the wait by min(dl, next tw expiry, now + TICTIME). */ +static void compute_wait_deadline(const struct timespec * dl, + struct timespec * out) { - memset(&ai.flows[fd], 0, sizeof(ai.flows[fd])); + struct timespec now; + struct timespec cap; + struct timespec expiry; + struct timespec tic = TIMESPEC_INIT_NS(TICTIME); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, &tic, &cap); + + tw_next_expiry(&expiry); - ai.flows[fd].flow_id = -1; - ai.flows[fd].pid = -1; + *out = (ts_diff_ns(&cap, &expiry) < 0) ? expiry : cap; + if (dl != NULL && ts_diff_ns(out, dl) > 0) + *out = *dl; } -#include "crypt.c" +/* + * proc.lock rdlock held across each iteration so flow_fini's wrlock + * waits for us to finish; FLOWDOWN already set means we exit promptly. + */ +static void flow_drain_rx_nb(struct flow * flow) +{ + ssize_t idx; + struct ssm_pk_buff * spb; + struct ssm_rbuff * rx_rb; + struct frcti * frcti; +#ifdef PROC_FLOW_STATS + struct timespec t_a; + struct timespec t_b; +#endif -static void flow_fini(int fd) + if (flow->frcti != NULL) + STAT_BUMP(flow->frcti, drain_calls); + + while (true) { + pthread_rwlock_rdlock(&proc.lock); + + rx_rb = flow->rx_rb; + if (rx_rb == NULL) { + pthread_rwlock_unlock(&proc.lock); + return; + } + + idx = ssm_rbuff_read(rx_rb); + if (idx < 0) { + pthread_rwlock_unlock(&proc.lock); + return; + } + + spb = ssm_pool_get(proc.pool, idx); + if (invalid_pkt(flow, spb)) { + ssm_pool_remove(proc.pool, idx); + pthread_rwlock_unlock(&proc.lock); + continue; + } + + frcti = flow->frcti; + if (frcti != NULL) { +#ifdef PROC_FLOW_STATS + clock_gettime(CLOCK_MONOTONIC, &t_a); + FRCTI_RCV(frcti, spb); + clock_gettime(CLOCK_MONOTONIC, &t_b); + STAT_ADD(frcti, rcv_proc_ns, + (size_t) ts_diff_ns(&t_b, &t_a)); +#else + FRCTI_RCV(frcti, spb); +#endif + } else { + ssm_pool_remove(proc.pool, idx); + } + + pthread_rwlock_unlock(&proc.lock); + + /* Per-packet so the delayed-ACK fires on time in a burst. */ +#ifdef PROC_FLOW_STATS + clock_gettime(CLOCK_MONOTONIC, &t_a); + tw_move_safe(); + clock_gettime(CLOCK_MONOTONIC, &t_b); + if (frcti != NULL) + STAT_ADD(frcti, tw_move_ns, + (size_t) ts_diff_ns(&t_b, &t_a)); +#else + tw_move_safe(); +#endif + } +} + +/* + * Wait clamped by caller deadline, next tw expiry, and TICTIME; + * a clamp-timeout means tw work is due, not caller-deadline. + */ +static int flow_rx_one(struct flow * flow, + struct timespec * abs) { - assert(fd >= 0 && fd < SYS_MAX_FLOWS); + struct timespec wait_abs; + struct ssm_pk_buff * spb; + struct ssm_rbuff * rx_rb; + ssize_t idx; + + while (true) { + compute_wait_deadline(abs, &wait_abs); + + /* rdlock gates flow_fini; FLOWDOWN preempts the block. */ + pthread_rwlock_rdlock(&proc.lock); - if (ai.flows[fd].flow_id != -1) { - port_destroy(&ai.ports[ai.flows[fd].flow_id]); - bmp_release(ai.fds, fd); + rx_rb = flow->rx_rb; + if (rx_rb == NULL) { + pthread_rwlock_unlock(&proc.lock); + return -EFLOWDOWN; + } + + idx = ssm_rbuff_read_b(rx_rb, &wait_abs); + if (idx == -ETIMEDOUT) { + pthread_rwlock_unlock(&proc.lock); + if (deadline_passed(abs)) + return -ETIMEDOUT; + tw_move_safe(); + continue; + } + if (idx < 0) { + pthread_rwlock_unlock(&proc.lock); + return idx; + } + + spb = ssm_pool_get(proc.pool, idx); + if (invalid_pkt(flow, spb)) { + ssm_pool_remove(proc.pool, idx); + pthread_rwlock_unlock(&proc.lock); + continue; + } + + if (flow->frcti != NULL) + FRCTI_RCV(flow->frcti, spb); + else + ssm_pool_remove(proc.pool, idx); + + pthread_rwlock_unlock(&proc.lock); + + tw_move_safe(); + return 0; } +} - if (ai.flows[fd].frcti != NULL) - frcti_destroy(ai.flows[fd].frcti); +/* 0 = window open; -EAGAIN = !block and would block; else flow_rx_one rc. */ +static __inline__ int flow_wait_window(struct flow * flow, + size_t n, + bool block, + struct timespec * dl) +{ + int rc; - if (ai.flows[fd].rx_rb != NULL) { - shm_rbuff_set_acl(ai.flows[fd].rx_rb, ACL_FLOWDOWN); - shm_rbuff_close(ai.flows[fd].rx_rb); + while (true) { + flow_drain_rx_nb(flow); + if (FRCTI_IS_WINDOW_OPEN_N(flow->frcti, n)) + return 0; + if (!block) + return -EAGAIN; + rc = flow_rx_one(flow, dl); + if (rc < 0) + return rc; } +} - if (ai.flows[fd].tx_rb != NULL) { - shm_rbuff_set_acl(ai.flows[fd].tx_rb, ACL_FLOWDOWN); - shm_rbuff_close(ai.flows[fd].tx_rb); +static void flow_clear(int fd) +{ + memset(&proc.flows[fd], 0, sizeof(proc.flows[fd])); + + proc.flows[fd].info.id = -1; +} + +/* + * Set ACL_FLOWDOWN on rx/tx so any in-flight blocking reads or writes + * wake up and drop their proc.lock rdlock. Must run BEFORE flow_fini's + * wrlock, else the wrlock blocks on those rdlock holders and the + * in-flight calls never see the FLOWDOWN signal. + */ +static void flow_quiesce(int fd) +{ + struct ssm_rbuff * rx_rb = proc.flows[fd].rx_rb; + struct ssm_rbuff * tx_rb = proc.flows[fd].tx_rb; + + if (rx_rb != NULL) + ssm_rbuff_set_acl(rx_rb, ACL_FLOWDOWN); + if (tx_rb != NULL) + ssm_rbuff_set_acl(tx_rb, ACL_FLOWDOWN); +} + +static void do_flow_fini(int fd) +{ + assert(fd >= 0 && fd < PROC_MAX_FLOWS); + + if (proc.flows[fd].frcti != NULL) + frcti_destroy(proc.flows[fd].frcti); + + if (proc.flows[fd].info.id != -1) { + flow_destroy(&proc.id_to_fd[proc.flows[fd].info.id]); + bmp_release(proc.fds, fd); } - if (ai.flows[fd].set != NULL) { - shm_flow_set_notify(ai.flows[fd].set, - ai.flows[fd].flow_id, + if (proc.flows[fd].rx_rb != NULL) + ssm_rbuff_close(proc.flows[fd].rx_rb); + + if (proc.flows[fd].tx_rb != NULL) + ssm_rbuff_close(proc.flows[fd].tx_rb); + + if (proc.flows[fd].set != NULL) { + ssm_flow_set_notify(proc.flows[fd].set, + proc.flows[fd].info.id, FLOW_DEALLOC); - shm_flow_set_close(ai.flows[fd].set); + ssm_flow_set_close(proc.flows[fd].set); } - if (ai.flows[fd].ctx != NULL) - crypt_fini(ai.flows[fd].ctx); + crypt_destroy_ctx(proc.flows[fd].crypt); flow_clear(fd); } -static int flow_init(int flow_id, - pid_t pid, - qosspec_t qs, - uint8_t * s) +static void flow_fini(int fd) { - int fd; - int err = -ENOMEM; + flow_quiesce(fd); + + pthread_rwlock_wrlock(&proc.lock); - pthread_rwlock_wrlock(&ai.lock); + do_flow_fini(fd); + + pthread_rwlock_unlock(&proc.lock); +} + +#define IS_ENCRYPTED(crypt) ((crypt)->nid != NID_undef) +#define IS_ORDERED(info) ((info)->qs.service != SVC_RAW) +#define IS_STREAM(info) ((info)->qs.service == SVC_STREAM) + +/* Raw MTU minus the wrapping (IV/Tag + optional CRC) dev.c adds. */ +static __inline__ size_t flow_user_mtu(const struct flow * flow, + size_t raw) +{ + size_t hdr; - fd = bmp_allocate(ai.fds); - if (!bmp_is_id_valid(ai.fds, fd)) { + hdr = flow->headsz + flow->tailsz; + if (flow->info.qs.ber == 0 && flow->crypt == NULL) + hdr += CRCLEN; + + return raw > hdr ? raw - hdr : 0; +} + +static int flow_init(struct flow_info * info, + struct crypt_sk * sk, + time_t rtt_hint) +{ + struct timespec now; + struct flow * flow; + int fd; + int err = -ENOMEM; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + pthread_rwlock_wrlock(&proc.lock); + + fd = bmp_allocate(proc.fds); + if (!bmp_is_id_valid(proc.fds, fd)) { err = -EBADF; goto fail_fds; } - ai.flows[fd].rx_rb = shm_rbuff_open(ai.pid, flow_id); - if (ai.flows[fd].rx_rb == NULL) + flow = &proc.flows[fd]; + + flow->info = *info; + + flow->rx_rb = ssm_rbuff_open(info->n_pid, info->id); + if (flow->rx_rb == NULL) goto fail_rx_rb; - ai.flows[fd].tx_rb = shm_rbuff_open(pid, flow_id); - if (ai.flows[fd].tx_rb == NULL) + flow->tx_rb = ssm_rbuff_open(info->n_1_pid, info->id); + if (flow->tx_rb == NULL) goto fail_tx_rb; - ai.flows[fd].set = shm_flow_set_open(pid); - if (ai.flows[fd].set == NULL) + flow->set = ssm_flow_set_open(info->n_1_pid); + if (flow->set == NULL) goto fail_set; - ai.flows[fd].flow_id = flow_id; - ai.flows[fd].oflags = FLOWFDEFAULT; - ai.flows[fd].pid = pid; - ai.flows[fd].part_idx = NO_PART; - ai.flows[fd].qs = qs; + flow->oflags = FLOWFDEFAULT; + flow->part_idx = NO_PART; + flow->snd_act = now; + flow->rcv_act = now; + flow->crypt = NULL; + flow->headsz = 0; + flow->tailsz = 0; + + if (IS_ENCRYPTED(sk)) { + sk->rot_bit = KEY_ROTATION_BIT; + flow->crypt = crypt_create_ctx(sk); + if (flow->crypt == NULL) + goto fail_crypt; + flow->headsz = crypt_get_ivsz(flow->crypt); + flow->tailsz = crypt_get_tagsz(flow->crypt); + } - if (qs.cypher_s > 0) { - assert(s != NULL); - if (crypt_init(&ai.flows[fd].ctx) < 0) - goto fail_ctx; + assert(flow->frcti == NULL); - memcpy(ai.flows[fd].key, s, SYMMKEYSZ); + if (IS_ORDERED(&flow->info)) { + uint32_t frct_mtu = flow_user_mtu(flow, info->mtu); + + flow->frcti = frcti_create(fd, DELT_A, DELT_R, + info->mpl, rtt_hint, + info->qs, frct_mtu); + if (flow->frcti == NULL) + goto fail_frcti; } - ai.ports[flow_id].fd = fd; + proc.id_to_fd[info->id].fd = fd; - port_set_state(&ai.ports[flow_id], PORT_ID_ASSIGNED); + flow_set_state(&proc.id_to_fd[info->id], FLOW_ALLOCATED); - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); return fd; - fail_ctx: - shm_flow_set_close(ai.flows[fd].set); + fail_frcti: + crypt_destroy_ctx(flow->crypt); + fail_crypt: + ssm_flow_set_close(flow->set); fail_set: - shm_rbuff_close(ai.flows[fd].tx_rb); + ssm_rbuff_close(flow->tx_rb); fail_tx_rb: - shm_rbuff_close(ai.flows[fd].rx_rb); + ssm_rbuff_close(flow->rx_rb); fail_rx_rb: - bmp_release(ai.fds, fd); + bmp_release(proc.fds, fd); fail_fds: - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); return err; } @@ -358,157 +768,206 @@ static void init(int argc, char ** argv, char ** envp) { - const char * prog = argv[0]; - int i; - + struct proc_info info; + char * prog = argv[0]; + int i; +#ifdef PROC_FLOW_STATS + char procstr[32]; +#endif (void) argc; (void) envp; - assert(ai.prog == NULL); - if (check_python(argv[0])) prog = argv[1]; - ai.pid = getpid(); + prog = path_strip(prog); + if (prog == NULL) { + fprintf(stderr, "FATAL: Could not determine program name.\n"); + goto fail_prog; + } + + memset(&info, 0, sizeof(info)); + info.pid = getpid(); + strncpy(info.prog, prog, PROG_NAME_SIZE); + + if (proc_announce(&info)) { + fprintf(stderr, "FATAL: Could not announce to IRMd.\n"); + goto fail_prog; + } + #ifdef HAVE_LIBGCRYPT if (!gcry_control(GCRYCTL_INITIALIZATION_FINISHED_P)) { - if (!gcry_check_version(GCRYPT_VERSION)) - goto fail_fds; + if (!gcry_check_version(GCRYPT_VERSION)) { + fprintf(stderr, "FATAL: Could not get gcry version.\n"); + goto fail_prog; + } gcry_control(GCRYCTL_DISABLE_SECMEM, 0); gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); } #endif - ai.fds = bmp_create(PROG_MAX_FLOWS - PROG_RES_FDS, PROG_RES_FDS); - if (ai.fds == NULL) + proc.fds = bmp_create(PROC_MAX_FLOWS - PROC_RES_FDS, PROC_RES_FDS); + if (proc.fds == NULL) { + fprintf(stderr, "FATAL: Could not create fd bitmap.\n"); goto fail_fds; + } - ai.fqueues = bmp_create(PROG_MAX_FQUEUES, 0); - if (ai.fqueues == NULL) + proc.fqueues = bmp_create(PROC_MAX_FQUEUES, 0); + if (proc.fqueues == NULL) { + fprintf(stderr, "FATAL: Could not create fqueue bitmap.\n"); goto fail_fqueues; + } + + if (is_ouroboros_member_uid(getuid())) + proc.pool = ssm_pool_open(0); + else + proc.pool = ssm_pool_open(getuid()); - ai.rdrb = shm_rdrbuff_open(); - if (ai.rdrb == NULL) + if (proc.pool == NULL) { + fprintf(stderr, "FATAL: Could not open packet buffer.\n"); goto fail_rdrb; + } - ai.flows = malloc(sizeof(*ai.flows) * PROG_MAX_FLOWS); - if (ai.flows == NULL) + proc.flows = malloc(sizeof(*proc.flows) * PROC_MAX_FLOWS); + if (proc.flows == NULL) { + fprintf(stderr, "FATAL: Could not malloc flows.\n"); goto fail_flows; + } - for (i = 0; i < PROG_MAX_FLOWS; ++i) + for (i = 0; i < PROC_MAX_FLOWS; ++i) flow_clear(i); - ai.ports = malloc(sizeof(*ai.ports) * SYS_MAX_FLOWS); - if (ai.ports == NULL) - goto fail_ports; + proc.id_to_fd = malloc(sizeof(*proc.id_to_fd) * SYS_MAX_FLOWS); + if (proc.id_to_fd == NULL) { + fprintf(stderr, "FATAL: Could not malloc id_to_fd.\n"); + goto fail_id_to_fd; + } - if (prog != NULL) { - ai.prog = strdup(path_strip((char *) prog)); - if (ai.prog == NULL) - goto fail_prog; + for (i = 0; i < SYS_MAX_FLOWS; ++i) + proc.id_to_fd[i].state = FLOW_INIT; - if (proc_announce((char *) ai.prog)) - goto fail_announce; + if (pthread_mutex_init(&proc.mtx, NULL)) { + fprintf(stderr, "FATAL: Could not init mutex.\n"); + goto fail_mtx; } - for (i = 0; i < SYS_MAX_FLOWS; ++i) { - ai.ports[i].state = PORT_INIT; - if (pthread_mutex_init(&ai.ports[i].state_lock, NULL)) { - int j; - for (j = 0; j < i; ++j) - pthread_mutex_destroy(&ai.ports[j].state_lock); - goto fail_announce; - } - if (pthread_cond_init(&ai.ports[i].state_cond, NULL)) { - int j; - for (j = 0; j < i; ++j) - pthread_cond_destroy(&ai.ports[j].state_cond); - goto fail_state_cond; - } + if (pthread_cond_init(&proc.cond, NULL) < 0) { + fprintf(stderr, "FATAL: Could not init condvar.\n"); + goto fail_cond; } - if (pthread_rwlock_init(&ai.lock, NULL)) - goto fail_lock; + if (pthread_rwlock_init(&proc.lock, NULL) < 0) { + fprintf(stderr, "FATAL: Could not initialize flow lock.\n"); + goto fail_flow_lock; + } - ai.fqset = shm_flow_set_open(getpid()); - if (ai.fqset == NULL) + proc.fqset = ssm_flow_set_open(getpid()); + if (proc.fqset == NULL) { + fprintf(stderr, "FATAL: Could not open flow set.\n"); goto fail_fqset; + } - if (timerwheel_init() < 0) + if (tw_init() < 0) { + fprintf(stderr, "FATAL: Could not initialize timerwheel.\n"); goto fail_timerwheel; + } + if (crypt_secure_malloc_init(PROC_SECMEM_MAX) < 0) { + fprintf(stderr, "FATAL: Could not init secure malloc.\n"); + goto fail_secmem; + } + +#if defined PROC_FLOW_STATS + if (strstr(argv[0], "ipcpd") == NULL) { + sprintf(procstr, "proc.%d", getpid()); + if (rib_init(procstr) < 0) { + fprintf(stderr, "FATAL: Could not initialize RIB.\n"); + goto fail_rib_init; + } + } +#endif return; +#if defined PROC_FLOW_STATS + fail_rib_init: + crypt_secure_malloc_fini(); +#endif + fail_secmem: + tw_fini(); fail_timerwheel: - shm_flow_set_close(ai.fqset); + ssm_flow_set_close(proc.fqset); fail_fqset: - pthread_rwlock_destroy(&ai.lock); - fail_lock: - for (i = 0; i < SYS_MAX_FLOWS; ++i) - pthread_cond_destroy(&ai.ports[i].state_cond); - fail_state_cond: - for (i = 0; i < SYS_MAX_FLOWS; ++i) - pthread_mutex_destroy(&ai.ports[i].state_lock); - fail_announce: - free(ai.prog); - fail_prog: - free(ai.ports); - fail_ports: - free(ai.flows); + pthread_rwlock_destroy(&proc.lock); + fail_flow_lock: + pthread_cond_destroy(&proc.cond); + fail_cond: + pthread_mutex_destroy(&proc.mtx); + fail_mtx: + free(proc.id_to_fd); + fail_id_to_fd: + free(proc.flows); fail_flows: - shm_rdrbuff_close(ai.rdrb); + ssm_pool_close(proc.pool); fail_rdrb: - bmp_destroy(ai.fqueues); + bmp_destroy(proc.fqueues); fail_fqueues: - bmp_destroy(ai.fds); + bmp_destroy(proc.fds); fail_fds: - fprintf(stderr, "FATAL: ouroboros-dev init failed. " - "Make sure an IRMd is running.\n\n"); - memset(&ai, 0, sizeof(ai)); + memset(&proc, 0, sizeof(proc)); + fail_prog: exit(EXIT_FAILURE); } static void fini(void) { - int i = 0; + int i; - if (ai.fds == NULL) + if (proc.fds == NULL) return; - if (ai.prog != NULL) - free(ai.prog); + /* Wake all in-flight readers/writers BEFORE wrlock acquire. */ + for (i = 0; i < PROC_MAX_FLOWS; ++i) + if (proc.flows[i].info.id != -1) + flow_quiesce(i); - pthread_rwlock_wrlock(&ai.lock); + pthread_rwlock_wrlock(&proc.lock); - for (i = 0; i < PROG_MAX_FLOWS; ++i) { - if (ai.flows[i].flow_id != -1) { + for (i = 0; i < PROC_MAX_FLOWS; ++i) { + struct flow * flow = &proc.flows[i]; + if (flow->info.id != -1) { ssize_t idx; - shm_rbuff_set_acl(ai.flows[i].rx_rb, ACL_FLOWDOWN); - while ((idx = shm_rbuff_read(ai.flows[i].rx_rb)) >= 0) - shm_rdrbuff_remove(ai.rdrb, idx); - flow_fini(i); + while ((idx = ssm_rbuff_read(flow->rx_rb)) >= 0) + ssm_pool_remove(proc.pool, idx); + do_flow_fini(i); } } - shm_flow_set_close(ai.fqset); + pthread_cond_destroy(&proc.cond); + pthread_mutex_destroy(&proc.mtx); - for (i = 0; i < SYS_MAX_FLOWS; ++i) { - pthread_mutex_destroy(&ai.ports[i].state_lock); - pthread_cond_destroy(&ai.ports[i].state_cond); - } + pthread_rwlock_unlock(&proc.lock); - timerwheel_fini(); +#ifdef PROC_FLOW_STATS + rib_fini(); +#endif + crypt_secure_malloc_fini(); + + tw_fini(); + + ssm_flow_set_close(proc.fqset); - shm_rdrbuff_close(ai.rdrb); + pthread_rwlock_destroy(&proc.lock); - free(ai.flows); - free(ai.ports); + free(proc.flows); + free(proc.id_to_fd); - bmp_destroy(ai.fds); - bmp_destroy(ai.fqueues); + ssm_pool_close(proc.pool); - pthread_rwlock_unlock(&ai.lock); + bmp_destroy(proc.fds); + bmp_destroy(proc.fqueues); - pthread_rwlock_destroy(&ai.lock); + proc_exit(); + + memset(&proc, 0, sizeof(proc)); } #if defined(__MACH__) && defined(__APPLE__) @@ -525,309 +984,261 @@ __attribute__((section(FINI_SECTION))) __typeof__(fini) * __fini = fini; int flow_accept(qosspec_t * qs, const struct timespec * timeo) { - irm_msg_t msg = IRM_MSG__INIT; - irm_msg_t * recv_msg; - int fd; - void * pkp; /* public key pair */ - uint8_t s[SYMMKEYSZ]; /* secret key for flow */ - uint8_t buf[MSGBUFSZ]; - int err = -EIRMD; - ssize_t key_len; + struct flow_info flow; + struct crypt_sk crypt; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + uint8_t key[SYMMKEYSZ]; + int fd; + int err; - memset(s, 0, SYMMKEYSZ); +#ifdef QOS_DISABLE_CRC + if (qs != NULL) + qs->ber = 1; +#endif + /* STREAM cannot tolerate loss: drops create silent gaps. */ + if (qs != NULL && qs->service == SVC_STREAM && qs->loss != 0) + return -EINVAL; - msg.code = IRM_MSG_CODE__IRM_FLOW_ACCEPT; - msg.has_pid = true; - msg.pid = ai.pid; + memset(&flow, 0, sizeof(flow)); - if (timeo != NULL) { - msg.has_timeo_sec = true; - msg.has_timeo_nsec = true; - msg.timeo_sec = timeo->tv_sec; - msg.timeo_nsec = timeo->tv_nsec; - } + flow.n_pid = getpid(); + flow.qs = qs == NULL ? qos_raw : *qs; - key_len = crypt_dh_pkp_create(&pkp, buf); - if (key_len < 0) { - err = -ECRYPT; - goto fail_crypt_pkp; - } + if (flow_accept__irm_req_ser(&msg, &flow, timeo)) + return -ENOMEM; - msg.has_pk = true; - msg.pk.data = buf; - msg.pk.len = (uint32_t) key_len; + err = send_recv_msg(&msg); + if (err < 0) + return err; - pthread_cleanup_push(crypt_dh_pkp_destroy, pkp); + crypt.key = key; - recv_msg = send_recv_irm_msg(&msg); + err = flow__irm_result_des(&msg, &flow, &crypt); + if (err < 0) + return err; - pthread_cleanup_pop(false); + /* No RTT in accept; rtt_hint=0 bootstraps from first ACK. */ + fd = flow_init(&flow, &crypt, 0); - if (recv_msg == NULL) - goto fail_recv; + crypt_secure_clear(key, SYMMKEYSZ); - if (!recv_msg->has_result) - goto fail_result; + if (qs != NULL) + *qs = flow.qs; - if (recv_msg->result != 0) { - err = recv_msg->result; - goto fail_result; - } + return fd; +} - if (!recv_msg->has_pid || !recv_msg->has_flow_id || - recv_msg->qosspec == NULL) - goto fail_result; +int flow_alloc(const char * dst, + qosspec_t * qs, + const struct timespec * timeo) +{ + struct flow_info flow; + struct crypt_sk crypt; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + uint8_t key[SYMMKEYSZ]; + int fd; + int err; + struct timespec t0; + struct timespec t1; - if (recv_msg->pk.len != 0 && - crypt_dh_derive(pkp, recv_msg->pk.data, - recv_msg->pk.len, s) < 0) { - err = -ECRYPT; - goto fail_result; - } +#ifdef QOS_DISABLE_CRC + if (qs != NULL) + qs->ber = 1; +#endif + /* STREAM cannot tolerate loss: drops create silent gaps. */ + if (qs != NULL && qs->service == SVC_STREAM && qs->loss != 0) + return -EINVAL; - crypt_dh_pkp_destroy(pkp); + memset(&flow, 0, sizeof(flow)); - fd = flow_init(recv_msg->flow_id, recv_msg->pid, - msg_to_spec(recv_msg->qosspec), s); + flow.n_pid = getpid(); + flow.qs = qs == NULL ? qos_raw : *qs; - irm_msg__free_unpacked(recv_msg, NULL); + if (flow_alloc__irm_req_ser(&msg, &flow, dst, timeo)) + return -ENOMEM; - if (fd < 0) - return fd; + clock_gettime(PTHREAD_COND_CLOCK, &t0); - pthread_rwlock_wrlock(&ai.lock); + err = send_recv_msg(&msg); + if (err < 0) + return err; - assert(ai.flows[fd].frcti == NULL); + clock_gettime(PTHREAD_COND_CLOCK, &t1); - if (ai.flows[fd].qs.in_order != 0) { - ai.flows[fd].frcti = frcti_create(fd); - if (ai.flows[fd].frcti == NULL) { - pthread_rwlock_unlock(&ai.lock); - flow_dealloc(fd); - return -ENOMEM; - } - } + crypt.key = key; - if (qs != NULL) - *qs = ai.flows[fd].qs; + err = flow__irm_result_des(&msg, &flow, &crypt); + if (err < 0) + return err; - pthread_rwlock_unlock(&ai.lock); + fd = flow_init(&flow, &crypt, ts_diff_ns(&t1, &t0)); - return fd; + crypt_secure_clear(key, SYMMKEYSZ); - fail_result: - irm_msg__free_unpacked(recv_msg, NULL); - fail_recv: - crypt_dh_pkp_destroy(pkp); - fail_crypt_pkp: - return err; + if (qs != NULL) + *qs = flow.qs; + + return fd; } -static int __flow_alloc(const char * dst, - qosspec_t * qs, - const struct timespec * timeo, - bool join) +int flow_join(const char * dst, + const struct timespec * timeo) { - irm_msg_t msg = IRM_MSG__INIT; - qosspec_msg_t qs_msg = QOSSPEC_MSG__INIT; - irm_msg_t * recv_msg; - int fd; - void * pkp = NULL; /* public key pair */ - uint8_t s[SYMMKEYSZ]; /* secret key for flow */ - uint8_t buf[MSGBUFSZ]; - int err = -EIRMD; + struct flow_info flow; + struct crypt_sk crypt; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + uint8_t key[SYMMKEYSZ]; + int fd; + int err; - memset(s, 0, SYMMKEYSZ); + memset(&flow, 0, sizeof(flow)); -#ifdef QOS_DISABLE_CRC - if (qs != NULL) - qs->ber = 1; -#endif - msg.code = join ? IRM_MSG_CODE__IRM_FLOW_JOIN - : IRM_MSG_CODE__IRM_FLOW_ALLOC; - msg.dst = (char *) dst; - msg.has_pid = true; - msg.pid = ai.pid; - qs_msg = spec_to_msg(qs); - msg.qosspec = &qs_msg; + flow.n_pid = getpid(); + flow.qs = qos_np1; - if (timeo != NULL) { - msg.has_timeo_sec = true; - msg.has_timeo_nsec = true; - msg.timeo_sec = timeo->tv_sec; - msg.timeo_nsec = timeo->tv_nsec; - } + if (flow_join__irm_req_ser(&msg, &flow, dst, timeo)) + return -ENOMEM; - if (!join && qs != NULL && qs->cypher_s != 0) { - ssize_t key_len; + err = send_recv_msg(&msg); + if (err < 0) + return err; - key_len = crypt_dh_pkp_create(&pkp, buf); - if (key_len < 0) { - err = -ECRYPT; - goto fail_crypt_pkp; - } + crypt.key = key; - msg.has_pk = true; - msg.pk.data = buf; - msg.pk.len = (uint32_t) key_len; - } + err = flow__irm_result_des(&msg, &flow, &crypt); + if (err < 0) + return err; - recv_msg = send_recv_irm_msg(&msg); - if (recv_msg == NULL) - goto fail_send; + fd = flow_init(&flow, &crypt, 0); - if (!recv_msg->has_result) - goto fail_result; + crypt_secure_clear(key, SYMMKEYSZ); - if (recv_msg->result != 0) { - err = recv_msg->result; - goto fail_result; - } + return fd; +} + +#define PKT_BUF_LEN 2048 +int flow_dealloc(int fd) +{ + struct flow_info info; + uint8_t pkt[PKT_BUF_LEN]; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + struct timespec tic = TIMESPEC_INIT_NS(TICTIME); + struct timespec timeo = TIMESPEC_INIT_S(0); + struct flow * flow; + int err; + + if (fd < 0 || fd >= PROC_MAX_FLOWS ) + return -EINVAL; - if (!recv_msg->has_pid || !recv_msg->has_flow_id) - goto fail_result; + memset(&info, 0, sizeof(info)); - if (!join && qs != NULL && qs->cypher_s != 0) { - if (!recv_msg->has_pk || recv_msg->pk.len == 0) { - err = -ECRYPT; - goto fail_result; - } + flow = &proc.flows[fd]; - if (crypt_dh_derive(pkp, recv_msg->pk.data, - recv_msg->pk.len, s) < 0) { - err = -ECRYPT; - goto fail_result; - } + pthread_rwlock_rdlock(&proc.lock); - crypt_dh_pkp_destroy(pkp); + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); + return -ENOTALLOC; } - fd = flow_init(recv_msg->flow_id, recv_msg->pid, - qs == NULL ? qos_raw : *qs, s); + flow->oflags = FLOWFDEFAULT | FLOWFRNOPART; - irm_msg__free_unpacked(recv_msg, NULL); + flow->rcv_timesout = true; + flow->rcv_timeo = tic; - if (fd < 0) - return fd; + pthread_rwlock_unlock(&proc.lock); - pthread_rwlock_wrlock(&ai.lock); + flow_read(fd, buf, SOCK_BUF_SIZE); - assert(ai.flows[fd].frcti == NULL); + pthread_rwlock_rdlock(&proc.lock); - if (ai.flows[fd].qs.in_order != 0) { - ai.flows[fd].frcti = frcti_create(fd); - if (ai.flows[fd].frcti == NULL) { - pthread_rwlock_unlock(&ai.lock); - flow_dealloc(fd); - return -ENOMEM; - } - } + while (FRCTI_LINGERING(flow->frcti)) { + ssize_t ret; - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); - return fd; + ret = flow_read(fd, pkt, PKT_BUF_LEN); - fail_result: - irm_msg__free_unpacked(recv_msg, NULL); - fail_send: - crypt_dh_pkp_destroy(pkp); - fail_crypt_pkp: - return err; -} + pthread_rwlock_rdlock(&proc.lock); -int flow_alloc(const char * dst, - qosspec_t * qs, - const struct timespec * timeo) -{ - return __flow_alloc(dst, qs, timeo, false); -} + if (ret == -EFLOWDOWN) + break; + } -int flow_join(const char * dst, - qosspec_t * qs, - const struct timespec * timeo) -{ - if (qs != NULL && qs->cypher_s != 0) - return -ECRYPT; + timeo.tv_sec = FRCTI_DEALLOC(flow->frcti); - return __flow_alloc(dst, qs, timeo, true); -} + pthread_cleanup_push(__cleanup_rwlock_unlock, &proc.lock); -int flow_dealloc(int fd) -{ - irm_msg_t msg = IRM_MSG__INIT; - irm_msg_t * recv_msg; - struct flow * f; - time_t timeo; + ssm_rbuff_fini(flow->tx_rb); - if (fd < 0 || fd >= SYS_MAX_FLOWS ) - return -EINVAL; + pthread_cleanup_pop(true); - msg.code = IRM_MSG_CODE__IRM_FLOW_DEALLOC; - msg.has_flow_id = true; - msg.has_pid = true; - msg.pid = ai.pid; - msg.has_timeo_sec = true; - msg.has_timeo_nsec = true; - msg.timeo_nsec = 0; + info.id = flow->info.id; + info.n_pid = getpid(); - f = &ai.flows[fd]; - - pthread_rwlock_rdlock(&ai.lock); - - if (f->flow_id < 0) { - pthread_rwlock_unlock(&ai.lock); - return -ENOTALLOC; + if (flow_dealloc__irm_req_ser(&msg, &info, &timeo) < 0) { + err = -ENOMEM; + goto out; } - msg.flow_id = f->flow_id; + err = send_recv_msg(&msg); + if (err < 0) + goto out; - timeo = frcti_dealloc(f->frcti); - while (timeo < 0) { /* keep the flow active for rtx */ - ssize_t ret; - uint8_t buf[128]; - - f->oflags = FLOWFDEFAULT | FLOWFRNOPART; + err = irm__irm_result_des(&msg); - f->rcv_timesout = true; - f->rcv_timeo.tv_sec = -timeo; - f->rcv_timeo.tv_nsec = 0; + out: + flow_fini(fd); - pthread_rwlock_unlock(&ai.lock); + return err; +} - ret = flow_read(fd, buf, 128); +int ipcp_flow_dealloc(int fd) +{ + struct flow_info info; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + struct flow * flow; + int err; - pthread_rwlock_rdlock(&ai.lock); + if (fd < 0 || fd >= PROC_MAX_FLOWS ) + return -EINVAL; - timeo = frcti_dealloc(f->frcti); + flow = &proc.flows[fd]; - if ((ret == -ETIMEDOUT || ret == -EFLOWDOWN) && timeo < 0) - timeo = -timeo; - } + memset(&info, 0, sizeof(info)); - msg.timeo_sec = timeo; + pthread_rwlock_rdlock(&proc.lock); - shm_rbuff_fini(ai.flows[fd].tx_rb); + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); + return -ENOTALLOC; + } - pthread_rwlock_unlock(&ai.lock); + info.id = flow->info.id; + info.n_1_pid = flow->info.n_1_pid; - recv_msg = send_recv_irm_msg(&msg); - if (recv_msg == NULL) - return -EIRMD; + pthread_rwlock_unlock(&proc.lock); - if (!recv_msg->has_result) { - irm_msg__free_unpacked(recv_msg, NULL); - return -EIRMD; + if (ipcp_flow_dealloc__irm_req_ser(&msg, &info) < 0) { + err = -ENOMEM; + goto out; } - irm_msg__free_unpacked(recv_msg, NULL); + err = send_recv_msg(&msg); + if (err < 0) + goto out; - pthread_rwlock_wrlock(&ai.lock); + err = irm__irm_result_des(&msg); + out: flow_fini(fd); - pthread_rwlock_unlock(&ai.lock); - - return 0; + return err; } int fccntl(int fd, @@ -836,6 +1247,7 @@ int fccntl(int fd, { uint32_t * fflags; uint16_t * cflags; + uint16_t csflags; va_list l; struct timespec * timeo; qosspec_t * qs; @@ -843,18 +1255,28 @@ int fccntl(int fd, uint32_t tx_acl; size_t * qlen; struct flow * flow; - - if (fd < 0 || fd >= SYS_MAX_FLOWS) + uint16_t old_acc; + uint16_t new_acc; + size_t max; + size_t * maxp; + size_t rsz; + size_t * rszp; + time_t rto; + time_t * rtop; + int rc; + bool emit_eos = false; + + if (fd < 0 || fd >= PROC_MAX_FLOWS) return -EBADF; - flow = &ai.flows[fd]; + flow = &proc.flows[fd]; va_start(l, cmd); - pthread_rwlock_wrlock(&ai.lock); + pthread_rwlock_wrlock(&proc.lock); - if (flow->flow_id < 0) { - pthread_rwlock_unlock(&ai.lock); + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); va_end(l); return -ENOTALLOC; } @@ -892,30 +1314,43 @@ int fccntl(int fd, goto einval; if (!flow->rcv_timesout) goto eperm; - *timeo = flow->snd_timeo; + *timeo = flow->rcv_timeo; break; case FLOWGQOSSPEC: qs = va_arg(l, qosspec_t *); if (qs == NULL) goto einval; - *qs = flow->qs; + *qs = flow->info.qs; break; case FLOWGRXQLEN: qlen = va_arg(l, size_t *); - *qlen = shm_rbuff_queued(flow->rx_rb); + *qlen = ssm_rbuff_queued(flow->rx_rb); break; case FLOWGTXQLEN: qlen = va_arg(l, size_t *); - *qlen = shm_rbuff_queued(flow->tx_rb); + *qlen = ssm_rbuff_queued(flow->tx_rb); + break; + case FLOWGMTU: + maxp = va_arg(l, size_t *); + if (maxp == NULL) + goto einval; + *maxp = flow_user_mtu(flow, flow->info.mtu); break; case FLOWSFLAGS: + old_acc = flow->oflags & FLOWFACCMODE; flow->oflags = va_arg(l, uint32_t); - rx_acl = shm_rbuff_get_acl(flow->rx_rb); - tx_acl = shm_rbuff_get_acl(flow->rx_rb); - /* - * Making our own flow write only means making the - * the other side of the flow read only. - */ + new_acc = flow->oflags & FLOWFACCMODE; + + /* Defer EOS emit until after proc.lock is dropped: */ + /* frcti_fin_snd may block on shm-pool/tx-rb. */ + if (new_acc == FLOWFRDONLY + && old_acc != FLOWFRDONLY + && flow->frcti != NULL) + emit_eos = true; + + rx_acl = ssm_rbuff_get_acl(flow->rx_rb); + tx_acl = ssm_rbuff_get_acl(flow->tx_rb); + /* Our flow write-only -> peer's read-only. */ if (flow->oflags & FLOWFWRONLY) rx_acl |= ACL_RDONLY; if (flow->oflags & FLOWFRDWR) @@ -924,19 +1359,19 @@ int fccntl(int fd, if (flow->oflags & FLOWFDOWN) { rx_acl |= ACL_FLOWDOWN; tx_acl |= ACL_FLOWDOWN; - shm_flow_set_notify(flow->set, - flow->flow_id, + ssm_flow_set_notify(flow->set, + flow->info.id, FLOW_DOWN); } else { rx_acl &= ~ACL_FLOWDOWN; tx_acl &= ~ACL_FLOWDOWN; - shm_flow_set_notify(flow->set, - flow->flow_id, + ssm_flow_set_notify(flow->set, + flow->info.id, FLOW_UP); } - shm_rbuff_set_acl(flow->rx_rb, rx_acl); - shm_rbuff_set_acl(flow->tx_rb, tx_acl); + ssm_rbuff_set_acl(flow->rx_rb, rx_acl); + ssm_rbuff_set_acl(flow->tx_rb, tx_acl); break; case FLOWGFLAGS: @@ -946,65 +1381,285 @@ int fccntl(int fd, *fflags = flow->oflags; break; case FRCTSFLAGS: - cflags = va_arg(l, uint16_t *); - if (cflags == NULL) - goto einval; + csflags = (uint16_t) va_arg(l, uint32_t); if (flow->frcti == NULL) goto eperm; - frcti_setflags(flow->frcti, *cflags); + frcti_setflags(flow->frcti, csflags); break; case FRCTGFLAGS: - cflags = (uint16_t *) va_arg(l, int *); + cflags = (uint16_t *) va_arg(l, uint32_t *); if (cflags == NULL) goto einval; if (flow->frcti == NULL) goto eperm; *cflags = frcti_getflags(flow->frcti); break; + case FRCTSMAXSDU: + max = va_arg(l, size_t); + if (flow->frcti == NULL) + goto eperm; + if (frcti_set_max_rcv_sdu(flow->frcti, max) < 0) + goto einval; + break; + case FRCTGMAXSDU: + maxp = va_arg(l, size_t *); + if (maxp == NULL) + goto einval; + if (flow->frcti == NULL) + goto eperm; + *maxp = frcti_get_max_rcv_sdu(flow->frcti); + break; + case FRCTSRRINGSZ: + rsz = va_arg(l, size_t); + if (flow->frcti == NULL) + goto eperm; + rc = frcti_set_rcv_ring_sz(flow->frcti, rsz); + if (rc < 0) { + pthread_rwlock_unlock(&proc.lock); + va_end(l); + return rc; + } + break; + case FRCTGRRINGSZ: + rszp = va_arg(l, size_t *); + if (rszp == NULL) + goto einval; + if (flow->frcti == NULL) + goto eperm; + *rszp = frcti_get_rcv_ring_sz(flow->frcti); + break; + case FRCTSRTOMIN: + if (flow->frcti == NULL) + goto eperm; + rto = va_arg(l, time_t); + rc = frcti_set_rto_min(flow->frcti, rto); + if (rc < 0) { + pthread_rwlock_unlock(&proc.lock); + va_end(l); + return rc; + } + break; + case FRCTGRTOMIN: + if (flow->frcti == NULL) + goto eperm; + rtop = va_arg(l, time_t *); + if (rtop == NULL) + goto einval; + *rtop = frcti_get_rto_min(flow->frcti); + break; default: - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); va_end(l); return -ENOTSUP; }; - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); + + if (emit_eos) + frcti_fin_snd(flow->frcti); va_end(l); return 0; einval: - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); va_end(l); return -EINVAL; eperm: - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); va_end(l); return -EPERM; } -static int chk_crc(struct shm_du_buff * sdb) +static int flow_tx_spb(struct flow * flow, + struct ssm_pk_buff * spb, + uint16_t flags, + bool block, + struct timespec * abstime) { - uint32_t crc; - uint8_t * head = shm_du_buff_head(sdb); - uint8_t * tail = shm_du_buff_tail_release(sdb, CRCLEN); + struct timespec now; + ssize_t idx; + size_t pci_total; + int ret; - mem_hash(HASH_CRC32, &crc, head, tail - head); + clock_gettime(PTHREAD_COND_CLOCK, &now); + flow->snd_act = now; - return !(crc == *((uint32_t *) tail)); + idx = ssm_pk_buff_get_off(spb); + + if (ssm_pk_buff_len(spb) > 0) { + if (FRCTI_SND(flow->frcti, spb, flags) < 0) + goto enomem; + + if (flow->info.qs.ber == 0) { + pci_total = flow->frcti != NULL + ? frcti_data_hdr_len(flow->frcti) : 0; + if (crc_add(spb, pci_total) != 0) + goto enomem; + } + + if (spb_encrypt(flow, spb) < 0) + goto enomem; + } + + if (!block) + ret = ssm_rbuff_write(flow->tx_rb, idx); + else + ret = ssm_rbuff_write_b(flow->tx_rb, idx, abstime); + + if (ret < 0) { + ssm_pool_remove(proc.pool, idx); + return ret; + } + + ssm_flow_set_notify(flow->set, flow->info.id, FLOW_PKT); + return 0; + + enomem: + ssm_pool_remove(proc.pool, idx); + return -ENOMEM; } -static int add_crc(struct shm_du_buff * sdb) +/* Per-fragment role for fragment i out of n; n == 1 yields SOLE. */ +static __inline__ uint16_t flow_frag_role(size_t i, size_t n) { - uint8_t * head = shm_du_buff_head(sdb); - uint8_t * tail = shm_du_buff_tail_alloc(sdb, CRCLEN); - if (tail == NULL) - return -1; + if (n == 1) + return FRCT_FR_SOLE; + if (i == 0) + return FRCT_FR_FIRST; + if (i + 1 == n) + return FRCT_FR_LAST; + + return FRCT_FR_MID; +} - mem_hash(HASH_CRC32, tail, head, tail - head); +/* + * Stream-mode write: split buf into chunks of + * (frag_mtu - PCI - PCI_STREAM) bytes; each chunk goes through the + * normal tx path. frcti_snd injects the [start,end) extension and + * advances snd_byte_next under its wrlock. No FFGM/LFGM role bits. + */ +static ssize_t flow_write_stream(struct flow * flow, + const void * buf, + size_t count, + int oflags, + struct timespec * dl) +{ + const uint8_t * src = buf; + size_t payload; + size_t off = 0; + bool block = !(oflags & FLOWFWNOBLOCK); + + if (!FRCTI_IS_FRTX(flow->frcti)) + return -EMSGSIZE; + + payload = FRCTI_PAYLOAD_CAP(flow->frcti); + + while (off < count) { + struct ssm_pk_buff * spb; + uint8_t * ptr; + ssize_t idx; + size_t clen; + int ret; + + ret = flow_wait_window(flow, 1, block, dl); + if (ret < 0) + return off > 0 ? (ssize_t) off : (ssize_t) ret; + + clen = MIN(count - off, payload); + + if (block) + idx = ssm_pool_alloc_b(proc.pool, clen, &ptr, + &spb, dl); + else + idx = ssm_pool_alloc(proc.pool, clen, &ptr, &spb); + if (idx < 0) + return off > 0 ? (ssize_t) off : idx; - return 0; + memcpy(ptr, src + off, clen); + + ret = flow_tx_spb(flow, spb, 0, block, dl); + if (ret < 0) + return off > 0 ? (ssize_t) off : (ssize_t) ret; + + off += clen; + } + + return (ssize_t) count; +} + +/* Per-fragment flow_tx_spb loop. Raw flows refuse; FRCT splits the SDU. */ +static ssize_t flow_write_frag(struct flow * flow, + const void * buf, + size_t count, + int oflags, + struct timespec * dl) +{ + const uint8_t * src = buf; + size_t frag_payload; + size_t n; + size_t off = 0; + size_t i; + int ret; + bool block = !(oflags & FLOWFWNOBLOCK); + + /* Raw flows carry no PCI; cannot fragment. */ + if (flow->frcti == NULL) + return -EMSGSIZE; + + frag_payload = FRCTI_PAYLOAD_CAP(flow->frcti); + + /* Guard the ceil-divide against size_t overflow. */ + if (count > SIZE_MAX - frag_payload + 1) + return -EMSGSIZE; + n = (count + frag_payload - 1) / frag_payload; + + /* SDU larger than the FC window can ever offer would deadlock. */ + if (n > RQ_SIZE) + return -EMSGSIZE; + + /* SDU-atomic FC: wait for n seqnos to avoid overshoot mid-SDU. */ + ret = flow_wait_window(flow, n, block, dl); + if (ret < 0) + return (ssize_t) ret; + + STAT_BUMP(flow->frcti, sdu_snd_frag); + + for (i = 0; i < n; ++i) { + struct ssm_pk_buff * spb; + uint8_t * ptr; + ssize_t idx; + size_t clen; + + clen = (i + 1 == n) ? (count - off) : frag_payload; + + if (block) + idx = ssm_pool_alloc_b(proc.pool, clen, &ptr, + &spb, dl); + else + idx = ssm_pool_alloc(proc.pool, clen, &ptr, &spb); + if (idx < 0) { + if (off > 0) + STAT_BUMP(flow->frcti, sdu_snd_alloc); + return off > 0 ? (ssize_t) off : idx; + } + + memcpy(ptr, src + off, clen); + + ret = flow_tx_spb(flow, spb, flow_frag_role(i, n), + block, dl); + if (ret < 0) { + if (off > 0) + STAT_BUMP(flow->frcti, sdu_snd_tx); + return off > 0 ? (ssize_t) off : (ssize_t) ret; + } + + off += clen; + } + + return (ssize_t) count; } ssize_t flow_write(int fd, @@ -1016,246 +1671,288 @@ ssize_t flow_write(int fd, int ret; int flags; struct timespec abs; - struct timespec * abstime = NULL; - struct timespec tic = {0, TICTIME}; - struct timespec tictime; - struct shm_du_buff * sdb; + struct timespec now; + struct timespec * dl = NULL; + struct ssm_pk_buff * spb; uint8_t * ptr; - if (buf == NULL) - return 0; + if (buf == NULL && count != 0) + return -EINVAL; - if (fd < 0 || fd > PROG_MAX_FLOWS) + if (fd < 0 || fd >= PROC_MAX_FLOWS) return -EBADF; - flow = &ai.flows[fd]; - - clock_gettime(PTHREAD_COND_CLOCK, &abs); + flow = &proc.flows[fd]; - pthread_rwlock_rdlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - if (flow->flow_id < 0) { - pthread_rwlock_unlock(&ai.lock); + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); return -ENOTALLOC; } - ts_add(&tic, &abs, &tictime); + flags = flow->oflags; - if (ai.flows[fd].snd_timesout) { - ts_add(&abs, &flow->snd_timeo, &abs); - abstime = &abs; - } + clock_gettime(PTHREAD_COND_CLOCK, &now); - flags = flow->oflags; + if (flow->snd_timesout) { + ts_add(&now, &flow->snd_timeo, &abs); + dl = &abs; + } - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); if ((flags & FLOWFACCMODE) == FLOWFRDONLY) return -EPERM; - if (flags & FLOWFWNOBLOCK) { - if (!frcti_is_window_open(flow->frcti)) - return -EAGAIN; - idx = shm_rdrbuff_alloc(ai.rdrb, count, &ptr, &sdb); - } else { - while ((ret = frcti_window_wait(flow->frcti, &tictime)) < 0) { - if (ret != -ETIMEDOUT) - return ret; + tw_move_safe(); - if (abstime != NULL && ts_diff_ns(&tictime, &abs) <= 0) - return -ETIMEDOUT; + if (flow->frcti != NULL) { + /* Pump rx_rb so a pure-writer processes ACKs. */ + ret = flow_wait_window(flow, 1, !(flags & FLOWFWNOBLOCK), dl); + if (ret < 0) + return ret; - frcti_tick(flow->frcti); + if (count > 0 && FRCTI_IS_STREAM(flow->frcti)) + return flow_write_stream(flow, buf, count, flags, dl); - ts_add(&tictime, &tic, &tictime); - } - idx = shm_rdrbuff_alloc_b(ai.rdrb, count, &ptr, &sdb, abstime); + if (FRCTI_NEEDS_FRAG(flow->frcti, count)) + return flow_write_frag(flow, buf, count, flags, dl); + } else if (flow->info.mtu > 0 + && count > flow_user_mtu(flow, flow->info.mtu)) { + /* Raw flows carry no PCI; refuse anything > one n-1 frame. */ + return -EMSGSIZE; } + if (flags & FLOWFWNOBLOCK) + idx = ssm_pool_alloc(proc.pool, count, &ptr, &spb); + else + idx = ssm_pool_alloc_b(proc.pool, count, &ptr, &spb, dl); if (idx < 0) return idx; - memcpy(ptr, buf, count); + if (count > 0) + memcpy(ptr, buf, count); - pthread_rwlock_rdlock(&ai.lock); + ret = flow_tx_spb(flow, spb, FRCT_FR_SOLE, + !(flags & FLOWFWNOBLOCK), dl); - if (frcti_snd(flow->frcti, sdb) < 0) - goto enomem; - - if (flow->qs.cypher_s > 0 && crypt_encrypt(flow, sdb) < 0) - goto enomem; + return ret < 0 ? (ssize_t) ret : (ssize_t) count; +} - if (flow->qs.ber == 0 && add_crc(sdb) != 0) - goto enomem; +static ssize_t flow_rx_spb(struct flow * flow, + struct ssm_pk_buff ** spb, + bool block, + struct timespec * abstime) +{ + ssize_t idx; + struct timespec now; - if (flags & FLOWFWNOBLOCK) - ret = shm_rbuff_write(flow->tx_rb, idx); - else - ret = shm_rbuff_write_b(flow->tx_rb, idx, abstime); + idx = block ? ssm_rbuff_read_b(flow->rx_rb, abstime) + : ssm_rbuff_read(flow->rx_rb); + if (idx < 0) + return idx; - if (ret < 0) - shm_rdrbuff_remove(ai.rdrb, idx); - else - shm_flow_set_notify(flow->set, flow->flow_id, FLOW_PKT); + clock_gettime(PTHREAD_COND_CLOCK, &now); + flow->rcv_act = now; - pthread_rwlock_unlock(&ai.lock); + *spb = ssm_pool_get(proc.pool, idx); - return ret < 0 ? (ssize_t) ret : (ssize_t) count; + if (invalid_pkt(flow, *spb)) { + ssm_pool_remove(proc.pool, idx); + return -EAGAIN; + } - enomem: - pthread_rwlock_unlock(&ai.lock); - shm_rdrbuff_remove(ai.rdrb, idx); - return -ENOMEM; + return idx; } -ssize_t flow_read(int fd, - void * buf, - size_t count) +static ssize_t raw_flow_read_pkt(struct flow * flow, + bool block, + struct timespec * dl) { + struct ssm_pk_buff * spb; + struct timespec wait_abs; ssize_t idx; - ssize_t n; - uint8_t * packet; - struct shm_rbuff * rb; - struct shm_du_buff * sdb; - struct timespec abs; - struct timespec tic = {0, TICTIME}; - struct timespec tictime; - struct timespec * abstime = NULL; - struct flow * flow; - bool noblock; - bool partrd; - if (fd < 0 || fd > PROG_MAX_FLOWS) - return -EBADF; + while (true) { + if (!block) { + idx = ssm_rbuff_read(flow->rx_rb); + if (idx < 0) + return -EAGAIN; + } else { + compute_wait_deadline(dl, &wait_abs); + idx = ssm_rbuff_read_b(flow->rx_rb, &wait_abs); + if (idx == -ETIMEDOUT) { + if (deadline_passed(dl)) + return -ETIMEDOUT; + continue; + } + if (idx < 0) + return idx; + } - flow = &ai.flows[fd]; + spb = ssm_pool_get(proc.pool, idx); + if (!invalid_pkt(flow, spb)) + return idx; - clock_gettime(PTHREAD_COND_CLOCK, &abs); + ssm_pool_remove(proc.pool, idx); + if (!block) + return -EAGAIN; + } +} - pthread_rwlock_rdlock(&ai.lock); +static ssize_t deliver_pkt(struct flow * flow, + struct ssm_pk_buff * spb, + ssize_t idx, + void * buf, + size_t count, + bool partrd) +{ + uint8_t * packet = ssm_pk_buff_head(spb); + ssize_t n = ssm_pk_buff_len(spb); - if (flow->part_idx == DONE_PART) { - pthread_rwlock_unlock(&ai.lock); - flow->part_idx = NO_PART; - return 0; - } + assert(n >= 0); - if (flow->flow_id < 0) { - pthread_rwlock_unlock(&ai.lock); - return -ENOTALLOC; + if (n <= (ssize_t) count) { + memcpy(buf, packet, n); + ipcp_spb_release(spb); + if (partrd && n == (ssize_t) count) + flow->part_idx = DONE_PART; + else + flow->part_idx = NO_PART; + + return n; } - rb = flow->rx_rb; - noblock = flow->oflags & FLOWFRNOBLOCK; - partrd = !(flow->oflags & FLOWFRNOPART); + if (partrd) { + memcpy(buf, packet, count); + ssm_pk_buff_pop(spb, n); + flow->part_idx = idx; + return count; + } - ts_add(&tic, &abs, &tictime); + ipcp_spb_release(spb); + return -EMSGSIZE; +} - if (ai.flows[fd].rcv_timesout) { - ts_add(&abs, &flow->rcv_timeo, &abs); - abstime = &abs; +/* Drive frcti_consume until it delivers or errors. */ +static ssize_t flow_read_frcti(struct flow * flow, + void * buf, + size_t count, + bool block, + struct timespec * dl) +{ + struct timespec now; + ssize_t bytes; + int rc; + + while (true) { + flow_drain_rx_nb(flow); + bytes = FRCTI_CONSUME(flow->frcti, buf, count); + if (bytes >= 0) + break; + if (bytes != -EAGAIN) + return bytes; + if (!block) + return -EAGAIN; + rc = flow_rx_one(flow, dl); + if (rc < 0) + return rc; } - idx = flow->part_idx; + clock_gettime(PTHREAD_COND_CLOCK, &now); + flow->rcv_act = now; - if (idx < 0) { - while ((idx = frcti_queued_pdu(flow->frcti)) < 0) { - pthread_rwlock_unlock(&ai.lock); + return bytes; +} - idx = noblock ? shm_rbuff_read(rb) : - shm_rbuff_read_b(rb, &tictime); - if (idx < 0) { - frcti_tick(flow->frcti); +ssize_t flow_read(int fd, + void * buf, + size_t count) +{ + struct flow * flow; + struct ssm_pk_buff * spb; + struct timespec abs; + struct timespec now; + struct timespec * dl = NULL; + ssize_t idx; + bool block; + bool partrd; - if (idx != -ETIMEDOUT) - return idx; + if (fd < 0 || fd >= PROC_MAX_FLOWS) + return -EBADF; - if (abstime != NULL - && ts_diff_ns(&tictime, &abs) <= 0) - return -ETIMEDOUT; + flow = &proc.flows[fd]; - ts_add(&tictime, &tic, &tictime); - pthread_rwlock_rdlock(&ai.lock); - continue; - } + pthread_rwlock_rdlock(&proc.lock); - sdb = shm_rdrbuff_get(ai.rdrb, idx); - if (flow->qs.ber == 0 && chk_crc(sdb) != 0) { - pthread_rwlock_rdlock(&ai.lock); - shm_rdrbuff_remove(ai.rdrb, idx); - continue; - } + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); + return -ENOTALLOC; + } - pthread_rwlock_rdlock(&ai.lock); + if (flow->part_idx == DONE_PART) { + flow->part_idx = NO_PART; + pthread_rwlock_unlock(&proc.lock); + return 0; + } - if (flow->qs.cypher_s > 0 - && crypt_decrypt(flow, sdb) < 0) { - pthread_rwlock_unlock(&ai.lock); - shm_rdrbuff_remove(ai.rdrb, idx); - return -ENOMEM; - } + block = !(flow->oflags & FLOWFRNOBLOCK); + partrd = !(flow->oflags & FLOWFRNOPART); - frcti_rcv(flow->frcti, sdb); - } + if (flow->rcv_timesout) { + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, &flow->rcv_timeo, &abs); + dl = &abs; } - frcti_tick(flow->frcti); - - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); - n = shm_rdrbuff_read(&packet, ai.rdrb, idx); + tw_move_safe(); - assert(n >= 0); + idx = flow->part_idx; + if (idx < 0 && flow->frcti != NULL) + return flow_read_frcti(flow, buf, count, block, dl); - if (n <= (ssize_t) count) { - memcpy(buf, packet, n); - shm_rdrbuff_remove(ai.rdrb, idx); + if (idx < 0) { + idx = raw_flow_read_pkt(flow, block, dl); + if (idx < 0) + return idx; + } - pthread_rwlock_wrlock(&ai.lock); + spb = ssm_pool_get(proc.pool, idx); - flow->part_idx = (partrd && n == (ssize_t) count) ? - DONE_PART : NO_PART; + clock_gettime(PTHREAD_COND_CLOCK, &now); + flow->rcv_act = now; - pthread_rwlock_unlock(&ai.lock); - return n; - } else { - if (partrd) { - memcpy(buf, packet, count); - sdb = shm_rdrbuff_get(ai.rdrb, idx); - shm_du_buff_head_release(sdb, n); - pthread_rwlock_wrlock(&ai.lock); - flow->part_idx = idx; - pthread_rwlock_unlock(&ai.lock); - return count; - } else { - shm_rdrbuff_remove(ai.rdrb, idx); - return -EMSGSIZE; - } - } + return deliver_pkt(flow, spb, idx, buf, count, partrd); } -/* fqueue functions. */ - -struct flow_set * fset_create() +struct flow_set * fset_create(void) { - struct flow_set * set = malloc(sizeof(*set)); + struct flow_set * set; + + set = malloc(sizeof(*set)); if (set == NULL) - return NULL; + goto fail_malloc; - assert(ai.fqueues); + assert(proc.fqueues); - pthread_rwlock_wrlock(&ai.lock); + pthread_rwlock_wrlock(&proc.lock); - set->idx = bmp_allocate(ai.fqueues); - if (!bmp_is_id_valid(ai.fqueues, set->idx)) { - pthread_rwlock_unlock(&ai.lock); - free(set); - return NULL; - } + set->idx = bmp_allocate(proc.fqueues); + if (!bmp_is_id_valid(proc.fqueues, set->idx)) + goto fail_bmp_alloc; - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); return set; + + fail_bmp_alloc: + pthread_rwlock_unlock(&proc.lock); + free(set); + fail_malloc: + return NULL; } void fset_destroy(struct flow_set * set) @@ -1265,22 +1962,22 @@ void fset_destroy(struct flow_set * set) fset_zero(set); - pthread_rwlock_wrlock(&ai.lock); + pthread_rwlock_wrlock(&proc.lock); - bmp_release(ai.fqueues, set->idx); + bmp_release(proc.fqueues, set->idx); - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); free(set); } -struct fqueue * fqueue_create() +struct fqueue * fqueue_create(void) { struct fqueue * fq = malloc(sizeof(*fq)); if (fq == NULL) return NULL; - memset(fq->fqueue, -1, (SHM_BUFFER_SIZE) * sizeof(*fq->fqueue)); + memset(fq->fqueue, -1, SSM_RBUFF_SIZE * sizeof(*fq->fqueue)); fq->fqsize = 0; fq->next = 0; @@ -1297,76 +1994,151 @@ void fset_zero(struct flow_set * set) if (set == NULL) return; - shm_flow_set_zero(ai.fqset, set->idx); + ssm_flow_set_zero(proc.fqset, set->idx); } int fset_add(struct flow_set * set, int fd) { - int ret; - size_t packets; - size_t i; + struct flow * flow; + int ret; - if (set == NULL || fd < 0 || fd > SYS_MAX_FLOWS) + if (set == NULL || fd < 0 || fd >= PROC_MAX_FLOWS) return -EINVAL; - pthread_rwlock_wrlock(&ai.lock); + flow = &proc.flows[fd]; - if (ai.flows[fd].flow_id < 0) { - pthread_rwlock_unlock(&ai.lock); - return -EINVAL; + pthread_rwlock_rdlock(&proc.lock); + + if (flow->info.id < 0) { + ret = -EINVAL; + goto fail; } - ret = shm_flow_set_add(ai.fqset, set->idx, ai.flows[fd].flow_id); + if (flow->frcti != NULL) + ssm_flow_set_del(proc.fqset, 0, flow->info.id); - packets = shm_rbuff_queued(ai.flows[fd].rx_rb); - for (i = 0; i < packets; i++) - shm_flow_set_notify(ai.fqset, ai.flows[fd].flow_id, FLOW_PKT); + ret = ssm_flow_set_add(proc.fqset, set->idx, flow->info.id); + if (ret < 0) + goto fail; - pthread_rwlock_unlock(&ai.lock); + if (ssm_rbuff_queued(flow->rx_rb)) + ssm_flow_set_notify(proc.fqset, flow->info.id, FLOW_PKT); + pthread_rwlock_unlock(&proc.lock); + + return ret; + + fail: + pthread_rwlock_unlock(&proc.lock); return ret; } void fset_del(struct flow_set * set, int fd) { - if (set == NULL || fd < 0 || fd > SYS_MAX_FLOWS) + struct flow * flow; + + if (set == NULL || fd < 0 || fd >= PROC_MAX_FLOWS) return; - pthread_rwlock_rdlock(&ai.lock); + flow = &proc.flows[fd]; - if (ai.flows[fd].flow_id >= 0) - shm_flow_set_del(ai.fqset, set->idx, ai.flows[fd].flow_id); + pthread_rwlock_rdlock(&proc.lock); - pthread_rwlock_unlock(&ai.lock); + if (flow->info.id >= 0) + ssm_flow_set_del(proc.fqset, set->idx, flow->info.id); + + if (flow->frcti != NULL) + ssm_flow_set_add(proc.fqset, 0, flow->info.id); + + pthread_rwlock_unlock(&proc.lock); } bool fset_has(const struct flow_set * set, int fd) { - bool ret = false; + struct flow * flow; + bool ret; - if (set == NULL || fd < 0 || fd > SYS_MAX_FLOWS) + if (set == NULL || fd < 0 || fd >= PROC_MAX_FLOWS) return false; - pthread_rwlock_rdlock(&ai.lock); + flow = &proc.flows[fd]; - if (ai.flows[fd].flow_id < 0) { - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); + + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); return false; } - ret = (shm_flow_set_has(ai.fqset, set->idx, ai.flows[fd].flow_id) == 1); + ret = (ssm_flow_set_has(proc.fqset, set->idx, flow->info.id) == 1); - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); return ret; } +static int fqueue_filter(struct fqueue * fq) +{ + struct ssm_pk_buff * spb; + int fd; + ssize_t idx; + struct frcti * frcti; + int ret = 0; + + /* proc.lock rdlock gates frcti_destroy via flow_fini wrlock. */ + pthread_rwlock_rdlock(&proc.lock); + + while (fq->next < fq->fqsize) { + if (fq->fqueue[fq->next].event != FLOW_PKT) { + ret = 1; + goto out; + } + + fd = proc.id_to_fd[fq->fqueue[fq->next].flow_id].fd; + if (fd < 0) { + ++fq->next; + continue; + } + + frcti = proc.flows[fd].frcti; + if (frcti == NULL) { + ret = 1; + goto out; + } + + if (FRCTI_PDU_READY(frcti)) { + ret = 1; + goto out; + } + + idx = flow_rx_spb(&proc.flows[fd], &spb, false, NULL); + if (idx < 0) + goto out; + + spb = ssm_pool_get(proc.pool, idx); + + FRCTI_RCV(frcti, spb); + + if (FRCTI_PDU_READY(frcti)) { + ret = 1; + goto out; + } + + ++fq->next; + } + + out: + pthread_rwlock_unlock(&proc.lock); + return ret; +} + int fqueue_next(struct fqueue * fq) { - int fd; + int fd; + struct flowevent * e; if (fq == NULL) return -EINVAL; @@ -1374,18 +2146,18 @@ int fqueue_next(struct fqueue * fq) if (fq->fqsize == 0 || fq->next == fq->fqsize) return -EPERM; - pthread_rwlock_rdlock(&ai.lock); - - if (fq->next != 0 && frcti_filter(fq) == 0) { - pthread_rwlock_unlock(&ai.lock); + if (fq->next != 0 && fqueue_filter(fq) == 0) return -EPERM; - } - fd = ai.ports[fq->fqueue[fq->next]].fd; + pthread_rwlock_rdlock(&proc.lock); - fq->next += 2; + e = fq->fqueue + fq->next; - pthread_rwlock_unlock(&ai.lock); + fd = proc.id_to_fd[e->flow_id].fd; + + ++fq->next; + + pthread_rwlock_unlock(&proc.lock); return fd; } @@ -1398,7 +2170,7 @@ enum fqtype fqueue_type(struct fqueue * fq) if (fq->fqsize == 0 || fq->next == 0) return -EPERM; - return fq->fqueue[fq->next - 1]; + return fq->fqueue[(fq->next - 1)].event; } ssize_t fevent(struct flow_set * set, @@ -1406,59 +2178,64 @@ ssize_t fevent(struct flow_set * set, const struct timespec * timeo) { ssize_t ret = 0; - struct timespec tic = {0, TICTIME}; - struct timespec tictime; struct timespec abs; - struct timespec * t = NULL; + struct timespec * dl = NULL; + struct timespec wait_abs; if (set == NULL || fq == NULL) return -EINVAL; if (fq->fqsize > 0 && fq->next != fq->fqsize) - return fq->fqsize; + return 1; - clock_gettime(PTHREAD_COND_CLOCK, &abs); + if (timeo != NULL) { + struct timespec now; + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, timeo, &abs); + dl = &abs; + } - ts_add(&tic, &abs, &tictime); - t = &tictime; + while (ret == 0) { + tw_move_safe(); - if (timeo != NULL) - ts_add(&abs, timeo, &abs); + compute_wait_deadline(dl, &wait_abs); - while (ret == 0) { - ret = shm_flow_set_wait(ai.fqset, set->idx, fq->fqueue, t); + ret = ssm_flow_set_wait(proc.fqset, set->idx, + fq->fqueue, &wait_abs); if (ret == -ETIMEDOUT) { - if (timeo != NULL && ts_diff_ns(t, &abs) < 0) { - fq->fqsize = 0; + if (deadline_passed(dl)) return -ETIMEDOUT; - } ret = 0; - ts_add(t, &tic, t); - pthread_rwlock_rdlock(&ai.lock); - timerwheel_move(); - pthread_rwlock_unlock(&ai.lock); continue; } - fq->fqsize = ret << 1; + fq->fqsize = ret; fq->next = 0; - ret = frcti_filter(fq); + ret = fqueue_filter(fq); } - assert(ret); + assert(ret != 0); return 1; } -/* ipcp-dev functions. */ - -int np1_flow_alloc(pid_t n_pid, - int flow_id, - qosspec_t qs) +int np1_flow_alloc(pid_t n_pid, + int flow_id) { - qs.cypher_s = 0; /* No encryption ctx for np1 */ - return flow_init(flow_id, n_pid, qs, NULL); + struct flow_info flow; + struct crypt_sk crypt = { .nid = NID_undef, .key = NULL }; + + memset(&flow, 0, sizeof(flow)); + + flow.id = flow_id; + flow.n_pid = getpid(); + flow.qs = qos_np1; + flow.mpl = 0; + /* np1 flow: n_1_pid is the upper. */ + flow.n_1_pid = n_pid; + + return flow_init(&flow, &crypt, 0); } int np1_flow_dealloc(int flow_id, @@ -1466,287 +2243,352 @@ int np1_flow_dealloc(int flow_id, { int fd; - /* - * TODO: Don't pass timeo to the IPCP but wait in IRMd. - * This will need async ops, waiting until we bootstrap - * the IRMd over ouroboros. - */ - + /* TODO: wait in IRMd, not here; needs async ops. */ sleep(timeo); - pthread_rwlock_rdlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - fd = ai.ports[flow_id].fd; + fd = proc.id_to_fd[flow_id].fd; - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); return fd; } -int np1_flow_resp(int flow_id) +int np1_flow_resp(int flow_id, + int resp) { int fd; - if (port_wait_assign(flow_id) != PORT_ID_ASSIGNED) + if (resp == 0 && flow_wait_assign(flow_id) != FLOW_ALLOCATED) return -1; - pthread_rwlock_rdlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - fd = ai.ports[flow_id].fd; + fd = proc.id_to_fd[flow_id].fd; - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); return fd; } -int ipcp_create_r(int result) +int ipcp_create_r(const struct ipcp_info * info) { - irm_msg_t msg = IRM_MSG__INIT; - irm_msg_t * recv_msg; - int ret; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + int err; - msg.code = IRM_MSG_CODE__IPCP_CREATE_R; - msg.has_pid = true; - msg.pid = getpid(); - msg.has_result = true; - msg.result = result; + if (ipcp_create_r__irm_req_ser(&msg,info) < 0) + return -ENOMEM; - recv_msg = send_recv_irm_msg(&msg); - if (recv_msg == NULL) - return -EIRMD; + err = send_recv_msg(&msg); + if (err < 0) + return err; - if (!recv_msg->has_result) { - irm_msg__free_unpacked(recv_msg, NULL); - return -1; - } + return irm__irm_result_des(&msg); +} - ret = recv_msg->result; - irm_msg__free_unpacked(recv_msg, NULL); +int ipcp_flow_req_arr(const buffer_t * dst, + qosspec_t qs, + time_t mpl, + uint32_t mtu, + const buffer_t * data) +{ + struct flow_info flow; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + struct crypt_sk crypt; + uint8_t key[SYMMKEYSZ]; + int err; - return ret; -} + memset(&flow, 0, sizeof(flow)); -int ipcp_flow_req_arr(const uint8_t * dst, - size_t len, - qosspec_t qs, - const void * data, - size_t dlen) -{ - irm_msg_t msg = IRM_MSG__INIT; - irm_msg_t * recv_msg; - qosspec_msg_t qs_msg; - int fd; - - assert(dst != NULL); - - msg.code = IRM_MSG_CODE__IPCP_FLOW_REQ_ARR; - msg.has_pid = true; - msg.pid = getpid(); - msg.has_hash = true; - msg.hash.len = len; - msg.hash.data = (uint8_t *) dst; - qs_msg = spec_to_msg(&qs); - msg.qosspec = &qs_msg; - msg.has_pk = true; - msg.pk.data = (uint8_t *) data; - msg.pk.len = dlen; - - recv_msg = send_recv_irm_msg(&msg); - if (recv_msg == NULL) - return -EIRMD; - - if (!recv_msg->has_flow_id || !recv_msg->has_pid) { - irm_msg__free_unpacked(recv_msg, NULL); - return -1; - } + assert(dst != NULL && dst->len != 0 && dst->data != NULL); - if (recv_msg->has_result && recv_msg->result) { - irm_msg__free_unpacked(recv_msg, NULL); - return -1; - } + flow.n_1_pid = getpid(); + flow.qs = qs; + flow.mpl = mpl; + flow.mtu = mtu; - qs.cypher_s = 0; /* No encryption ctx for np1 */ - fd = flow_init(recv_msg->flow_id, recv_msg->pid, qs, NULL); + if (ipcp_flow_req_arr__irm_req_ser(&msg, dst, &flow, data) < 0) + return -ENOMEM; - irm_msg__free_unpacked(recv_msg, NULL); + err = send_recv_msg(&msg); + if (err < 0) + return err; - return fd; -} + crypt.key = key; -int ipcp_flow_alloc_reply(int fd, - int response, - const void * data, - size_t len) -{ - irm_msg_t msg = IRM_MSG__INIT; - irm_msg_t * recv_msg; - int ret; + err = flow__irm_result_des(&msg, &flow, &crypt); + if (err < 0) + return err; - assert(fd >= 0 && fd < SYS_MAX_FLOWS); + /* np1 flows are not encrypted. */ + assert(crypt.nid == NID_undef); - msg.code = IRM_MSG_CODE__IPCP_FLOW_ALLOC_REPLY; - msg.has_flow_id = true; - msg.has_pk = true; - msg.pk.data = (uint8_t *) data; - msg.pk.len = (uint32_t) len; + /* Inverted for np1_flow. */ + flow.n_1_pid = flow.n_pid; + flow.n_pid = getpid(); + flow.mpl = 0; + flow.mtu = 0; + flow.qs = qos_np1; - pthread_rwlock_rdlock(&ai.lock); + crypt.nid = NID_undef; - msg.flow_id = ai.flows[fd].flow_id; + return flow_init(&flow, &crypt, 0); +} - pthread_rwlock_unlock(&ai.lock); +int ipcp_flow_alloc_reply(int fd, + int response, + time_t mpl, + uint32_t mtu, + const buffer_t * data) +{ + struct flow_info flow; + uint8_t buf[SOCK_BUF_SIZE]; + buffer_t msg = {SOCK_BUF_SIZE, buf}; + int err; - msg.has_response = true; - msg.response = response; + assert(fd >= 0 && fd < PROC_MAX_FLOWS); - recv_msg = send_recv_irm_msg(&msg); - if (recv_msg == NULL) - return -EIRMD; + pthread_rwlock_rdlock(&proc.lock); - if (!recv_msg->has_result) { - irm_msg__free_unpacked(recv_msg, NULL); - return -1; - } + flow.id = proc.flows[fd].info.id; - ret = recv_msg->result; + pthread_rwlock_unlock(&proc.lock); - irm_msg__free_unpacked(recv_msg, NULL); + flow.mpl = mpl; + flow.mtu = mtu; - return ret; + if (ipcp_flow_alloc_reply__irm_msg_ser(&msg, &flow, response, data) < 0) + return -ENOMEM; + + err = send_recv_msg(&msg); + if (err < 0) + return err; + + return irm__irm_result_des(&msg); } int ipcp_flow_read(int fd, - struct shm_du_buff ** sdb) + struct ssm_pk_buff ** spb) { - struct flow * flow; - struct shm_rbuff * rb; - ssize_t idx = -1; + struct flow * flow; + ssize_t idx = -1; - assert(fd >= 0 && fd < SYS_MAX_FLOWS); - assert(sdb); + assert(fd >= 0 && fd < PROC_MAX_FLOWS); + assert(spb); - flow = &ai.flows[fd]; + flow = &proc.flows[fd]; - pthread_rwlock_rdlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - assert(flow->flow_id >= 0); + assert(flow->info.id >= 0); - rb = flow->rx_rb; + /* Raw flow: deliver the popped pkt directly (no FRCT rq). */ + if (flow->frcti == NULL) { + pthread_rwlock_unlock(&proc.lock); + idx = flow_rx_spb(flow, spb, false, NULL); + return idx < 0 ? (int) idx : 0; + } - while ((idx = frcti_queued_pdu(flow->frcti)) < 0) { - pthread_rwlock_unlock(&ai.lock); + while (!FRCTI_PDU_READY(flow->frcti)) { + pthread_rwlock_unlock(&proc.lock); - idx = shm_rbuff_read(rb); + idx = flow_rx_spb(flow, spb, false, NULL); if (idx < 0) return idx; - pthread_rwlock_rdlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - *sdb = shm_rdrbuff_get(ai.rdrb, idx); - if (flow->qs.ber == 0 && chk_crc(*sdb) != 0) - continue; - - frcti_rcv(flow->frcti, *sdb); + FRCTI_RCV(flow->frcti, *spb); } - frcti_tick(flow->frcti); - - pthread_rwlock_unlock(&ai.lock); - - *sdb = shm_rdrbuff_get(ai.rdrb, idx); + pthread_rwlock_unlock(&proc.lock); return 0; } int ipcp_flow_write(int fd, - struct shm_du_buff * sdb) + struct ssm_pk_buff * spb) { - struct flow * flow; - int ret; - ssize_t idx; + struct flow * flow; + int ret; - assert(fd >= 0 && fd < SYS_MAX_FLOWS); - assert(sdb); + assert(fd >= 0 && fd < PROC_MAX_FLOWS); + assert(spb); - flow = &ai.flows[fd]; + flow = &proc.flows[fd]; - pthread_rwlock_rdlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - if (flow->flow_id < 0) { - pthread_rwlock_unlock(&ai.lock); + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); return -ENOTALLOC; } if ((flow->oflags & FLOWFACCMODE) == FLOWFRDONLY) { - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); return -EPERM; } - assert(flow->tx_rb); + pthread_rwlock_unlock(&proc.lock); - idx = shm_du_buff_get_idx(sdb); + ret = flow_tx_spb(flow, spb, FRCT_FR_SOLE, true, NULL); - if (frcti_snd(flow->frcti, sdb) < 0) { - pthread_rwlock_unlock(&ai.lock); + return ret; +} + +/* Copy src into dst_pool without consuming src. Caller owns both halves. */ +static int pool_dup_spb(struct ssm_pool * src_pool, + size_t src_off, + struct ssm_pool * dst_pool, + struct ssm_pk_buff ** dst_spb) +{ + struct ssm_pk_buff * src; + uint8_t * ptr; + size_t len; + + src = ssm_pool_get(src_pool, src_off); + len = ssm_pk_buff_len(src); + + if (ssm_pool_alloc(dst_pool, len, &ptr, dst_spb) < 0) return -ENOMEM; + + memcpy(ptr, ssm_pk_buff_head(src), len); + + return 0; +} + +int np1_flow_read(int fd, + struct ssm_pk_buff ** spb, + struct ssm_pool * pool) +{ + struct flow * flow; + ssize_t off; + + assert(fd >= 0 && fd < PROC_MAX_FLOWS); + assert(spb); + + flow = &proc.flows[fd]; + + assert(flow->info.id >= 0); + + pthread_rwlock_rdlock(&proc.lock); + + off = ssm_rbuff_read(flow->rx_rb); + if (off < 0) { + pthread_rwlock_unlock(&proc.lock); + return off; } - if (flow->qs.ber == 0 && add_crc(sdb) != 0) { - pthread_rwlock_unlock(&ai.lock); - shm_rdrbuff_remove(ai.rdrb, idx); - return -ENOMEM; + pthread_rwlock_unlock(&proc.lock); + + if (pool == NULL) { + *spb = ssm_pool_get(proc.pool, off); + } else { + /* Cross-pool copy: PUP -> GSPP */ + if (pool_dup_spb(pool, off, proc.pool, spb) < 0) { + ssm_pool_remove(pool, off); + return -ENOMEM; + } + ssm_pool_remove(pool, off); } - ret = shm_rbuff_write_b(flow->tx_rb, idx, NULL); - if (ret == 0) - shm_flow_set_notify(flow->set, flow->flow_id, FLOW_PKT); - else - shm_rdrbuff_remove(ai.rdrb, idx); + return 0; +} - pthread_rwlock_unlock(&ai.lock); +int np1_flow_write(int fd, + struct ssm_pk_buff * spb, + struct ssm_pool * pool) +{ + struct flow * flow; + struct ssm_pk_buff * dst; + int ret; + size_t off; + size_t dst_off; - assert(ret <= 0); + assert(fd >= 0 && fd < PROC_MAX_FLOWS); + assert(spb); - return ret; + flow = &proc.flows[fd]; + + pthread_rwlock_rdlock(&proc.lock); + + if (flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); + return -ENOTALLOC; + } + + if ((flow->oflags & FLOWFACCMODE) == FLOWFRDONLY) { + pthread_rwlock_unlock(&proc.lock); + return -EPERM; + } + + pthread_rwlock_unlock(&proc.lock); + + off = ssm_pk_buff_get_off(spb); + + if (pool == NULL) { + ret = ssm_rbuff_write_b(flow->tx_rb, off, NULL); + if (ret < 0) + return ret; + ssm_flow_set_notify(flow->set, flow->info.id, FLOW_PKT); + } else { + /* Cross-pool copy: GSPP -> PUP. Src kept on error. */ + if (pool_dup_spb(proc.pool, off, pool, &dst) < 0) + return -ENOMEM; + dst_off = ssm_pk_buff_get_off(dst); + ret = ssm_rbuff_write_b(flow->tx_rb, dst_off, NULL); + if (ret < 0) { + ssm_pool_remove(pool, dst_off); + return ret; + } + ssm_flow_set_notify(flow->set, flow->info.id, FLOW_PKT); + ssm_pool_remove(proc.pool, off); + } + + return 0; } -int ipcp_sdb_reserve(struct shm_du_buff ** sdb, +int ipcp_spb_reserve(struct ssm_pk_buff ** spb, size_t len) { - return shm_rdrbuff_alloc_b(ai.rdrb, len, NULL, sdb, NULL) < 0 ? -1 : 0; + return ssm_pool_alloc_b(proc.pool, len, NULL, spb, NULL) < 0 + ? -1 : 0; } -void ipcp_sdb_release(struct shm_du_buff * sdb) +void ipcp_spb_release(struct ssm_pk_buff * spb) { - shm_rdrbuff_remove(ai.rdrb, shm_du_buff_get_idx(sdb)); + ssm_pool_remove(proc.pool, ssm_pk_buff_get_off(spb)); } int ipcp_flow_fini(int fd) { - struct shm_rbuff * rx_rb; + struct ssm_rbuff * rx_rb; - assert(fd >= 0 && fd < SYS_MAX_FLOWS); + assert(fd >= 0 && fd < PROC_MAX_FLOWS); - pthread_rwlock_rdlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - if (ai.flows[fd].flow_id < 0) { - pthread_rwlock_unlock(&ai.lock); + if (proc.flows[fd].info.id < 0) { + pthread_rwlock_unlock(&proc.lock); return -1; } - shm_rbuff_set_acl(ai.flows[fd].rx_rb, ACL_FLOWDOWN); - shm_rbuff_set_acl(ai.flows[fd].tx_rb, ACL_FLOWDOWN); + ssm_rbuff_set_acl(proc.flows[fd].rx_rb, ACL_FLOWDOWN); + ssm_rbuff_set_acl(proc.flows[fd].tx_rb, ACL_FLOWDOWN); - shm_flow_set_notify(ai.flows[fd].set, - ai.flows[fd].flow_id, + ssm_flow_set_notify(proc.flows[fd].set, + proc.flows[fd].info.id, FLOW_DEALLOC); - rx_rb = ai.flows[fd].rx_rb; + rx_rb = proc.flows[fd].rx_rb; - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); if (rx_rb != NULL) - shm_rbuff_fini(rx_rb); + ssm_rbuff_fini(rx_rb); return 0; } @@ -1754,16 +2596,16 @@ int ipcp_flow_fini(int fd) int ipcp_flow_get_qoscube(int fd, qoscube_t * cube) { - assert(fd >= 0 && fd < SYS_MAX_FLOWS); + assert(fd >= 0 && fd < PROC_MAX_FLOWS); assert(cube); - pthread_rwlock_rdlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - assert(ai.flows[fd].flow_id >= 0); + assert(proc.flows[fd].info.id >= 0); - *cube = qos_spec_to_cube(ai.flows[fd].qs); + *cube = qos_spec_to_cube(proc.flows[fd].info.qs); - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); return 0; } @@ -1772,55 +2614,79 @@ size_t ipcp_flow_queued(int fd) { size_t q; - pthread_rwlock_rdlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - assert(ai.flows[fd].flow_id >= 0); + assert(proc.flows[fd].info.id >= 0); - q = shm_rbuff_queued(ai.flows[fd].tx_rb); + q = ssm_rbuff_queued(proc.flows[fd].tx_rb); - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_unlock(&proc.lock); return q; } -ssize_t local_flow_read(int fd) +int local_flow_transfer(int src_fd, + int dst_fd, + struct ssm_pool * src_pool, + struct ssm_pool * dst_pool) { - ssize_t ret; + struct flow * src_flow; + struct flow * dst_flow; + struct ssm_pk_buff * dst_spb; + struct ssm_pool * sp; + struct ssm_pool * dp; + ssize_t off; + int ret; - assert(fd >= 0); + assert(src_fd >= 0); + assert(dst_fd >= 0); - pthread_rwlock_rdlock(&ai.lock); + src_flow = &proc.flows[src_fd]; + dst_flow = &proc.flows[dst_fd]; - ret = shm_rbuff_read(ai.flows[fd].rx_rb); + sp = src_pool == NULL ? proc.pool : src_pool; + dp = dst_pool == NULL ? proc.pool : dst_pool; - pthread_rwlock_unlock(&ai.lock); + pthread_rwlock_rdlock(&proc.lock); - return ret; -} - -int local_flow_write(int fd, - size_t idx) -{ - struct flow * flow; - int ret; + off = ssm_rbuff_read(src_flow->rx_rb); + if (off < 0) { + pthread_rwlock_unlock(&proc.lock); + return off; + } - assert(fd >= 0); + if (dst_flow->info.id < 0) { + pthread_rwlock_unlock(&proc.lock); + ssm_pool_remove(sp, off); + return -ENOTALLOC; + } - flow = &ai.flows[fd]; + pthread_rwlock_unlock(&proc.lock); - pthread_rwlock_rdlock(&ai.lock); + if (sp == dp) { + /* Same pool: zero-copy */ + ret = ssm_rbuff_write_b(dst_flow->tx_rb, off, NULL); + if (ret < 0) + ssm_pool_remove(sp, off); + else + ssm_flow_set_notify(dst_flow->set, + dst_flow->info.id, FLOW_PKT); + } else { + /* Different pools: single copy */ + if (pool_dup_spb(sp, off, dp, &dst_spb) < 0) { + ssm_pool_remove(sp, off); + return -ENOMEM; + } - if (flow->flow_id < 0) { - pthread_rwlock_unlock(&ai.lock); - return -ENOTALLOC; + ssm_pool_remove(sp, off); + off = ssm_pk_buff_get_off(dst_spb); + ret = ssm_rbuff_write_b(dst_flow->tx_rb, off, NULL); + if (ret < 0) + ssm_pool_remove(dp, off); + else + ssm_flow_set_notify(dst_flow->set, + dst_flow->info.id, FLOW_PKT); } - ret = shm_rbuff_write_b(flow->tx_rb, idx, NULL); - if (ret == 0) - shm_flow_set_notify(flow->set, flow->flow_id, FLOW_PKT); - else - shm_rdrbuff_remove(ai.rdrb, idx); - - pthread_rwlock_unlock(&ai.lock); return ret; } diff --git a/src/lib/frct.c b/src/lib/frct.c index 919e2617..2e8955e3 100644 --- a/src/lib/frct.c +++ b/src/lib/frct.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * - * Flow and Retransmission Control + * Flow and Retransmission Control Task (FRCT) * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -20,198 +20,1807 @@ * Foundation, Inc., http://www.fsf.org/about/contact/. */ -#define DELT_RDV (100 * MILLION) /* ns */ -#define MAX_RDV (1 * BILLION) /* ns */ +/* Included by dev.c; uses dev.c statics (proc, spb_encrypt, ...). */ + +#define DELT_RDV (100 * MILLION) /* ns */ +#define MAX_RDV (1 * BILLION) /* ns */ + +#define MAX_RTO_MUL 8 /* caps the RTO backoff shift */ +#define MAX_TLP_PER_EP 2 /* RFC 8985 §7.3: up to 2 TLPs */ +#define INITIAL_RTO (1 * BILLION) /* RFC 6298 §2.1: 1 s default */ +#define RTT_BOOT_NS (10 * MILLION) /* rtt_hint floor + initial mdev */ +#define SRTT_FLOOR_NS 1000L /* 1 us; smoothed RTT floor */ +#define MDEV_FLOOR_NS 100L /* 100 ns; mdev sanity floor */ +#define RTT_CLAMP_MUL 16 /* probe sample cap = N * srtt */ +#define MIN_RTT_WIN_NS (300ULL * BILLION) /* 5 min, Linux tcp default */ +#define NACK_COOLDOWN_NS (100 * MILLION) /* pre-DRF NACK cooldown */ +#define FRCT_TX_TIMEO_NS (250 * 1000) /* tx ring write deadline */ +#define ACK_DELAY_NS (2ULL * TICTIME) /* delayed-ACK fire delay */ + +#define FRCT "frct" +#define FRCT_PCILEN (sizeof(struct frct_pci)) +#define FRCT_NAME_STRLEN 32 + +/* Wire-protocol cap on SACK blocks per packet; binds both peers. */ +#define SACK_MAX_BLOCKS 2048 +#define SACK_BLOCK_SIZE (2 * sizeof(uint32_t)) +/* 2B count + 2B pad to 4-byte align the block list. */ +#define SACK_HDR_SIZE (sizeof(uint32_t)) +#define SACK_MIN_GAP_NS (250u * 1000u) /* 250 us SACK gap */ +#define MIN_REORDER_NS (250u * 1000u) /* 250 us RACK floor */ +#define SACK_RXM_MAX 32 /* Cap on retransmits staged from single SACK.*/ +#define DUP_THRESH 3 /* RFC 8985 §6.2 step 2.2 SACK count gate. */ + +/* RFC 8985 §7.2 RACK reorder-window scaling cap. */ +#define REO_WND_MULT_MAX 20 +/* RFC 8985 §7.2 step 5: round trips of no DSACK before halving. */ +#define REO_DECAY_PKTS 16 +/* DSACK seqno sanity: reject reports older/farther than one rcv window. */ +#define MAX_DSACK_LAG RQ_SIZE + +/* Signed ns elapsed; negative under concurrent update (no underflow). */ +static __inline__ int64_t ts_age_ns(uint64_t now_ns, + uint64_t then_ns) +{ + return (int64_t)(now_ns - then_ns); +} -#define FRCT_PCILEN (sizeof(struct frct_pci)) +/* True iff strictly more than thr_ns elapsed since then_ns. */ +static __inline__ bool ts_aged_ns(uint64_t now_ns, + uint64_t then_ns, + uint64_t thr_ns) +{ + return ts_age_ns(now_ns, then_ns) > (int64_t) thr_ns; +} -struct frct_cr { - uint32_t lwe; /* Left window edge */ - uint32_t rwe; /* Right window edge */ +/* FRCT r-timer: do not retransmit packet older than t_r (from first send). */ +#define RXM_AGED_OUT(t0, now_ns, t_r) \ + ts_aged_ns((now_ns), (t0), (uint64_t)(t_r)) - uint8_t cflags; - uint32_t seqno; /* SEQ to send, or last SEQ Ack'd */ +/* FRCT a-timer: do not (re)transmit ACK after t_a from last data receive. */ +#define ACK_AGED_OUT(act, now_ns, t_a) \ + ts_aged_ns((now_ns), (act), (uint64_t)(t_a)) - struct timespec act; /* Last seen activity */ - time_t inact; /* Inactivity (s) */ +struct sack_args { + uint16_t n; + bool dsack; /* RFC 2883: block[0] is a DSACK report */ + uint32_t ack; + uint32_t rwe; + uint32_t blocks[][2]; /* flexible — sized at alloc time */ }; -struct frcti { - int fd; +/* NewReno-careful (RFC 6582) exit pad; gates RTT samples post-signal. */ +#define RTT_QUARANTINE 32 +#define RTTP_NONCE_LEN 16 - time_t mpl; - time_t a; - time_t r; - time_t rdv; - - time_t srtt; /* Smoothed rtt */ - time_t mdev; /* Deviation */ - time_t rto; /* Retransmission timeout */ - uint32_t rttseq; - struct timespec t_probe; /* Probe time */ - bool probe; /* Probe active */ - - struct frct_cr snd_cr; - struct frct_cr rcv_cr; - - ssize_t rq[RQ_SIZE]; - pthread_rwlock_t lock; - - bool open; /* Window open/closed */ - struct timespec t_wnd; /* Window closed time */ - struct timespec t_rdvs; /* Last rendez-vous sent */ - pthread_cond_t cond; - pthread_mutex_t mtx; -}; +/* RTT-probe wire payload (after the FRCT PCI). */ +struct frct_rttp { + uint32_t probe_id; /* sender counter; 0 on reply */ + uint32_t echo_id; /* peer's probe_id; 0 outbound */ + uint8_t nonce[RTTP_NONCE_LEN]; /* random; echoed verbatim */ +} __attribute__((packed)); + +#define RTTP_PAYLOAD sizeof(struct frct_rttp) +#define RTTP_POS(id) ((id) & (RTTP_RING - 1)) +/* + * Flag values are assigned MSB-first on the wire (RFC convention): + * bit 0 = 0x8000 occupies wire-position 0 of the 16-bit flags + * field, bit 12 = 0x0008 is the last assigned bit, and the three + * LSBs (0x0007) are reserved. + */ enum frct_flags { - FRCT_DATA = 0x01, /* PDU carries data */ - FRCT_DRF = 0x02, /* Data run flag */ - FRCT_ACK = 0x04, /* ACK field valid */ - FRCT_FC = 0x08, /* FC window valid */ - FRCT_RDVS = 0x10, /* Rendez-vous */ - FRCT_FFGM = 0x20, /* First Fragment */ - FRCT_MFGM = 0x40, /* More fragments */ + FRCT_DATA = 0x8000, /* PDU carries data */ + FRCT_DRF = 0x4000, /* Data run flag */ + FRCT_ACK = 0x2000, /* ACK field valid */ + FRCT_NACK = 0x1000, /* Neg-ACK: pci->seqno is arrival_seqno - 1 */ + FRCT_FC = 0x0800, /* FC window valid */ + FRCT_RDVS = 0x0400, /* Rendez-vous */ + FRCT_FFGM = 0x0200, /* First fragment (begin) */ + FRCT_LFGM = 0x0100, /* Last fragment (end) */ + FRCT_RXM = 0x0080, /* Retransmission */ + FRCT_SACK = 0x0040, /* SACK block list follows */ + FRCT_RTTP = 0x0020, /* RTT probe / echo */ + FRCT_KA = 0x0010, /* Keepalive */ + FRCT_FIN = 0x0008, /* End of stream */ }; -struct frct_pci { - uint8_t flags; +/* + * DATA-packet fragment role (FFGM = begin, LFGM = end), SCTP-style: + * 1 1 = sole / un-fragmented SDU (begin AND end) + * 1 0 = first fragment of a multi-fragment SDU + * 0 0 = middle fragment + * 0 1 = last fragment + */ +#define FRCT_FR_MASK (FRCT_FFGM | FRCT_LFGM) +#define FRCT_FR_SOLE (FRCT_FFGM | FRCT_LFGM) +#define FRCT_FR_FIRST (FRCT_FFGM) +#define FRCT_FR_MID (0) +#define FRCT_FR_LAST (FRCT_LFGM) + +/* Default cap on a single reassembled SDU. App can raise via FRCTSMAXSDU */ +#define FRCT_MAX_SDU (1U << 20) + +/* Stream-mode PCI extension: [start, end) byte range on every DATA pkt. */ +struct frct_pci_stream { + uint32_t start; + uint32_t end; +} __attribute__((packed)); + +#define FRCT_PCI_STREAM_LEN (sizeof(struct frct_pci_stream)) + +/* Bytes following PCI: SACK list / RTTP nonce / control payload. */ +#define FRCT_BODY(pci) ((uint8_t *) (pci) + FRCT_PCILEN) +/* Typed access to the stream PCI extension on stream DATA packets. */ +#define FRCT_SPCI(pci) \ + ((struct frct_pci_stream *) ((uint8_t *) (pci) + FRCT_PCILEN)) + +/* Push the FRCT header onto spb's head. */ +#define FRCT_HDR_PUSH(spb, frcti) \ + ((struct frct_pci *) ssm_pk_buff_push((spb), \ + frcti_data_hdr_len(frcti))) - uint8_t pad; /* 24 bit window! */ - uint16_t window; +/* Pop a fixed-size header off spb's head; cast to type *. */ +#define FRCT_HDR_POP(spb, type) \ + ((struct type *) ssm_pk_buff_pop((spb), sizeof(struct type))) +/* Default / max per-flow stream rx ring (pow2); min N * per_pkt. */ +#define FRCT_STREAM_RING_MIN_PKTS 4 +#define FRCT_STREAM_RING_SZ (1U << 20) /* 1 MiB default */ +#define FRCT_STREAM_RING_SZ_MAX (1U << 27) /* 128 MiB */ + +struct frct_pci { + uint16_t flags; + uint16_t hcs; + + uint32_t window; uint32_t seqno; uint32_t ackno; } __attribute__((packed)); -static bool before(uint32_t seq1, - uint32_t seq2) +/* Stat counters; fold to no-ops without PROC_FLOW_STATS. */ +#ifdef PROC_FLOW_STATS +struct frcti_stat { + size_t rxm_rto; /* RTO-timer driven retransmits */ + size_t rxm_rcv; /* RXM packets received (all) */ + size_t rxm_dup_rcv; /* RXM dups (peer already had it) */ + size_t rxm_sack; /* SACK-mechanism retransmits */ + size_t rxm_rack; /* RACK-driven retransmits */ + size_t rxm_dupthresh; /* DupThresh-driven retransmits */ + size_t rxm_nack; /* NACK-pulled retransmits */ + size_t rxm_due_count; /* rxm_due entries (pre-bail) */ + size_t rxm_due_acked; /* bail: seqno < snd_lwe */ + size_t rxm_due_unowned; /* bail: slot.rxm replaced */ + size_t rxm_due_aged; /* bail: r->t0 + t_r < now */ + size_t rxm_due_defer; /* bail: non-HoL, deferred to HoL */ + size_t rxm_arm_fail; /* rxm_arm: malloc failed */ + size_t rxm_cancel; /* entries cancelled at teardown */ + size_t rxm_tx_dead; /* RXM tx into terminal flow */ + size_t tx_drop; /* frct_tx fail (any cause) */ + size_t tx_drop_ack; /* bare ACK dropped */ + size_t tx_drop_sack; /* SACK dropped */ + size_t tx_drop_ka; /* keepalive dropped */ + size_t tx_drop_rttp; /* RTT probe/echo dropped */ + size_t tx_drop_nack; /* pre-DRF NACK dropped */ + size_t tx_drop_rdv; /* rendez-vous dropped */ + size_t tx_drop_other; /* anything not matched above */ + size_t ack_snd; /* ACK packets sent (bare + SACK) */ + size_t ack_fire; /* delayed-ACK timer fires */ + size_t ack_supp_seqno; /* fire suppressed: seqno */ + size_t ack_supp_inact; /* fire suppressed: inact */ + size_t ack_supp_rate; /* fire suppressed: rate */ + size_t ack_rcv; /* ACK packets received */ + size_t ack_rtt; /* ACKs that fed RTT estimator */ + size_t ack_dup_rcv; /* ACK packet wire dups dropped */ + size_t dup_rcv; /* duplicates received */ + size_t out_rcv; /* pkts out of window */ + size_t rqo_rcv; /* pkts out of rqueue */ + size_t ooo_rcv; /* OOO arrivals */ + size_t sack_snd; /* SACK packets sent */ + size_t sack_rcv; /* SACK packets received */ + size_t dsack_snd; /* SACK pkts carrying a DSACK */ + size_t dsack_rcv; /* DSACK blocks parsed */ + size_t dsack_drop; /* DSACK blocks past MAX_DSACK_LAG */ + size_t nack_snd; /* pre-DRF NACKs sent */ + size_t nack_rcv; /* pre-DRF NACKs received */ + size_t tlp_snd; /* tail loss probes sent */ + size_t inact_drop; /* inactivity drop (NACK on cd) */ + size_t drf_rebase; /* DRF-triggered window rebase */ + size_t rq_released; /* slots cleared by release_rq */ + size_t rttp_snd; /* RTT probes sent */ + size_t rttp_rcv; /* RTT probe replies rcvd */ + size_t rtt_smpl; /* RTT estimator samples */ + size_t rdv_snd; /* rendez-vous packets sent */ + size_t rdv_rcv; /* rendez-vous packets rcvd */ + size_t ka_snd; /* keepalives sent */ + size_t ka_rcv; /* keepalives received */ + size_t sdu_snd_frag; /* writes that fragmented */ + size_t sdu_snd_alloc; /* alloc fail truncated SDU send */ + size_t sdu_snd_tx; /* tx fail truncated SDU send */ + size_t frag_snd; /* fragments sent: FIRST/MID/LAST */ + size_t frag_rcv; /* fragments stashed in rq[] */ + size_t sdu_reasm; /* SDUs delivered reassembled */ + size_t sdu_sole; /* SOLE SDUs delivered (n==1) */ + size_t frag_drop; /* dropped at malformed run */ + size_t strm_snd_byte; /* bytes sent on stream */ + size_t strm_rcv_byte; /* bytes copied to ring */ + size_t strm_dlv_byte; /* bytes delivered to reader */ + size_t strm_drop; /* stream rcvs dropped */ + size_t strm_fin_drop; /* stream FIN packets rejected */ + /* Profiling instrumentation. */ + size_t rcv_proc_ns; /* time inside FRCTI_RCV (ns) */ + size_t tw_move_ns; /* time inside tw_move (ns) */ + size_t drain_calls; /* flow_drain_rx_nb invocations */ +}; + +#define STAT_BUMP(frcti, field) FETCH_ADD_RELAXED(&(frcti)->stat.field, 1) +#define STAT_ADD(frcti, field, v) FETCH_ADD_RELAXED(&(frcti)->stat.field, (v)) +#define STAT_LOAD(frcti, field) LOAD_RELAXED(&(frcti)->stat.field) +#else +#define STAT_BUMP(frcti, field) ((void) (frcti)) +#define STAT_ADD(frcti, field, v) ((void) (frcti)) +#define STAT_LOAD(frcti, field) ((void) (frcti), (size_t) 0) +#endif + +#define frcti_to_flow(f) (&proc.flows[(f)->fd]) + +#define RTTP_RING 8 +#define RTTP_COLD_NS (100 * MILLION) /* cold-probe cadence */ +#define RQ_SLOT(seqno) ((seqno) & (RQ_SIZE - 1)) + +struct rxm_entry; + +enum snd_slot_flags { + SND_RTX = 0x01, /* Any retransmit; Karn skips next RTT sample. */ + SND_FAST_RXM = 0x02, /* Fast-retx one-shot gate per loss event. */ + SND_TLP = 0x04, /* Tail loss probe; ACK resets rto_mul. */ +}; + +struct snd_slot { + struct rxm_entry * rxm; /* RXM entry, NULL if none. */ + uint64_t time; /* ts_to_ns of last send (any kind). */ + uint8_t flags; /* SND_* bits above. */ +}; + +/* Per-seqno reorder slot (FRTX) and stream-mode byte/FIN metadata. */ +struct rcv_slot { + ssize_t idx; /* spb idx; -1 = empty */ + uint32_t start; /* stream byte start */ + uint32_t end; /* stream byte end */ + uint8_t fin; /* stream FIN bit */ +}; + +struct frct_cr { + uint32_t lwe; /* Left window edge */ + uint32_t rwe; /* Right window edge */ + + uint8_t cflags; + uint32_t seqno; /* SEQ to send, or last SEQ Ack'd */ + uint32_t ackno; /* snd: ACK-pkt seqno; rcv: dedup */ + + uint64_t act; /* ts_to_ns of last activity */ + uint64_t inact; /* Inactivity threshold (ns) */ +}; + +struct frcti { + /* IMM: set once in frcti_create; read-only thereafter. */ + int fd; + uint64_t t_mpl; /* MPL (ns) */ + uint64_t t_a; /* a-timer (ns) */ + uint64_t t_r; /* r-timer (ns) */ + uint64_t t_rdv; /* RDV cooldown (ns) */ + time_t ber; /* cached qs.ber */ + bool lossy; /* qs.loss != 0 */ + time_t qs_timeout; /* cached qs.timeout (ms) */ + size_t frag_mtu; /* max FRCT pkt: PCI + payload */ + uint16_t sack_n_max; /* SACK blocks that fit MTU */ + bool stream; + + /* All fields below are protected by lock (rwlock/LOAD_ACQUIRE). */ + struct { + struct frct_cr snd_cr; + struct frct_cr rcv_cr; + + /* RTT/RACK estimator */ + time_t srtt; /* smoothed RTT */ + time_t mdev; /* mean deviation */ + time_t min_rtt; /* RACK base, ns */ + uint64_t t_min_rtt; /* min_rtt last set */ + time_t rto; /* retransmit TO */ + time_t rto_min; /* RTO floor (ns) */ + uint8_t rto_mul; /* RTO backoff bits */ + uint32_t rtt_lwe; /* RTT-sample fence */ + uint64_t t_rcv_rtt; /* last RTT feed */ + uint64_t t_snd_probe; /* last probe sent */ + uint64_t t_latest_ack; /* RACK.fack snd-ts */ + uint32_t probe_id_next; + struct { + uint32_t id; + uint64_t ts; /* ts_to_ns send */ + uint8_t nonce[RTTP_NONCE_LEN]; /* echoed back */ + } probes[RTTP_RING]; + + /* rcv reassembly */ + size_t max_rcv_sdu; /* max reasm bytes */ + uint8_t * rcv_ring; /* lazy alloc */ + size_t rcv_ring_sz; /* power of 2 */ + uint32_t ring_seq_cap; /* ring/per_pkt */ + + uint32_t snd_byte_next; + bool snd_fin_sent; + uint32_t snd_fin_seqno; + uint32_t rcv_byte_next; + uint32_t rcv_byte_high; /* contiguous high */ + uint32_t rcv_byte_fin; /* set when FIN */ + bool rcv_fin_seen; + + struct rcv_slot rcv_slots[RQ_SIZE]; + struct snd_slot snd_slots[RQ_SIZE]; /* .rxm is ATOM */ + + /* rcv SACK dedup */ + uint64_t t_snd_sack; + uint32_t sack_lwe; /* rcv lwe at SACK */ + uint16_t sack_n; /* SACK block count */ + + /* RFC 2883 D-SACK: pending report (single-slot, latest). */ + uint32_t dsack_seqno; + bool dsack_valid; + + /* RFC 8985 §7.2 RACK reorder-window scaling. */ + uint8_t reo_wnd_mult; /* REO_WND_MULT_MAX */ + uint32_t dsack_lwe_snap; /* lwe @ last DSACK */ + uint64_t t_last_reo_widen; /* once-per-RTT */ + + uint32_t dup_thresh; /* RFC 8985 */ + uint32_t tlp_high_seq; /* §7.3: 0 = none */ + uint8_t tlp_count; /* §7.3 per-episode */ + uint64_t t_nack; + bool open; /* FC window state */ + bool in_recovery; + uint32_t recovery_high; /* seqno @ entry */ + uint32_t rack_fired_lwe; /* lwe @ last RACK */ + struct timespec t_wnd; /* window-closed ts */ + struct timespec t_last_rdv; /* last RDV sent */ + struct list_head rxm_list; /* live rxm entries */ + + pthread_rwlock_t lock; + }; + + /* Read/written via __atomic without holding lock. */ + uint64_t t_ka_rcv; /* ts_to_ns of last KA rx */ + uint8_t ack_pending; /* delayed-ACK dedup */ + uint8_t tlp_pending; /* TLP arm dedup (lazy) */ + + /* Timer entries; ownership belongs to the tw module. */ + struct tw_entry ack_tw; /* delayed-ACK timer */ + struct tw_entry ka_tw; /* keepalive timer */ + struct tw_entry tlp_tw; /* tail-loss probe timer */ + +#ifdef PROC_FLOW_STATS + /* STAT: lock-free relaxed atomic counters. */ + struct frcti_stat stat; +#endif +}; + +#ifdef PROC_FLOW_STATS + +__attribute__((cold)) +static int frct_rib_read(const char * path, + char * buf, + size_t len) +{ + struct frcti * frcti; + struct timespec now; + uint64_t now_ns; + char * entry; + int fd; + int written; + /* Snapshot under the locks; format outside (pure userspace). */ + struct { + uint64_t t_mpl; + uint64_t t_a; + uint64_t t_r; + time_t srtt; + time_t mdev; + time_t rto; + time_t min_rtt; + struct frct_cr snd_cr; + struct frct_cr rcv_cr; + size_t rx_q_now; + size_t tx_q_now; + struct frcti_stat stat; + } s; + + entry = strstr(path, RIB_SEPARATOR); + assert(entry); + *entry = '\0'; + + fd = atoi(path); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + if (fd < 0 || fd >= PROC_MAX_FLOWS) + return 0; + + pthread_rwlock_rdlock(&proc.lock); + + frcti = proc.flows[fd].frcti; + if (frcti == NULL) { + pthread_rwlock_unlock(&proc.lock); + return 0; + } + + s.t_mpl = frcti->t_mpl; + s.t_a = frcti->t_a; + s.t_r = frcti->t_r; + + s.rx_q_now = proc.flows[fd].rx_rb != NULL + ? ssm_rbuff_queued(proc.flows[fd].rx_rb) : 0; + s.tx_q_now = proc.flows[fd].tx_rb != NULL + ? ssm_rbuff_queued(proc.flows[fd].tx_rb) : 0; + + pthread_rwlock_rdlock(&frcti->lock); + + s.srtt = frcti->srtt; + s.mdev = frcti->mdev; + s.rto = frcti->rto; + s.min_rtt = frcti->min_rtt; + s.snd_cr = frcti->snd_cr; + s.rcv_cr = frcti->rcv_cr; + s.stat = frcti->stat; + + pthread_rwlock_unlock(&frcti->lock); + pthread_rwlock_unlock(&proc.lock); + + written = snprintf(buf, len, + "Maximum packet lifetime (ns): %20" PRIu64 "\n" + "Max time to Ack (ns): %20" PRIu64 "\n" + "Max time to Retransmit (ns): %20" PRIu64 "\n" + "Smoothed rtt (ns): %20ld\n" + "RTT standard deviation (ns): %20ld\n" + "Retransmit timeout RTO (ns): %20ld\n" + "Minimum RTT (RACK base, ns): %20ld\n" + "Sender left window edge: %20u\n" + "Sender right window edge: %20u\n" + "Sender inactive (ns): %20lld\n" + "Sender current sequence number: %20u\n" + "Receiver left window edge: %20u\n" + "Receiver right window edge: %20u\n" + "Receiver inactive (ns): %20lld\n" + "Receiver last ack: %20u\n" + "RXM (RTO-driven) sent: %20zu\n" + "RXM packets received: %20zu\n" + " duplicates received: %20zu\n" + "RXM (SACK mechanism) sent: %20zu\n" + "RXM (RACK-driven) sent: %20zu\n" + "RXM (DupThresh-driven) sent: %20zu\n" + "RXM (NACK-driven) sent: %20zu\n" + "ACK packets sent: %20zu\n" + "Delayed-ACK timer fires: %20zu\n" + " suppressed (seqno): %20zu\n" + " suppressed (inact): %20zu\n" + " suppressed (rate): %20zu\n" + "ACK packets received: %20zu\n" + " fed RTT estimator: %20zu\n" + " wire dups dropped: %20zu\n" + "Duplicates received: %20zu\n" + "Out-of-window pkts received: %20zu\n" + "Out-of-rqueue pkts received: %20zu\n" + "OOO arrivals: %20zu\n" + "SACKs sent: %20zu\n" + "SACKs received: %20zu\n" + "D-SACKs sent: %20zu\n" + "D-SACKs received: %20zu\n" + "D-SACK out-of-range dropped: %20zu\n" + "Pre-DRF NACKs sent: %20zu\n" + "Pre-DRF NACKs received: %20zu\n" + "Tail loss probes sent: %20zu\n" + "Inactivity drops (silent): %20zu\n" + "DRF window rebases: %20zu\n" + "rq slots cleared by release_rq: %20zu\n" + "RTT probes sent: %20zu\n" + "RTT probe replies received: %20zu\n" + "RTT estimator samples: %20zu\n" + "Rendez-vous packets sent: %20zu\n" + "Rendez-vous packets received: %20zu\n" + "Keepalives sent: %20zu\n" + "Keepalives received: %20zu\n" + "SDU writes fragmented: %20zu\n" + " alloc fail mid-SDU: %20zu\n" + " tx fail mid-SDU: %20zu\n" + "Fragments sent: %20zu\n" + "Fragments received: %20zu\n" + "SDUs delivered reassembled: %20zu\n" + "SDUs delivered (SOLE): %20zu\n" + "Fragments dropped (malformed): %20zu\n" + "Stream bytes sent: %20zu\n" + "Stream bytes received: %20zu\n" + "Stream bytes delivered: %20zu\n" + "Stream packets dropped: %20zu\n" + "Stream FINs dropped: %20zu\n" + "FRCTI_RCV time (ns): %20zu\n" + "tw_move time (ns): %20zu\n" + "drain_rx_nb calls: %20zu\n" + "RX rbuff queued: %20zu\n" + "TX rbuff queued: %20zu\n" + "RXM-due entries: %20zu\n" + " bail (acked): %20zu\n" + " bail (unowned): %20zu\n" + " bail (aged): %20zu\n" + " bail (defer): %20zu\n" + "RXM-arm malloc failures: %20zu\n" + "RXM cancels (teardown): %20zu\n" + "RXM tx into dead flow: %20zu\n" + "Tx ring drops (any cause): %20zu\n" + " ack: %20zu\n" + " sack: %20zu\n" + " ka: %20zu\n" + " rttp: %20zu\n" + " nack: %20zu\n" + " rdv: %20zu\n" + " other: %20zu\n", + /* Check getattr size below when adding stats. */ + s.t_mpl, s.t_a, s.t_r, + s.srtt, s.mdev, s.rto, s.min_rtt, + s.snd_cr.lwe, s.snd_cr.rwe, + (long long)(now_ns - s.snd_cr.act), + s.snd_cr.seqno, + s.rcv_cr.lwe, s.rcv_cr.rwe, + (long long)(now_ns - s.rcv_cr.act), + s.rcv_cr.seqno, + s.stat.rxm_rto, s.stat.rxm_rcv, s.stat.rxm_dup_rcv, + s.stat.rxm_sack, s.stat.rxm_rack, s.stat.rxm_dupthresh, + s.stat.rxm_nack, + s.stat.ack_snd, s.stat.ack_fire, + s.stat.ack_supp_seqno, s.stat.ack_supp_inact, + s.stat.ack_supp_rate, + s.stat.ack_rcv, s.stat.ack_rtt, s.stat.ack_dup_rcv, + s.stat.dup_rcv, s.stat.out_rcv, s.stat.rqo_rcv, + s.stat.ooo_rcv, + s.stat.sack_snd, s.stat.sack_rcv, + s.stat.dsack_snd, s.stat.dsack_rcv, s.stat.dsack_drop, + s.stat.nack_snd, s.stat.nack_rcv, s.stat.tlp_snd, + s.stat.inact_drop, s.stat.drf_rebase, s.stat.rq_released, + s.stat.rttp_snd, s.stat.rttp_rcv, s.stat.rtt_smpl, + s.stat.rdv_snd, s.stat.rdv_rcv, + s.stat.ka_snd, s.stat.ka_rcv, + s.stat.sdu_snd_frag, s.stat.sdu_snd_alloc, s.stat.sdu_snd_tx, + s.stat.frag_snd, s.stat.frag_rcv, + s.stat.sdu_reasm, s.stat.sdu_sole, s.stat.frag_drop, + s.stat.strm_snd_byte, s.stat.strm_rcv_byte, + s.stat.strm_dlv_byte, + s.stat.strm_drop, s.stat.strm_fin_drop, + s.stat.rcv_proc_ns, s.stat.tw_move_ns, + s.stat.drain_calls, + s.rx_q_now, s.tx_q_now, + s.stat.rxm_due_count, + s.stat.rxm_due_acked, s.stat.rxm_due_unowned, + s.stat.rxm_due_aged, s.stat.rxm_due_defer, + s.stat.rxm_arm_fail, + s.stat.rxm_cancel, + s.stat.rxm_tx_dead, s.stat.tx_drop, + s.stat.tx_drop_ack, s.stat.tx_drop_sack, + s.stat.tx_drop_ka, s.stat.tx_drop_rttp, + s.stat.tx_drop_nack, s.stat.tx_drop_rdv, + s.stat.tx_drop_other); + + if (written < 0) + return 0; + + if ((size_t) written >= len) + return (int) (len - 1); + + return written; +} + +__attribute__((cold)) +static int frct_rib_readdir(char *** buf) +{ + *buf = malloc(sizeof(**buf)); + if (*buf == NULL) + goto fail_malloc; + + (*buf)[0] = strdup("frct"); + if ((*buf)[0] == NULL) + goto fail_strdup; + + return 1; + + fail_strdup: + free(*buf); + fail_malloc: + return -ENOMEM; +} + +__attribute__((cold)) +static int frct_rib_getattr(const char * path, + struct rib_attr * attr) +{ + (void) path; + + /* Must be >= the sprintf output in frct_rib_read. */ + attr->size = 8192; + attr->mtime = 0; + + return 0; +} + + +static struct rib_ops r_ops = { + .read = frct_rib_read, + .readdir = frct_rib_readdir, + .getattr = frct_rib_getattr +}; + +#endif /* PROC_FLOW_STATS */ + +static __inline__ bool before(uint32_t s1, uint32_t s2) +{ + return (int32_t)(s1 - s2) < 0; +} + +static __inline__ bool after(uint32_t s1, uint32_t s2) +{ + return (int32_t)(s2 - s1) < 0; +} + +static __inline__ bool within(uint32_t seq, uint32_t lo, uint32_t hi) +{ + return after(seq, lo) && !after(seq, hi); +} + +static __inline__ bool in_window(uint32_t seq, const struct frct_cr * cr) +{ + return !before(seq, cr->lwe) && before(seq, cr->rwe); +} + +/* DRF arrival that stays within the current receive epoch. */ +static __inline__ bool same_epoch_drf(uint32_t seq, + uint16_t flags, + const struct frct_cr * cr) +{ + if (cr->lwe == cr->rwe) + return false; + + return (flags & FRCT_RXM) || in_window(seq, cr); +} + +/* + * RACK reorder window R (RFC 8985 §6.2): + * R = MIN(reo_wnd_mult * RACK.min_RTT / 4, SRTT) + * reo_wnd_mult scales on D-SACK evidence of under-tolerance (§7.2). + * Fall back to srtt when no min_rtt sample exists yet; MIN_REORDER_NS + * floor guards collapse below the timer-tick resolution. + */ +static __inline__ uint64_t rack_reorder_window(struct frcti * frcti) +{ + uint64_t mult = frcti->reo_wnd_mult > 0 ? frcti->reo_wnd_mult : 1; + uint64_t base = frcti->min_rtt > 0 ? (uint64_t) frcti->min_rtt + : (uint64_t) frcti->srtt; + uint64_t R = mult * (base / 4); + + R = MAX(R, (uint64_t) MIN_REORDER_NS); + R = MIN(R, (uint64_t) frcti->srtt); + + return R; +} + +static __inline__ int frct_spb_reserve(size_t len, + struct ssm_pk_buff ** spb) +{ + ssize_t idx = ssm_pool_alloc_b(proc.pool, len, NULL, spb, NULL); + + return idx < 0 ? (int) idx : 0; +} + +static __inline__ void frct_spb_release(struct ssm_pk_buff * spb) +{ + ssm_pool_remove(proc.pool, ssm_pk_buff_get_off(spb)); +} + +static __inline__ void frct_spb_release_idx(size_t idx) { - return (int32_t)(seq1 - seq2) < 0; + ssm_pool_remove(proc.pool, idx); } -static bool after(uint32_t seq1, - uint32_t seq2) +/* Fetch the spb stashed at the rq slot for seqno. */ +static __inline__ struct ssm_pk_buff * rq_frag(const struct frcti * frcti, + uint32_t seqno) { - return (int32_t)(seq2 - seq1) < 0; + return ssm_pool_get(proc.pool, frcti->rcv_slots[RQ_SLOT(seqno)].idx); } -static void __send_frct_pkt(int fd, - uint8_t flags, - uint32_t ackno, - uint32_t rwe) +static __inline__ size_t frcti_data_hdr_len(const struct frcti * frcti) { - struct shm_du_buff * sdb; + return FRCT_PCILEN + (frcti->stream ? FRCT_PCI_STREAM_LEN : 0); +} + +static __inline__ size_t frcti_ctrl_hdr_len(const struct frcti * frcti) +{ + (void) frcti; + + return FRCT_PCILEN; +} + +/* + * HCS at offset 2 inside PCI. Covers flags (bytes 0..1) and + * window/seqno/ackno (bytes 4..15), plus SPCI for stream DATA. + */ +static void frct_hcs_set(struct frct_pci * pci, + bool stream) +{ + uint16_t hcs = 0; + size_t tail; + + tail = sizeof(*pci) - sizeof(pci->flags) - sizeof(pci->hcs); + if (stream) + tail += FRCT_PCI_STREAM_LEN; + + crc16_ccitt_false(&hcs, pci, sizeof(pci->flags)); + crc16_ccitt_false(&hcs, &pci->window, tail); + + pci->hcs = hton16(hcs); +} + +static int frct_hcs_check(const struct frct_pci * pci, + const struct frcti * frcti) +{ + uint16_t hcs = 0; + uint16_t flags; + size_t tail; + + /* Untrusted flag read; mismatch on HCS will drop on corrupt. */ + flags = ntoh16(pci->flags); + + tail = sizeof(*pci) - sizeof(pci->flags) - sizeof(pci->hcs); + if (frcti->stream && (flags & FRCT_DATA)) + tail += FRCT_PCI_STREAM_LEN; + + crc16_ccitt_false(&hcs, pci, sizeof(pci->flags)); + crc16_ccitt_false(&hcs, &pci->window, tail); + + return hcs != ntoh16(pci->hcs); +} + +/* Bump tx_drop plus the per-frame-type counter matching `flags`. */ +static void frct_tx_drop_bump(struct frcti * frcti, + uint16_t flags) +{ + STAT_BUMP(frcti, tx_drop); + + if (flags & FRCT_SACK) { + STAT_BUMP(frcti, tx_drop_sack); + return; + } + + if (flags & FRCT_KA) { + STAT_BUMP(frcti, tx_drop_ka); + return; + } + + if (flags & FRCT_RTTP) { + STAT_BUMP(frcti, tx_drop_rttp); + return; + } + + if (flags & FRCT_NACK) { + STAT_BUMP(frcti, tx_drop_nack); + return; + } + + if (flags & FRCT_RDVS) { + STAT_BUMP(frcti, tx_drop_rdv); + return; + } + + if (flags & FRCT_ACK) { + STAT_BUMP(frcti, tx_drop_ack); + return; + } + + STAT_BUMP(frcti, tx_drop_other); +} + +static int frct_tx(struct frcti * frcti, struct ssm_pk_buff * spb) +{ + struct flow * f = frcti_to_flow(frcti); + const struct frct_pci * pci; + const struct timespec * dl = NULL; + struct timespec now; + struct timespec intv = TIMESPEC_INIT_NS(FRCT_TX_TIMEO_NS); + struct timespec deadline; + uint16_t flags; + ssize_t idx; + int ret = -ENOMEM; + + pci = (const struct frct_pci *) ssm_pk_buff_head(spb); + flags = ntoh16(pci->flags); + + /* CRC32 covers plaintext body; PCI is in HCS. Pre-encrypt. */ + if (flags & FRCT_SACK) { + if (crc_add(spb, frcti_ctrl_hdr_len(frcti)) != 0) + goto fail; + } else if ((flags & FRCT_DATA) && f->info.qs.ber == 0) { + if (crc_add(spb, frcti_data_hdr_len(frcti)) != 0) + goto fail; + } + + if (spb_encrypt(f, spb) < 0) + goto fail; + + idx = ssm_pk_buff_get_off(spb); + + /* DATA blocks; control times out so a full ring can't stall wheel. */ + if (!(flags & FRCT_DATA)) { + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, &intv, &deadline); + dl = &deadline; + } + + ret = ssm_rbuff_write_b(f->tx_rb, idx, dl); + if (ret < 0) + goto fail; + + ssm_flow_set_notify(f->set, f->info.id, FLOW_PKT); + + return 0; + + fail: + frct_tx_drop_bump(frcti, flags); + ssm_pool_remove(proc.pool, ssm_pk_buff_get_off(spb)); + return ret; +} + +__attribute__((cold)) +static void frct_mark_flow_down(struct frcti * frcti) +{ + struct flow * f = frcti_to_flow(frcti); + + if (f->rx_rb != NULL) + ssm_rbuff_set_acl(f->rx_rb, ACL_FLOWDOWN); + + if (f->tx_rb != NULL) + ssm_rbuff_set_acl(f->tx_rb, ACL_FLOWDOWN); +} + +__attribute__((cold)) +static void frct_mark_peer_dead(struct frcti * frcti) +{ + struct flow * f = frcti_to_flow(frcti); + + if (f->rx_rb != NULL) + ssm_rbuff_set_acl(f->rx_rb, ACL_FLOWPEER); + + if (proc.fqset != NULL) + ssm_flow_set_notify(proc.fqset, f->info.id, FLOW_PEER); +} + +static __inline__ int frct_ctrl_alloc(struct ssm_pk_buff ** spb, + struct frct_pci ** pci, + size_t payload_len) +{ + if (frct_spb_reserve(FRCT_PCILEN + payload_len, spb) < 0) + return -1; + + *pci = (struct frct_pci *) ssm_pk_buff_head(*spb); + memset(*pci, 0, FRCT_PCILEN); + + return 0; +} + +/* + * Advertised rwe. Stream mode clamps to lwe + ring_seq_cap so the + * byte-equivalent fits the rx ring. Caller holds at least the rdlock. + */ +static __inline__ uint32_t frcti_advert_rwe(struct frcti * frcti) +{ + uint32_t rwe; + uint32_t cap; + + rwe = frcti->rcv_cr.rwe; + + if (!frcti->stream) + return rwe; + + cap = frcti->rcv_cr.lwe + frcti->ring_seq_cap; + + return before(cap, rwe) ? cap : rwe; +} + +static void frcti_pkt_snd(struct frcti * frcti, + uint16_t flags, + uint32_t ackno, + uint32_t rwe) +{ + struct ssm_pk_buff * spb; struct frct_pci * pci; - ssize_t idx; - struct flow * f; - /* Raw calls needed to bypass frcti. */ -#ifdef RXM_BLOCKING - idx = shm_rdrbuff_alloc_b(ai.rdrb, sizeof(*pci), NULL, &sdb, NULL); -#else - idx = shm_rdrbuff_alloc(ai.rdrb, sizeof(*pci), NULL, &sdb); -#endif - if (idx < 0) + if (frct_ctrl_alloc(&spb, &pci, 0) < 0) return; - pci = (struct frct_pci *) shm_du_buff_head(sdb); - memset(pci, 0, sizeof(*pci)); + pci->flags = hton16(flags); + pci->window = hton32(rwe); + pci->ackno = hton32(ackno); + if (flags & FRCT_ACK) { + /* reuse ackno for the sequence number of delayed ACK */ + ackno = FETCH_ADD_RELAXED(&frcti->snd_cr.ackno, 1); + pci->seqno = hton32(ackno + 1); + } - *((uint32_t *) pci) = hton32(rwe); + frct_hcs_set(pci, false); - pci->flags = flags; - pci->ackno = hton32(ackno); + frct_tx(frcti, spb); +} - f = &ai.flows[fd]; -#ifdef RXM_BLOCKING - if (shm_rbuff_write_b(f->tx_rb, idx, NULL)) { -#else - if (shm_rbuff_write(f->tx_rb, idx)) { +/* RTO floor scales with srtt; hard floor rto_min guards sub-ms RTT. */ +static void rtt_init(struct frcti * frcti, + time_t rtt_hint) +{ + time_t floor; + + if (rtt_hint > 0) { + rtt_hint = MAX(rtt_hint, (time_t) RTT_BOOT_NS); + frcti->srtt = rtt_hint; + frcti->mdev = rtt_hint >> 3; + floor = MAX(frcti->rto_min, 2 * frcti->srtt); + frcti->rto = MAX(floor, rtt_hint + (frcti->mdev << MDEV_MUL)); + frcti->min_rtt = rtt_hint; + } else { + /* Boot from first ACK. */ + frcti->srtt = 0; + frcti->mdev = RTT_BOOT_NS; + frcti->rto = MAX((time_t) INITIAL_RTO, frcti->rto_min); + frcti->min_rtt = 0; + } + + frcti->rto_mul = 0; +} + +/* RFC 8985 §6.2: replace min_RTT on unset, smaller sample, or expiry. */ +static __inline__ bool min_rtt_stale(struct frcti * frcti, + time_t mrtt, + uint64_t now_ns) +{ + if (frcti->min_rtt == 0) + return true; + + if (mrtt < frcti->min_rtt) + return true; + + return ts_aged_ns(now_ns, frcti->t_min_rtt, MIN_RTT_WIN_NS); +} + +/* Linux-style windowed-min refresh of RACK.min_RTT. */ +static __inline__ void min_rtt_update(struct frcti * frcti, + time_t mrtt, + uint64_t now_ns) +{ + if (!min_rtt_stale(frcti, mrtt, now_ns)) + return; + + frcti->min_rtt = mrtt; + frcti->t_min_rtt = now_ns; +} + +static void rtt_update(struct frcti * frcti, + time_t mrtt, + uint64_t now_ns) +{ + time_t srtt = frcti->srtt; + time_t rttvar = frcti->mdev; + time_t floor; + time_t rto; + + if (srtt == 0) { + srtt = mrtt; + rttvar = mrtt >> 1; + } else { + /* RFC 6298 symmetric EWMA. */ + time_t delta = mrtt - srtt; + srtt += (delta >> 3); + delta = (ABS(delta) - rttvar) >> 2; +#ifdef FRCT_LINUX_RTT_ESTIMATOR + if (delta < 0) + delta >>= 3; #endif - ipcp_sdb_release(sdb); + rttvar += delta; + } + STAT_BUMP(frcti, rtt_smpl); + frcti->srtt = MAX(SRTT_FLOOR_NS, srtt); + frcti->mdev = MAX(MDEV_FLOOR_NS, rttvar); + + min_rtt_update(frcti, mrtt, now_ns); + + floor = MAX(frcti->rto_min, 2 * frcti->srtt); + rto = MAX(floor, frcti->srtt + (frcti->mdev << MDEV_MUL)); + + STORE_RELEASE(&frcti->rto, rto); + STORE_RELEASE(&frcti->rto_mul, 0); +} + +/* Fill probes[pos], return new probe_id; 0 on entropy failure. Wrlock. */ +static uint32_t rttp_alloc_probe(struct frcti * frcti, + uint64_t now_ns, + uint8_t nonce[RTTP_NONCE_LEN]) +{ + uint32_t probe_id; + size_t pos; + + if (random_buffer(nonce, RTTP_NONCE_LEN) < 0) + return 0; + + probe_id = frcti->probe_id_next++; + if (probe_id == 0) + probe_id = frcti->probe_id_next++; + + pos = RTTP_POS(probe_id); + frcti->probes[pos].id = probe_id; + frcti->probes[pos].ts = now_ns; + memcpy(frcti->probes[pos].nonce, nonce, RTTP_NONCE_LEN); + frcti->t_snd_probe = now_ns; + + STAT_BUMP(frcti, rttp_snd); + + return probe_id; +} + +/* Caller wrlock; out args valid on true (caller emits post-unlock). */ +static bool rtt_probe_arm(struct frcti * frcti, + uint64_t now_ns, + uint32_t * probe_id, + uint8_t nonce[RTTP_NONCE_LEN]) +{ + if (frcti->srtt == 0) + return false; + + if (!after(frcti->snd_cr.seqno, frcti->snd_cr.lwe)) + return false; + + if (!ts_aged_ns(now_ns, frcti->t_rcv_rtt, + 2u * (uint64_t) frcti->srtt)) + return false; + + if (!ts_aged_ns(now_ns, frcti->t_snd_probe, + (uint64_t) frcti->srtt)) + return false; + + *probe_id = rttp_alloc_probe(frcti, now_ns, nonce); + + return *probe_id != 0; +} + +static void frcti_rttp_snd(struct frcti * frcti, + uint32_t probe_id, + uint32_t echo_id, + const uint8_t * nonce) +{ + struct ssm_pk_buff * spb; + struct frct_pci * pci; + struct frct_rttp * rttp; + + if (frct_ctrl_alloc(&spb, &pci, RTTP_PAYLOAD) < 0) + return; + + pci->flags = hton16(FRCT_RTTP); + + frct_hcs_set(pci, false); + + rttp = (struct frct_rttp *) FRCT_BODY(pci); + rttp->probe_id = hton32(probe_id); + rttp->echo_id = hton32(echo_id); + memcpy(rttp->nonce, nonce, sizeof(rttp->nonce)); + + frct_tx(frcti, spb); +} + +struct rxm_entry { + struct tw_entry tw; + struct list_head next; /* in frcti->rxm_list */ + struct frcti * frcti; + uint32_t seqno; + uint64_t t0; + size_t len; + uint8_t pkt[]; /* flexible — sized at alloc time */ +}; + +static struct rxm_entry * rxm_entry_create(struct frcti * frcti, + uint32_t seqno, + const struct ssm_pk_buff * spb) +{ + struct rxm_entry * r; + struct timespec now; + size_t len = ssm_pk_buff_len(spb); + + r = malloc(sizeof(*r) + len); + if (r == NULL) { + STAT_BUMP(frcti, rxm_arm_fail); + return NULL; + } + + memcpy(r->pkt, ssm_pk_buff_head(spb), len); + r->len = len; + r->frcti = frcti; + r->seqno = seqno; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + r->t0 = TS_TO_UINT64(now); + + tw_init_entry(&r->tw); + + return r; +} + +static void rxm_entry_destroy(struct rxm_entry * r) +{ + free(r); +} + +static bool rxm_still_owned(struct frcti * frcti, + size_t pos, + struct rxm_entry * r) +{ + return LOAD_ACQUIRE(&frcti->snd_slots[pos].rxm) == r; +} + +/* + * All in-flight slots share the HoL backoff; otherwise non-HoL timers + * cycle at base RTO and storm the wire while HoL is still backing off. + */ +static uint64_t rxm_next_deadline(struct frcti * frcti, + uint64_t now_ns) +{ + time_t rto = LOAD_RELAXED(&frcti->rto); + uint8_t rto_mul = LOAD_RELAXED(&frcti->rto_mul); + + return now_ns + ((uint64_t) rto << rto_mul); +} + +/* Copy pkt, set FRCT_RXM, refresh ackno, re-seal HCS. */ +static struct ssm_pk_buff * rxm_pkt_prepare(const void * pkt, + size_t len, + uint32_t rcv_lwe, + bool stream) +{ + struct ssm_pk_buff * spb; + struct frct_pci * pci; + uint16_t flags; + + if (frct_spb_reserve(len, &spb) < 0) + return NULL; + + pci = (struct frct_pci *) ssm_pk_buff_head(spb); + memcpy(pci, pkt, len); + + flags = ntoh16(pci->flags) | FRCT_RXM; + pci->flags = hton16(flags); + pci->ackno = hton32(rcv_lwe); + + frct_hcs_set(pci, stream); + + return spb; +} + +/* Caller must NOT hold frcti->lock. */ +static void rxm_snd(struct frcti * frcti, + uint32_t seqno, + const void * pkt, + size_t len) +{ + struct ssm_pk_buff * spb; + struct timespec now; + struct snd_slot * slot; + uint32_t snd_lwe; + uint32_t rcv_lwe; + size_t pos; + int ret; + + snd_lwe = LOAD_RELAXED(&frcti->snd_cr.lwe); + rcv_lwe = LOAD_RELAXED(&frcti->rcv_cr.lwe); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + pthread_rwlock_wrlock(&frcti->lock); + + pos = RQ_SLOT(seqno); + slot = &frcti->snd_slots[pos]; + + slot->time = TS_TO_UINT64(now); + /* RTO supersedes any pending TLP/fast-rxm on this slot. */ + slot->flags = (slot->flags & ~(SND_FAST_RXM | SND_TLP)) | SND_RTX; + /* §7.3: RTO supersedes TLP probes and ends the probe episode. */ + frcti->tlp_high_seq = 0; + frcti->tlp_count = 0; + + frcti->rtt_lwe = seqno + 1; + + /* Only the HoL retransmit bumps the global RTO backoff. */ + if (seqno == snd_lwe && frcti->rto_mul < MAX_RTO_MUL) + STORE_RELEASE(&frcti->rto_mul, frcti->rto_mul + 1); + + /* RFC 8985 §7.2 step 4: RTO on HoL resets RACK reo scaling. */ + if (seqno == snd_lwe) + frcti->reo_wnd_mult = 1; + + pthread_rwlock_unlock(&frcti->lock); + + STAT_BUMP(frcti, rxm_rto); + + spb = rxm_pkt_prepare(pkt, len, rcv_lwe, frcti->stream); + if (spb == NULL) + return; + + /* ETIMEDOUT/ENOMEM: let r-timer drive teardown. */ + ret = frct_tx(frcti, spb); + if (ret == -EFLOWDOWN || ret == -ENOTALLOC) + STAT_BUMP(frcti, rxm_tx_dead); +} + +static void rxm_due(void * arg) +{ + struct rxm_entry * r = arg; + struct frcti * frcti = r->frcti; + struct timespec now; + uint64_t now_ns; + uint32_t snd_lwe; + size_t pos = RQ_SLOT(r->seqno); + + STAT_BUMP(frcti, rxm_due_count); + + snd_lwe = LOAD_RELAXED(&frcti->snd_cr.lwe); + + /* Already ACK'd: expected for the steady-state majority. */ + if (before(r->seqno, snd_lwe)) { + STAT_BUMP(frcti, rxm_due_acked); + goto cleanup; + } + + /* SACK/RACK-cleared the slot (caller NULL'd snd_slots[pos].rxm). */ + if (!rxm_still_owned(frcti, pos, r)) { + STAT_BUMP(frcti, rxm_due_unowned); + goto cleanup; + } + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + /* R-timer expired: peer unreachable. */ + if (RXM_AGED_OUT(r->t0, now_ns, frcti->t_r)) { + STAT_BUMP(frcti, rxm_due_aged); + frct_mark_flow_down(frcti); + goto cleanup; + } + + /* HoL-only retx; defer at base rto so HoL transitions react. */ + if (r->seqno != snd_lwe) { + STAT_BUMP(frcti, rxm_due_defer); + tw_post(&r->tw, now_ns + LOAD_RELAXED(&frcti->rto), + rxm_due, r); return; } - shm_flow_set_notify(f->set, f->flow_id, FLOW_PKT); + rxm_snd(frcti, r->seqno, r->pkt, r->len); + + /* Re-check ownership: fire path may have replaced our entry. */ + if (rxm_still_owned(frcti, pos, r)) { + uint64_t anchor; + + /* Per-slot anchor breaks co-fire re-bin. */ + anchor = frcti->snd_slots[pos].time; + tw_post(&r->tw, rxm_next_deadline(frcti, anchor), rxm_due, r); + return; + } + + cleanup: + pthread_rwlock_wrlock(&frcti->lock); + + if (rxm_still_owned(frcti, pos, r)) + STORE_RELEASE(&frcti->snd_slots[pos].rxm, NULL); + + list_del(&r->next); + + pthread_rwlock_unlock(&frcti->lock); + + rxm_entry_destroy(r); +} + +static int rxm_arm(struct frcti * frcti, + uint32_t seqno, + const struct ssm_pk_buff * spb) +{ + struct rxm_entry * r; + time_t rto; + uint8_t rto_mul; + uint64_t deadline; + + r = rxm_entry_create(frcti, seqno, spb); + if (r == NULL) + return -ENOMEM; + + rto = LOAD_RELAXED(&frcti->rto); + rto_mul = LOAD_RELAXED(&frcti->rto_mul); + deadline = r->t0 + ((uint64_t) rto << rto_mul); + + pthread_rwlock_wrlock(&frcti->lock); + + list_add_tail(&r->next, &frcti->rxm_list); + STORE_RELEASE(&frcti->snd_slots[RQ_SLOT(seqno)].rxm, r); + + pthread_rwlock_unlock(&frcti->lock); + + tw_post(&r->tw, deadline, rxm_due, r); + + return 0; +} + +static void rxm_cancel_all(struct frcti * frcti) +{ + struct list_head * p; + struct list_head * t; + + list_for_each_safe(p, t, &frcti->rxm_list) { + struct rxm_entry * r = list_entry(p, struct rxm_entry, next); + list_del(&r->next); + tw_cancel(&r->tw); + rxm_entry_destroy(r); + STAT_BUMP(frcti, rxm_cancel); + } +} + +static __inline__ void sack_block_put(uint8_t * payload, + uint16_t i, + uint32_t s, + uint32_t e) +{ + uint32_t * blk = (uint32_t *) + (payload + SACK_HDR_SIZE + i * SACK_BLOCK_SIZE); + + blk[0] = hton32(s); + blk[1] = hton32(e); +} + +static __inline__ void sack_block_get(const uint8_t * payload, + uint16_t i, + uint32_t * s, + uint32_t * e) +{ + const uint32_t * blk = (const uint32_t *) + (payload + SACK_HDR_SIZE + i * SACK_BLOCK_SIZE); + + *s = ntoh32(blk[0]); + *e = ntoh32(blk[1]); +} + +/* + * Build SACK blocks for ranges *above* rcv_cr.lwe. Wire invariant + * (see doc/frct.txt §1.3): every block produced here satisfies + * blocks[i].start > rcv_cr.lwe = ackno, which makes the "first block + * below ackno" convention used to mark a D-SACK (RFC 2883 §4 case 1) + * unambiguous. Caller holds frcti->lock. + */ +static uint16_t sack_blocks_build(struct frcti * frcti, + uint32_t blocks[][2], + uint16_t max_n) +{ + const struct rcv_slot * slots = frcti->rcv_slots; + uint32_t s; + uint32_t end; + uint16_t n = 0; + + s = frcti->rcv_cr.lwe + 1; + end = frcti->rcv_cr.lwe + RQ_SIZE; + if (after(end, frcti->rcv_cr.rwe)) + end = frcti->rcv_cr.rwe; + + while (before(s, end) && n < max_n) { + while (before(s, end) && slots[RQ_SLOT(s)].idx == -1) + ++s; + + if (!before(s, end)) + break; + + blocks[n][0] = s; + while (before(s, end) && slots[RQ_SLOT(s)].idx != -1) + ++s; + blocks[n][1] = s; + ++n; + } + + return n; } -static void send_frct_pkt(struct frcti * frcti) +/* + * Prepend the pending D-SACK report (if any) as block[0]; clear flag. + * Returns the number of slots consumed at the head (0 or 1). Caller + * holds wrlock. + */ +static __inline__ uint16_t dsack_consume(struct frcti * frcti, + uint32_t blocks[][2]) +{ + if (!frcti->dsack_valid || frcti->sack_n_max == 0) + return 0; + + blocks[0][0] = frcti->dsack_seqno; + blocks[0][1] = frcti->dsack_seqno + 1; + frcti->dsack_valid = false; + return 1; +} + +/* Caller must NOT hold frcti->lock. */ +static void frcti_sack_snd(struct frcti * frcti, + const struct sack_args * sa) +{ + struct ssm_pk_buff * spb; + struct frct_pci * pci; + buffer_t buf; + uint16_t i; + + assert(sa->n <= SACK_MAX_BLOCKS); + + buf.len = SACK_HDR_SIZE + sa->n * SACK_BLOCK_SIZE; + + if (frct_ctrl_alloc(&spb, &pci, buf.len) < 0) + return; + + pci->flags = hton16(FRCT_ACK | FRCT_FC | FRCT_SACK); + pci->window = hton32(sa->rwe); + pci->ackno = hton32(sa->ack); + pci->seqno = hton32(FETCH_ADD_RELAXED(&frcti->snd_cr.ackno, 1) + 1); + + frct_hcs_set(pci, false); + + buf.data = FRCT_BODY(pci); + memset(buf.data, 0, SACK_HDR_SIZE); + *(uint16_t *) buf.data = hton16(sa->n); + for (i = 0; i < sa->n; ++i) + sack_block_put(buf.data, i, sa->blocks[i][0], sa->blocks[i][1]); + + frct_tx(frcti, spb); +} + +static void ack_snd(struct frcti * frcti, + bool with_sack) { struct timespec now; + uint64_t now_ns; time_t diff; uint32_t ackno; uint32_t rwe; - int fd; + struct sack_args * sa = NULL; + size_t sa_sz; + bool sacking = false; assert(frcti); - pthread_rwlock_rdlock(&frcti->lock); + STAT_BUMP(frcti, ack_fire); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + if (with_sack && frcti->sack_n_max > 0) { + sa_sz = sizeof(*sa) + frcti->sack_n_max * sizeof(sa->blocks[0]); + sa = malloc(sa_sz); + /* If alloc fails, fall through and send a bare cum-ACK. */ + } + + pthread_rwlock_wrlock(&frcti->lock); - if (frcti->rcv_cr.lwe == frcti->rcv_cr.seqno) { + /* D-SACK rides through cum-ACK freshness; signal is the duplicate. */ + if (!after(frcti->rcv_cr.lwe, frcti->rcv_cr.seqno) + && !frcti->dsack_valid) { pthread_rwlock_unlock(&frcti->lock); - return; + STAT_BUMP(frcti, ack_supp_seqno); + goto out; } - fd = frcti->fd; ackno = frcti->rcv_cr.lwe; - rwe = frcti->rcv_cr.rwe; + rwe = frcti_advert_rwe(frcti); - clock_gettime(PTHREAD_COND_CLOCK, &now); + if (ACK_AGED_OUT(frcti->rcv_cr.act, now_ns, frcti->t_a)) { + pthread_rwlock_unlock(&frcti->lock); + STAT_BUMP(frcti, ack_supp_inact); + goto out; + } - diff = ts_diff_ns(&frcti->rcv_cr.act, &now); + diff = (time_t) ts_age_ns(now_ns, frcti->snd_cr.act); + if (diff < TICTIME && !frcti->dsack_valid) { + pthread_rwlock_unlock(&frcti->lock); + STAT_BUMP(frcti, ack_supp_rate); + goto out; + } + + /* RFC 2018: piggyback SACK on timer ACK; dedup unchanged board. */ + if (sa == NULL || (frcti->sack_n == 0 && !frcti->dsack_valid)) + goto no_sack; + + sa->dsack = false; + sa->n = dsack_consume(frcti, sa->blocks); + if (sa->n == 1) + sa->dsack = true; + + sa->n += sack_blocks_build(frcti, sa->blocks + sa->n, + frcti->sack_n_max - sa->n); + if (sa->n == 0) + goto no_sack; + + if (!sa->dsack && ackno == frcti->sack_lwe && sa->n == frcti->sack_n) + goto no_sack; + + sa->ack = ackno; + sa->rwe = rwe; + frcti->sack_lwe = ackno; + frcti->sack_n = sa->n; + frcti->t_snd_sack = now_ns; + sacking = true; + + no_sack: + frcti->rcv_cr.seqno = frcti->rcv_cr.lwe; pthread_rwlock_unlock(&frcti->lock); - if (diff > frcti->a || diff < DELT_ACK) + STAT_BUMP(frcti, ack_snd); + + if (sacking) { + STAT_BUMP(frcti, sack_snd); + if (sa->dsack) + STAT_BUMP(frcti, dsack_snd); + frcti_sack_snd(frcti, sa); + } else { + frcti_pkt_snd(frcti, FRCT_ACK | FRCT_FC, ackno, rwe); + } + + out: + free(sa); +} + +/* Delayed-ACK timer: per-flow, dedup'd via atomic test-and-set. */ +static void ack_due(void * arg) +{ + struct frcti * frcti = arg; + + __atomic_clear(&frcti->ack_pending, __ATOMIC_RELAXED); + + ack_snd(frcti, true); +} + +static int ack_arm(struct frcti * frcti) +{ + struct timespec now; + uint64_t deadline; + + if (__atomic_test_and_set(&frcti->ack_pending, __ATOMIC_RELAXED)) + return 0; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + deadline = TS_TO_UINT64(now) + ACK_DELAY_NS; + + tw_post(&frcti->ack_tw, deadline, ack_due, frcti); + + return 0; +} + +/* Forward decl breaks the keepalive cycle: ka_arm <-> ka_due. */ +static void ka_due(void * arg); + +static int ka_arm(struct frcti * frcti) +{ + struct timespec now; + uint64_t now_ns; + uint64_t timeo_ns; + uint64_t snd_ns; + uint64_t rcv_ns; + uint64_t deadline; + + timeo_ns = (uint64_t) frcti->qs_timeout * MILLION; /* IMM */ + snd_ns = LOAD_RELAXED(&frcti->snd_cr.act) + timeo_ns / 4; + rcv_ns = LOAD_RELAXED(&frcti->rcv_cr.act) + timeo_ns; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + deadline = MIN(snd_ns, rcv_ns); + if (deadline <= now_ns) + deadline = now_ns + timeo_ns / 4; + + tw_post(&frcti->ka_tw, deadline, ka_due, frcti); + + return 0; +} + +__attribute__((cold)) +static void ka_snd(struct frcti * frcti) +{ + struct ssm_pk_buff * spb; + struct frct_pci * pci; + struct timespec now; + uint64_t now_ns; + time_t timeo_ns; + uint64_t rcv_act; + uint64_t ka_rcv; + int64_t rcv_idle; + int64_t snd_idle; + uint32_t ackno; + + assert(frcti); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + timeo_ns = (time_t)(frcti->qs_timeout) * MILLION; /* IMM */ + rcv_act = LOAD_RELAXED(&frcti->rcv_cr.act); + ka_rcv = LOAD_RELAXED(&frcti->t_ka_rcv); + rcv_idle = ts_age_ns(now_ns, rcv_act > ka_rcv ? rcv_act : ka_rcv); + snd_idle = ts_age_ns(now_ns, LOAD_RELAXED(&frcti->snd_cr.act)); + + if (rcv_idle > timeo_ns) { + frct_mark_peer_dead(frcti); return; + } + + if (snd_idle <= timeo_ns / 4) { + ka_arm(frcti); + return; + } + + if (frct_ctrl_alloc(&spb, &pci, 0) < 0) { + ka_arm(frcti); + return; + } + + ackno = LOAD_RELAXED(&frcti->rcv_cr.lwe); + + pci->flags = hton16(FRCT_KA | FRCT_ACK); + pci->ackno = hton32(ackno); + + frct_hcs_set(pci, false); + + STAT_BUMP(frcti, ka_snd); + frct_tx(frcti, spb); + + ka_arm(frcti); +} + +/* Keepalive timer: re-posted by the fire callback itself. */ +static void ka_due(void * arg) +{ + ka_snd((struct frcti *) arg); +} + +static void frcti_rdv_snd(struct frcti * frcti) +{ + frcti_pkt_snd(frcti, FRCT_RDVS, 0, 0); +} + +#define HAS_RESCNTL(cr) ((cr)->cflags & FRCTFRESCNTL) +static bool frcti_is_window_open(struct frcti * frcti) +{ + struct frct_cr * snd_cr = &frcti->snd_cr; + struct timespec now; + time_t diff; + bool ret = false; - __send_frct_pkt(fd, FRCT_ACK | FRCT_FC, ackno, rwe); + if (!HAS_RESCNTL(snd_cr)) + return true; + if (before(snd_cr->seqno, LOAD_RELAXED(&snd_cr->rwe))) + return true; + + /* Window may be closed; wrlock for RDV state mutations. */ pthread_rwlock_wrlock(&frcti->lock); - if (after(frcti->rcv_cr.lwe, frcti->rcv_cr.seqno)) - frcti->rcv_cr.seqno = frcti->rcv_cr.lwe; + if (before(snd_cr->seqno, snd_cr->rwe)) { + ret = true; + goto unlock; + } + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + if (frcti->open) { + frcti->open = false; + frcti->t_wnd = now; + frcti->t_last_rdv = now; + goto unlock; + } + + diff = ts_diff_ns(&now, &frcti->t_wnd); + if (diff > MAX_RDV) + goto unlock; + diff = ts_diff_ns(&now, &frcti->t_last_rdv); + if (diff > (time_t) frcti->t_rdv) { + frcti->t_last_rdv = now; + frcti_rdv_snd(frcti); + STAT_BUMP(frcti, rdv_snd); + } + unlock: pthread_rwlock_unlock(&frcti->lock); + + return ret; } -static void __send_rdv(int fd) +/* n contiguous seqnos free? No RDV: the n=1 path drives it. */ +static bool frcti_is_window_open_n(struct frcti * frcti, + size_t n) { - struct shm_du_buff * sdb; - struct frct_pci * pci; - ssize_t idx; - struct flow * f; + struct frct_cr * snd_cr = &frcti->snd_cr; - /* Raw calls needed to bypass frcti. */ - idx = shm_rdrbuff_alloc_b(ai.rdrb, sizeof(*pci), NULL, &sdb, NULL); - if (idx < 0) - return; + if (!HAS_RESCNTL(snd_cr)) + return true; - pci = (struct frct_pci *) shm_du_buff_head(sdb); - memset(pci, 0, sizeof(*pci)); + if (n <= 1) + return frcti_is_window_open(frcti); - pci->flags = FRCT_RDVS; + return before(snd_cr->seqno + (uint32_t)(n - 1), + LOAD_RELAXED(&snd_cr->rwe)); +} - f = &ai.flows[fd]; +static void release_rq(struct frcti * frcti) +{ + size_t i; - if (shm_rbuff_write_b(f->tx_rb, idx, NULL)) { - ipcp_sdb_release(sdb); - return; + for (i = 0; i < RQ_SIZE; ++i) { + if (frcti->rcv_slots[i].idx == -1) + continue; + + /* Stream rq entries are sentinels (no spb owned). */ + if (!frcti->stream) + frct_spb_release_idx(frcti->rcv_slots[i].idx); + + frcti->rcv_slots[i].idx = -1; + STAT_BUMP(frcti, rq_released); } +} - shm_flow_set_notify(f->set, f->flow_id, FLOW_PKT); +static __inline__ bool stream_ring_sz_ok(struct frcti * frcti, + size_t n) +{ + size_t per_pkt; + + if (n > FRCT_STREAM_RING_SZ_MAX) + return false; + + if ((n & (n - 1)) != 0) + return false; + + per_pkt = frcti->frag_mtu - frcti_data_hdr_len(frcti); + + return n >= FRCT_STREAM_RING_MIN_PKTS * per_pkt; } -static struct frcti * frcti_create(int fd) +/* Default ring sized for full RQ_SIZE seqno window; pow2, capped. */ +static size_t default_stream_ring_sz(size_t per_pkt) { - struct frcti * frcti; - ssize_t idx; - struct timespec now; - time_t mpl; - time_t a; - time_t r; - pthread_condattr_t cattr; + size_t need; + size_t sz; + + need = (size_t) RQ_SIZE * per_pkt; + sz = FRCT_STREAM_RING_SZ; + + while (sz < need && sz < FRCT_STREAM_RING_SZ_MAX) + sz <<= 1; + + return sz; +} + +struct frcti * frcti_create(int fd, + uint64_t a, + uint64_t r, + uint64_t mpl, + time_t rtt_hint, + qosspec_t qs, + uint32_t mtu) +{ + struct frcti * frcti; + ssize_t idx; + struct timespec now; + uint64_t now_ns; + size_t bb; + size_t per_pkt; +#ifdef PROC_FLOW_STATS + char frctstr[FRCT_NAME_STRLEN + 1]; +#endif + mpl *= MILLION; /* ms -> ns */ + a *= MILLION; /* ms -> ns */ + r *= MILLION; /* ms -> ns */ frcti = malloc(sizeof(*frcti)); if (frcti == NULL) @@ -219,40 +1828,76 @@ static struct frcti * frcti_create(int fd) memset(frcti, 0, sizeof(*frcti)); + list_head_init(&frcti->rxm_list); + if (pthread_rwlock_init(&frcti->lock, NULL)) goto fail_lock; - if (pthread_mutex_init(&frcti->mtx, NULL)) - goto fail_mutex; - - if (pthread_condattr_init(&cattr)) - goto fail_cattr; -#ifndef __APPLE__ - pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); +#ifdef PROC_FLOW_STATS + sprintf(frctstr, "%d", fd); + if (rib_reg(frctstr, &r_ops)) + goto fail_rib_reg; #endif - if (pthread_cond_init(&frcti->cond, &cattr)) - goto fail_cond; for (idx = 0; idx < RQ_SIZE; ++idx) - frcti->rq[idx] = -1; + frcti->rcv_slots[idx].idx = -1; clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + frcti->t_mpl = mpl; + frcti->t_a = a; + frcti->t_r = r; + frcti->t_rdv = DELT_RDV; + frcti->fd = fd; + frcti->ber = (time_t) qs.ber; + frcti->lossy = (qs.loss != 0); + frcti->qs_timeout = (time_t) qs.timeout; + + frcti->frag_mtu = (size_t) mtu; + + /* Cap blocks per SACK at what fits in the per-flow frag_mtu. */ + bb = (frcti->frag_mtu - FRCT_PCILEN - SACK_HDR_SIZE) + / SACK_BLOCK_SIZE; + if (bb > SACK_MAX_BLOCKS) + bb = SACK_MAX_BLOCKS; + frcti->sack_n_max = (uint16_t) bb; + + frcti->max_rcv_sdu = FRCT_MAX_SDU; + + frcti->stream = (qs.service == SVC_STREAM); + if (frcti->stream) { + per_pkt = frcti->frag_mtu - frcti_data_hdr_len(frcti); + frcti->rcv_ring_sz = default_stream_ring_sz(per_pkt); + frcti->ring_seq_cap = + (uint32_t) (frcti->rcv_ring_sz / per_pkt); + } - frcti->mpl = mpl = DELT_MPL; - frcti->a = a = DELT_A; - frcti->r = r = DELT_R; - frcti->rdv = DELT_RDV; - frcti->fd = fd; - - - frcti->rttseq = 0; - frcti->probe = false; - - frcti->srtt = 0; /* Updated on first ACK */ - frcti->mdev = 10 * MILLION; /* Initial rxm will be after 20 ms */ - frcti->rto = 20 * MILLION; /* Initial rxm will be after 20 ms */ - - if (ai.flows[fd].qs.loss == 0) { + frcti->rto_min = (time_t) MAX(RTO_MIN, 1ULL << RXMQ_RES); + rtt_init(frcti, rtt_hint); + frcti->t_min_rtt = now_ns; + frcti->probe_id_next = 1; + frcti->t_rcv_rtt = now_ns; + frcti->t_snd_probe = now_ns; + frcti->t_snd_sack = 0; + frcti->sack_lwe = 0; + frcti->sack_n = 0; + frcti->dsack_seqno = 0; + frcti->dsack_valid = false; + frcti->reo_wnd_mult = 1; + frcti->dsack_lwe_snap = 0; + frcti->t_last_reo_widen = 0; + /* So the first pre-DRF NACK fires without waiting cooldown. */ + frcti->t_nack = now_ns - BILLION; + frcti->in_recovery = false; + frcti->recovery_high = 0; + frcti->rack_fired_lwe = 0; + + tw_init_entry(&frcti->ack_tw); + tw_init_entry(&frcti->ka_tw); + tw_init_entry(&frcti->tlp_tw); + + if (!frcti->lossy) { frcti->snd_cr.cflags |= FRCTFRTX | FRCTFLINGER; frcti->rcv_cr.cflags |= FRCTFRTX; } @@ -260,20 +1905,31 @@ static struct frcti * frcti_create(int fd) frcti->snd_cr.cflags |= FRCTFRESCNTL; frcti->snd_cr.rwe = START_WINDOW; + if (frcti->lossy) + frcti->snd_cr.rwe = RQ_SIZE; + + frcti->snd_cr.inact = 3 * mpl + a + r + BILLION; /* ns */ + frcti->snd_cr.act = now_ns - frcti->snd_cr.inact - BILLION; + + frcti->rcv_cr.inact = 2 * mpl + a + r + BILLION; /* ns */ + frcti->rcv_cr.act = now_ns - frcti->rcv_cr.inact - BILLION; - frcti->snd_cr.inact = (3 * mpl + a + r) / BILLION + 1; /* s */ - frcti->snd_cr.act.tv_sec = now.tv_sec - (frcti->snd_cr.inact + 1); + frcti->t_ka_rcv = now_ns; - frcti->rcv_cr.inact = (2 * mpl + a + r) / BILLION + 1; /* s */ - frcti->rcv_cr.act.tv_sec = now.tv_sec - (frcti->rcv_cr.inact + 1); + /* qs_timeout == 0: no KA, silent peer crash goes undetected. */ + if (frcti->qs_timeout > 0) { + if (ka_arm(frcti) < 0) + goto fail_ka_arm; + } return frcti; - fail_cond: - pthread_condattr_destroy(&cattr); -fail_cattr: - pthread_mutex_destroy(&frcti->mtx); - fail_mutex: + fail_ka_arm: +#ifdef PROC_FLOW_STATS + sprintf(frctstr, "%d", fd); + rib_unreg(frctstr); + fail_rib_reg: +#endif pthread_rwlock_destroy(&frcti->lock); fail_lock: free(frcti); @@ -281,16 +1937,55 @@ fail_cattr: return NULL; } -static void frcti_destroy(struct frcti * frcti) +void frcti_destroy(struct frcti * frcti) { - pthread_cond_destroy(&frcti->cond); - pthread_mutex_destroy(&frcti->mtx); +#ifdef PROC_FLOW_STATS + char frctstr[FRCT_NAME_STRLEN + 1]; +#endif + /* Drop every wheel entry referencing frcti before freeing it. */ + rxm_cancel_all(frcti); + tw_cancel(&frcti->ack_tw); + tw_cancel(&frcti->ka_tw); + tw_cancel(&frcti->tlp_tw); + +#if defined(PROC_FLOW_STATS) && defined(FRCT_DEBUG_STDOUT) + printf("[FRCT teardown] pid=%d fd=%d " + "sdu_snd=%zu sdu_reasm=%zu sdu_sole=%zu " + "frag_snd=%zu frag_rcv=%zu frag_drop=%zu " + "rxm_rto=%zu rxm_sack=%zu rxm_dup=%zu " + "rxm_due=%zu acked=%zu unowned=%zu aged=%zu defer=%zu " + "cancel=%zu arm_fail=%zu inflight=%u " + "nack_snd=%zu nack_rcv=%zu inact_drop=%zu " + "drf_rebase=%zu rq_released=%zu\n", + (int) getpid(), frcti->fd, + frcti->stat.sdu_snd_frag, frcti->stat.sdu_reasm, + frcti->stat.sdu_sole, + frcti->stat.frag_snd, frcti->stat.frag_rcv, + frcti->stat.frag_drop, + frcti->stat.rxm_rto, frcti->stat.rxm_sack, + frcti->stat.rxm_dupthresh, + frcti->stat.rxm_due_count, frcti->stat.rxm_due_acked, + frcti->stat.rxm_due_unowned, frcti->stat.rxm_due_aged, + frcti->stat.rxm_due_defer, + frcti->stat.rxm_cancel, frcti->stat.rxm_arm_fail, + frcti->snd_cr.seqno - frcti->snd_cr.lwe, + frcti->stat.nack_snd, frcti->stat.nack_rcv, + frcti->stat.inact_drop, + frcti->stat.drf_rebase, frcti->stat.rq_released); +#endif + + release_rq(frcti); + free(frcti->rcv_ring); +#ifdef PROC_FLOW_STATS + sprintf(frctstr, "%d", frcti->fd); + rib_unreg(frctstr); +#endif pthread_rwlock_destroy(&frcti->lock); free(frcti); } -static uint16_t frcti_getflags(struct frcti * frcti) +uint16_t frcti_getflags(struct frcti * frcti) { uint16_t ret; @@ -298,87 +1993,91 @@ static uint16_t frcti_getflags(struct frcti * frcti) pthread_rwlock_rdlock(&frcti->lock); - ret = frcti->snd_cr.cflags; + ret = frcti->snd_cr.cflags & FRCTFMASK; pthread_rwlock_unlock(&frcti->lock); return ret; } -static void frcti_setflags(struct frcti * frcti, - uint16_t flags) +void frcti_setflags(struct frcti * frcti, + uint16_t flags) { - flags |= FRCTFRTX; /* Should not be set by command */ - assert(frcti); - pthread_rwlock_wrlock(&frcti->lock); + flags &= FRCTFSETMASK; - frcti->snd_cr.cflags &= FRCTFRTX; /* Zero other flags */ + pthread_rwlock_wrlock(&frcti->lock); - frcti->snd_cr.cflags &= flags; + frcti->snd_cr.cflags = (frcti->snd_cr.cflags & ~FRCTFSETMASK) | flags; pthread_rwlock_unlock(&frcti->lock); } -#define frcti_queued_pdu(frcti) \ - (frcti == NULL ? idx : __frcti_queued_pdu(frcti)) +size_t frcti_get_max_rcv_sdu(struct frcti * frcti) +{ + size_t ret; -#define frcti_snd(frcti, sdb) \ - (frcti == NULL ? 0 : __frcti_snd(frcti, sdb)) + assert(frcti); -#define frcti_rcv(frcti, sdb) \ - (frcti == NULL ? 0 : __frcti_rcv(frcti, sdb)) + pthread_rwlock_rdlock(&frcti->lock); + ret = frcti->max_rcv_sdu; + pthread_rwlock_unlock(&frcti->lock); -#define frcti_tick(frcti) \ - (frcti == NULL ? 0 : __frcti_tick()) + return ret; +} -#define frcti_dealloc(frcti) \ - (frcti == NULL ? 0 : __frcti_dealloc(frcti)) +int frcti_set_max_rcv_sdu(struct frcti * frcti, + size_t max) +{ + assert(frcti); -#define frcti_is_window_open(frcti) \ - (frcti == NULL ? true : __frcti_is_window_open(frcti)) + if (max == 0) + return -EINVAL; -#define frcti_window_wait(frcti, abstime) \ - (frcti == NULL ? 0 : __frcti_window_wait(frcti, abstime)) + pthread_rwlock_wrlock(&frcti->lock); + frcti->max_rcv_sdu = max; + pthread_rwlock_unlock(&frcti->lock); + return 0; +} -static bool __frcti_is_window_open(struct frcti * frcti) +size_t frcti_get_rcv_ring_sz(struct frcti * frcti) { - struct frct_cr * snd_cr = &frcti->snd_cr; - int ret = true; + size_t ret; + + assert(frcti); pthread_rwlock_rdlock(&frcti->lock); + ret = frcti->rcv_ring_sz; + pthread_rwlock_unlock(&frcti->lock); - if (snd_cr->cflags & FRCTFRESCNTL) - ret = before(snd_cr->seqno, snd_cr->rwe); + return ret; +} - if (!ret) { - struct timespec now; +/* Set before any stream byte has been delivered; -EBUSY otherwise. */ +int frcti_set_rcv_ring_sz(struct frcti * frcti, + size_t n) +{ + int ret = 0; + size_t per_pkt; - clock_gettime(PTHREAD_COND_CLOCK, &now); + assert(frcti); - pthread_mutex_lock(&frcti->mtx); - if (frcti->open) { - frcti->open = false; - frcti->t_wnd = now; - frcti->t_rdvs = now; - } else { - time_t diff; - diff = ts_diff_ns(&frcti->t_wnd, &now); - if (diff > MAX_RDV) { - pthread_mutex_unlock(&frcti->mtx); - return false; - } - - diff = ts_diff_ns(&frcti->t_rdvs, &now); - if (diff > frcti->rdv) { - frcti->t_rdvs = now; - __send_rdv(frcti->fd); - } - } + if (!frcti->stream) + return -ENOTSUP; + if (!stream_ring_sz_ok(frcti, n)) + return -EINVAL; - pthread_mutex_unlock(&frcti->mtx); + per_pkt = frcti->frag_mtu - frcti_data_hdr_len(frcti); + + pthread_rwlock_wrlock(&frcti->lock); + + if (frcti->rcv_ring != NULL) { + ret = -EBUSY; + } else { + frcti->rcv_ring_sz = n; + frcti->ring_seq_cap = (uint32_t) (n / per_pkt); } pthread_rwlock_unlock(&frcti->lock); @@ -386,426 +2085,2101 @@ static bool __frcti_is_window_open(struct frcti * frcti) return ret; } -static int __frcti_window_wait(struct frcti * frcti, - struct timespec * abstime) +time_t frcti_get_rto_min(struct frcti * frcti) { - struct frct_cr * snd_cr = &frcti->snd_cr; - int ret = 0; + time_t v; + + assert(frcti); pthread_rwlock_rdlock(&frcti->lock); + v = frcti->rto_min; + pthread_rwlock_unlock(&frcti->lock); - if (!(snd_cr->cflags & FRCTFRESCNTL)) { - pthread_rwlock_unlock(&frcti->lock); + return v; +} + +/* Floor at the timer-wheel resolution; finer granularity is unrepresentable. */ +int frcti_set_rto_min(struct frcti * frcti, + time_t rto_min) +{ + time_t floor = (time_t) (1ULL << RXMQ_RES); + time_t rto_floor; + time_t rto; + + assert(frcti); + + if (rto_min < floor) + return -EINVAL; + + pthread_rwlock_wrlock(&frcti->lock); + + frcti->rto_min = rto_min; + if (frcti->srtt > 0) { + rto_floor = MAX(rto_min, 2 * frcti->srtt); + rto = MAX(rto_floor, + frcti->srtt + (frcti->mdev << MDEV_MUL)); + STORE_RELEASE(&frcti->rto, rto); + } else if (frcti->rto < rto_min) { + STORE_RELEASE(&frcti->rto, rto_min); + } + + pthread_rwlock_unlock(&frcti->lock); + + return 0; +} + +/* Re-arm a fresh rxm so a lost fast-retx still recovers via RTO. */ +static void sack_rxm_snd(struct frcti * frcti, + void * pkt, + size_t len) +{ + struct ssm_pk_buff * spb; + const struct frct_pci * pci; + uint32_t rcv_lwe; + uint32_t seqno; + int ret; + + rcv_lwe = LOAD_RELAXED(&frcti->rcv_cr.lwe); + + spb = rxm_pkt_prepare(pkt, len, rcv_lwe, frcti->stream); + if (spb == NULL) + return; + + pci = (const struct frct_pci *) ssm_pk_buff_head(spb); + seqno = ntoh32(pci->seqno); + + /* Register fresh rxm before send; old entry self-cleans. */ + if (rxm_arm(frcti, seqno, spb) < 0) { + frct_spb_release(spb); + return; + } + + STAT_BUMP(frcti, rxm_sack); + ret = frct_tx(frcti, spb); + if (ret == -EFLOWDOWN || ret == -ENOTALLOC) + STAT_BUMP(frcti, rxm_tx_dead); +} + +/* Additive HoL emit; original snd_slots[hp].rxm stays armed (NewReno). */ +static int fast_rxm_send(struct frcti * frcti, + void * pkt, + size_t len) +{ + struct ssm_pk_buff * spb; + uint32_t rcv_lwe; + + rcv_lwe = LOAD_RELAXED(&frcti->rcv_cr.lwe); + + spb = rxm_pkt_prepare(pkt, len, rcv_lwe, frcti->stream); + if (spb == NULL) return 0; + + return frct_tx(frcti, spb); +} + +/* PCI bytes survive head_release at receive; just rewind the pointer. */ +static __inline__ uint16_t frag_role_peek(struct ssm_pk_buff * spb) +{ + const struct frct_pci * pci; + + assert(ssm_pk_buff_head(spb) != NULL); + + pci = (const struct frct_pci *) (ssm_pk_buff_head(spb) - FRCT_PCILEN); + + return ntoh16(pci->flags) & FRCT_FR_MASK; +} + +enum frag_state { + FRAG_NOT_READY, /* head missing / FIRST..LAST run incomplete */ + FRAG_DELIVER, /* *count fragments form a deliverable SDU */ + FRAG_DROP, /* *count fragments at lwe are malformed */ +}; + +/* + * On a gap in the run: FRTX waits (NOT_READY); best-effort scans forward + * for the next FIRST/SOLE and returns DROP for the broken prefix. *count + * gets the offset from the trailing edge. NOT_READY if no later run is + * in window. Caller rdlock. + */ +static enum frag_state frag_inspect_gap(struct frcti * frcti, + size_t start, + size_t * count) +{ + const struct rcv_slot * slots = frcti->rcv_slots; + struct ssm_pk_buff * spb; + uint32_t k; + uint16_t role; + size_t m; + + if (frcti->rcv_cr.cflags & FRCTFRTX) + return FRAG_NOT_READY; + + k = frcti->rcv_cr.rwe - RQ_SIZE; + + for (m = start; m < RQ_SIZE; ++m) { + if (slots[RQ_SLOT(k + m)].idx == -1) + continue; + + spb = rq_frag(frcti, k + m); + role = frag_role_peek(spb); + + if (role == FRCT_FR_SOLE || role == FRCT_FR_FIRST) { + if (m == 0) + return FRAG_NOT_READY; + + *count = m; + return FRAG_DROP; + } } - while (snd_cr->seqno == snd_cr->rwe && ret != -ETIMEDOUT) { - struct timespec now; + return FRAG_NOT_READY; +} - pthread_rwlock_unlock(&frcti->lock); - pthread_mutex_lock(&frcti->mtx); +/* + * Inspect rq[lwe..]; set *count and return DELIVER/DROP/NOT_READY. DROP + * covers broken prefixes (mid/last at HoL, FIRST..[non-LAST]..new-FIRST). + * Non-FRTX flows skip past gaps to the next FIRST/SOLE. Caller rdlock. + */ +static enum frag_state frag_run_inspect(struct frcti * frcti, + size_t * count) +{ + const struct rcv_slot * slots = frcti->rcv_slots; + struct ssm_pk_buff * spb; + uint32_t k = frcti->rcv_cr.rwe - RQ_SIZE; + uint16_t role; + size_t n = 0; + + if (slots[RQ_SLOT(k)].idx == -1) + return frag_inspect_gap(frcti, 0, count); + + spb = rq_frag(frcti, k); + role = frag_role_peek(spb); + + if (role == FRCT_FR_SOLE) { + *count = 1; + return FRAG_DELIVER; + } + + if (role != FRCT_FR_FIRST) { + *count = 1; + return FRAG_DROP; + } + + while (true) { + if (n == RQ_SIZE || slots[RQ_SLOT(k + n)].idx == -1) + return frag_inspect_gap(frcti, n, count); - if (frcti->open) { - clock_gettime(PTHREAD_COND_CLOCK, &now); + spb = rq_frag(frcti, k + n); + role = frag_role_peek(spb); + ++n; - frcti->t_wnd = now; - frcti->t_rdvs = now; - frcti->open = false; + if (role == FRCT_FR_LAST) { + *count = n; + return FRAG_DELIVER; } - pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock, - (void *) &frcti->mtx); + if (n > 1 && role != FRCT_FR_MID) { + /* SOLE or new FIRST mid-run: drop the prefix. */ + *count = n - 1; + return FRAG_DROP; + } + } +} + +/* Caller wrlock. Delivery edge is implicit: rwe - RQ_SIZE. */ +static void frag_drop(struct frcti * frcti, + size_t count) +{ + uint32_t k = frcti->rcv_cr.rwe - RQ_SIZE; + uint32_t edge; + size_t i; - ret = -pthread_cond_timedwait(&frcti->cond, - &frcti->mtx, - abstime); + for (i = 0; i < count; ++i) { + size_t pos = RQ_SLOT(k + i); - pthread_cleanup_pop(false); + if (frcti->rcv_slots[pos].idx == -1) + continue; - if (ret == -ETIMEDOUT) { - time_t diff; + frct_spb_release_idx(frcti->rcv_slots[pos].idx); + frcti->rcv_slots[pos].idx = -1; + } + + frcti->rcv_cr.rwe += count; - clock_gettime(PTHREAD_COND_CLOCK, &now); + /* Drop may span a gap; pull lwe up to preserve rwe - RQ_SIZE <= lwe. */ + edge = frcti->rcv_cr.rwe - RQ_SIZE; + if (before(frcti->rcv_cr.lwe, edge)) + STORE_RELEASE(&frcti->rcv_cr.lwe, edge); +} - diff = ts_diff_ns(&frcti->t_wnd, &now); - if (diff > MAX_RDV) { - pthread_mutex_unlock(&frcti->mtx); - return -ECONNRESET; /* write fails! */ - } +/* Copy `count` fragments at rq[lwe..] into buf; release + advance lwe. */ +static size_t frag_gather(struct frcti * frcti, + size_t count, + uint8_t * buf) +{ + struct ssm_pk_buff * frag; + size_t off = 0; + size_t i; + uint32_t k = frcti->rcv_cr.rwe - RQ_SIZE; + + for (i = 0; i < count; ++i) { + size_t pos = RQ_SLOT(k + i); + size_t flen; + + frag = rq_frag(frcti, k + i); + flen = ssm_pk_buff_len(frag); + memcpy(buf + off, ssm_pk_buff_head(frag), flen); + off += flen; + frct_spb_release_idx(frcti->rcv_slots[pos].idx); + frcti->rcv_slots[pos].idx = -1; + } - diff = ts_diff_ns(&frcti->t_rdvs, &now); - if (diff > frcti->rdv) { - frcti->t_rdvs = now; - __send_rdv(frcti->fd); - } + frcti->rcv_cr.rwe += count; + + return off; +} + +/* Caller holds lock. */ +static size_t frag_total_len(struct frcti * frcti, + size_t count, + bool * overflow) +{ + struct ssm_pk_buff * frag; + size_t total = 0; + size_t i; + uint32_t k = frcti->rcv_cr.rwe - RQ_SIZE; + + *overflow = false; + + for (i = 0; i < count; ++i) { + size_t flen; + + frag = rq_frag(frcti, k + i); + flen = ssm_pk_buff_len(frag); + if (total + flen < total) { + *overflow = true; + return 0; } + total += flen; + } + + return total; +} + +/* + * Process a delivered slot at lwe: latch FIN if acceptable, + * advance byte_high (clamped to byte_fin once latched). + */ +static __inline__ void stream_deliver_slot(struct frcti * frcti, + size_t lp) +{ + uint32_t end; + + end = frcti->rcv_slots[lp].end; + + if (frcti->rcv_slots[lp].fin) { + if (end == frcti->rcv_byte_high && !frcti->rcv_fin_seen) { + frcti->rcv_fin_seen = true; + frcti->rcv_byte_fin = end; + } else { + STAT_BUMP(frcti, strm_fin_drop); + } + } + + if (frcti->rcv_fin_seen && after(end, frcti->rcv_byte_fin)) + end = frcti->rcv_byte_fin; + + frcti->rcv_byte_high = end; +} + +/* Two-segment memcpy from buf into the rx ring at byte offset start. */ +static void stream_ring_write(struct frcti * frcti, + uint32_t start, + buffer_t buf) +{ + size_t mask = frcti->rcv_ring_sz - 1; + size_t off = start & mask; + + if (off + buf.len <= frcti->rcv_ring_sz) { + memcpy(frcti->rcv_ring + off, buf.data, buf.len); + } else { + size_t first = frcti->rcv_ring_sz - off; + memcpy(frcti->rcv_ring + off, buf.data, first); + memcpy(frcti->rcv_ring, buf.data + first, buf.len - first); + } +} + +/* Two-segment memcpy from the rx ring at byte offset start into buf. */ +static void stream_ring_read(struct frcti * frcti, + uint32_t start, + buffer_t buf) +{ + size_t mask = frcti->rcv_ring_sz - 1; + size_t off = start & mask; + + if (off + buf.len <= frcti->rcv_ring_sz) { + memcpy(buf.data, frcti->rcv_ring + off, buf.len); + } else { + size_t first = frcti->rcv_ring_sz - off; + memcpy(buf.data, frcti->rcv_ring + off, first); + memcpy(buf.data + first, frcti->rcv_ring, buf.len - first); + } +} + +/* Deliver-or-drop one stashed slot at lwe; advance lwe/rwe. Caller wrlock. */ +static void stream_advance_lwe(struct frcti * frcti) +{ + size_t lp; + + lp = RQ_SLOT(frcti->rcv_cr.lwe); + + if (frcti->rcv_slots[lp].start != frcti->rcv_byte_high) + STAT_BUMP(frcti, strm_drop); + else + stream_deliver_slot(frcti, lp); + + frcti->rcv_slots[lp].fin = 0; + frcti->rcv_slots[lp].idx = -1; + STORE_RELEASE(&frcti->rcv_cr.lwe, frcti->rcv_cr.lwe + 1); + frcti->rcv_cr.rwe++; +} + +/* + * Validate a stream DATA packet before stashing. Returns 0 if the + * packet may be written into rcv_ring + rq[], -1 otherwise. + */ +static __inline__ int stream_stash_check(struct frcti * frcti, + uint32_t start, + uint32_t end, + size_t plen, + uint16_t flags) +{ + if (end - start != (uint32_t) plen) + return -1; + + /* FIN MUST be 0-byte. */ + if ((flags & FRCT_FIN) && plen != 0) + return -1; + + /* Post-EOS: no further FIN once latched. */ + if (frcti->rcv_fin_seen && (flags & FRCT_FIN)) + return -1; + + /* Post-EOS: reject data at or past byte_fin. */ + if (frcti->rcv_fin_seen && !before(start, frcti->rcv_byte_fin)) + return -1; + + /* Stale: peer is behind the delivered edge. */ + if (before(end, frcti->rcv_byte_next)) + return -1; + + /* Exact-edge: only an empty-stream FIN is meaningful. */ + if (end == frcti->rcv_byte_next && !(flags & FRCT_FIN)) + return -1; + + if (end - frcti->rcv_byte_next > frcti->rcv_ring_sz) + return -1; + + return 0; +} + +/* + * Stream-mode DATA receive: validate, stash payload in rcv_ring, mark + * rq[pos], advance lwe through any newly-contiguous run. Returns 0 + * (spb released) or -1 (caller releases). Caller wrlock. + */ +static int frcti_stream_data_rcv(struct frcti * frcti, + struct ssm_pk_buff * spb, + size_t pos, + uint16_t flags) +{ + struct frct_pci_stream * spci; + uint32_t start; + uint32_t end; + buffer_t buf; + size_t skip; + + if (ssm_pk_buff_len(spb) < FRCT_PCI_STREAM_LEN) + return -1; + + if (frcti->rcv_ring == NULL) { + frcti->rcv_ring = calloc(1, frcti->rcv_ring_sz); + if (frcti->rcv_ring == NULL) + return -ENOMEM; + } + + spci = FRCT_HDR_POP(spb, frct_pci_stream); + start = ntoh32(spci->start); + end = ntoh32(spci->end); + + buf.data = ssm_pk_buff_head(spb); + buf.len = ssm_pk_buff_len(spb); + + if (stream_stash_check(frcti, start, end, buf.len, flags) < 0) + return -1; + + /* Trim front-overlap with already-delivered region. */ + if (before(start, frcti->rcv_byte_next)) { + skip = frcti->rcv_byte_next - start; + buf.data += skip; + buf.len -= skip; + start = frcti->rcv_byte_next; + } - pthread_mutex_unlock(&frcti->mtx); - pthread_rwlock_rdlock(&frcti->lock); + stream_ring_write(frcti, start, buf); + STAT_ADD(frcti, strm_rcv_byte, buf.len); + + frcti->rcv_slots[pos].idx = 1; + frcti->rcv_slots[pos].start = start; + frcti->rcv_slots[pos].end = end; + frcti->rcv_slots[pos].fin = (flags & FRCT_FIN) ? 1 : 0; + + while (frcti->rcv_slots[RQ_SLOT(frcti->rcv_cr.lwe)].idx != -1) + stream_advance_lwe(frcti); + + frct_spb_release(spb); + + return 0; +} + +/* + * DATA receive: stash idx at rq[pos], advance lwe through any + * contiguous run. Caller wrlock. + */ +static void frcti_data_stash(struct frcti * frcti, + ssize_t idx, + size_t pos, + uint16_t flags) +{ + frcti->rcv_slots[pos].idx = idx; + + if ((flags & FRCT_FR_MASK) != FRCT_FR_SOLE) + STAT_BUMP(frcti, frag_rcv); + + /* lwe = cum-ACK edge; advance per fragment through contiguous run. */ + while (before(frcti->rcv_cr.lwe, frcti->rcv_cr.rwe) + && frcti->rcv_slots[RQ_SLOT(frcti->rcv_cr.lwe)].idx != -1) + STORE_RELEASE(&frcti->rcv_cr.lwe, frcti->rcv_cr.lwe + 1); +} + +/* Stream consume: copy up to `count` contiguous bytes from ring into buf. */ +static ssize_t frcti_consume_stream(struct frcti * frcti, + uint8_t * buf, + size_t count) +{ + size_t avail; + size_t copy; + ssize_t ret; + buffer_t dst; + + assert(frcti); + + pthread_rwlock_wrlock(&frcti->lock); + + avail = (size_t) (frcti->rcv_byte_high - frcti->rcv_byte_next); + if (avail == 0) { + /* EOS drained: signal EOF to the reader. */ + if (frcti->rcv_fin_seen + && frcti->rcv_byte_next == frcti->rcv_byte_fin) + ret = 0; + else + ret = -EAGAIN; + goto unlock; } + copy = MIN(avail, count); + + dst.data = buf; + dst.len = copy; + stream_ring_read(frcti, frcti->rcv_byte_next, dst); + + frcti->rcv_byte_next += (uint32_t) copy; + STAT_ADD(frcti, strm_dlv_byte, copy); + + ret = (ssize_t) copy; + + unlock: pthread_rwlock_unlock(&frcti->lock); return ret; } -static ssize_t __frcti_queued_pdu(struct frcti * frcti) +/* + * FRTX consume: copy next ready PDU (full SDU or nothing). Returns bytes, + * -EAGAIN (no PDU), or -EMSGSIZE (oversize: run dropped to unblock flow). + */ +static ssize_t frcti_consume(struct frcti * frcti, + uint8_t * buf, + size_t count) { - ssize_t idx; - size_t pos; + size_t n; + size_t total; + bool overflow; + enum frag_state st; + ssize_t ret; assert(frcti); - /* See if we already have the next PDU. */ pthread_rwlock_wrlock(&frcti->lock); - pos = frcti->rcv_cr.lwe & (RQ_SIZE - 1); - - idx = frcti->rq[pos]; - if (idx != -1) { - ++frcti->rcv_cr.lwe; - ++frcti->rcv_cr.rwe; - frcti->rq[pos] = -1; + while (true) { + st = frag_run_inspect(frcti, &n); + if (st == FRAG_NOT_READY) { + ret = -EAGAIN; + goto unlock; + } + if (st == FRAG_DROP) { + STAT_ADD(frcti, frag_drop, n); + frag_drop(frcti, n); + continue; + } + /* FRAG_DELIVER */ + total = frag_total_len(frcti, n, &overflow); + if (overflow || total > frcti->max_rcv_sdu || total > count) { + STAT_ADD(frcti, frag_drop, n); + frag_drop(frcti, n); + ret = -EMSGSIZE; + goto unlock; + } + ret = (ssize_t) frag_gather(frcti, n, buf); + if (n > 1) + STAT_BUMP(frcti, sdu_reasm); + else + STAT_BUMP(frcti, sdu_sole); + goto unlock; } + unlock: pthread_rwlock_unlock(&frcti->lock); - return idx; + return ret; } -static ssize_t __frcti_pdu_ready(struct frcti * frcti) +static bool frcti_pdu_ready(struct frcti * frcti) { - ssize_t idx; - size_t pos; + size_t pos; + size_t count; + bool ready; assert(frcti); - /* See if we already have the next PDU. */ pthread_rwlock_rdlock(&frcti->lock); - pos = frcti->rcv_cr.lwe & (RQ_SIZE - 1); - idx = frcti->rq[pos]; + if (frcti->stream) { + ready = frcti->rcv_byte_high != frcti->rcv_byte_next; + pthread_rwlock_unlock(&frcti->lock); + return ready; + } + + if (frag_run_inspect(frcti, &count) != FRAG_DELIVER) { + /* Drop case: frcti_consume will handle it; not ready. */ + pthread_rwlock_unlock(&frcti->lock); + return false; + } + + pos = RQ_SLOT(frcti->rcv_cr.rwe - RQ_SIZE); + ready = frcti->rcv_slots[pos].idx != -1; pthread_rwlock_unlock(&frcti->lock); - return idx; + return ready; } -#include <timerwheel.c> +/* No srtt yet: probe at the cold-probe cadence to seed it. */ +#define PROBE_DUE_COLD(frcti, now_ns) \ + ((now_ns) - (frcti)->t_snd_probe > (uint64_t) RTTP_COLD_NS) + +/* Have srtt: probe when peer quiet for > 2*srtt and last probe > srtt. */ +#define PROBE_DUE_WARM(frcti, now_ns) \ + ((now_ns) - (frcti)->t_rcv_rtt > 2u * (uint64_t)(frcti)->srtt \ + && (now_ns) - (frcti)->t_snd_probe > (uint64_t)(frcti)->srtt) + +/* Seeds srtt for receive-only sides so they don't fall back to 1 s RTO. */ +__attribute__((cold)) +static void frcti_rcv_probe(struct frcti * frcti, + uint64_t now_ns) +{ + uint32_t probe_id; + uint8_t nonce[RTTP_NONCE_LEN] = { 0 }; + + pthread_rwlock_wrlock(&frcti->lock); + + if (frcti->srtt == 0 && !PROBE_DUE_COLD(frcti, now_ns)) { + pthread_rwlock_unlock(&frcti->lock); + return; + } + + if (frcti->srtt != 0 && !PROBE_DUE_WARM(frcti, now_ns)) { + pthread_rwlock_unlock(&frcti->lock); + return; + } + + probe_id = rttp_alloc_probe(frcti, now_ns, nonce); + + pthread_rwlock_unlock(&frcti->lock); + + if (probe_id != 0) + frcti_rttp_snd(frcti, probe_id, 0, nonce); +} + +/* Echo at slot `pos` matches our probe: id, slot, nonce all intact. */ +static __inline__ bool probe_echo_matches(struct frcti * frcti, + size_t pos, + uint32_t echo_id, + const uint8_t nonce[RTTP_NONCE_LEN]) +{ + if (frcti->probes[pos].id != echo_id) + return false; + + if (frcti->probes[pos].ts == 0) + return false; + + return memcmp(frcti->probes[pos].nonce, nonce, RTTP_NONCE_LEN) == 0; +} /* - * Send a final ACK for everything that has not been ACK'd. - * If the flow should be kept active for retransmission, - * the returned time will be negative. + * RTT probe (echo_id == 0): bounce the nonce back to peer. + * RTT echo (echo_id != 0): verify nonce + feed sample. */ -static time_t __frcti_dealloc(struct frcti * frcti) +static void frcti_rttp_rcv(struct frcti * frcti, + buffer_t pkt, + uint64_t now_ns) { - struct timespec now; - time_t wait; - int ackno; - int fd = -1; + const struct frct_rttp * rttp; + uint32_t probe_id; + uint32_t echo_id; + uint8_t nonce[RTTP_NONCE_LEN]; + size_t ring_pos; + int64_t elapsed; + uint64_t sample; + + if (pkt.len < RTTP_PAYLOAD) + return; + + rttp = (const struct frct_rttp *) pkt.data; + probe_id = ntoh32(rttp->probe_id); + echo_id = ntoh32(rttp->echo_id); + + /* Forged/malformed: bouncing this would loop on echo_id == 0. */ + if (probe_id == 0 && echo_id == 0) + return; + + memcpy(nonce, rttp->nonce, sizeof(nonce)); + + if (echo_id == 0) { + /* Probe: echo back with same nonce so peer can verify. */ + STAT_BUMP(frcti, rttp_rcv); + frcti_rttp_snd(frcti, 0, probe_id, nonce); + return; + } + + ring_pos = RTTP_POS(echo_id); + + pthread_rwlock_wrlock(&frcti->lock); + + if (!probe_echo_matches(frcti, ring_pos, echo_id, nonce)) { + pthread_rwlock_unlock(&frcti->lock); + return; + } + + elapsed = ts_age_ns(now_ns, frcti->probes[ring_pos].ts); + frcti->probes[ring_pos].ts = 0; + frcti->t_rcv_rtt = now_ns; + + if (elapsed <= 0) { + pthread_rwlock_unlock(&frcti->lock); + return; + } + sample = (uint64_t) elapsed; + + /* Clamp probe sample to RTT_CLAMP_MUL * srtt to avoid poisoning. */ + if (frcti->srtt > 0) + sample = MIN(sample, (uint64_t) frcti->srtt * RTT_CLAMP_MUL); + + rtt_update(frcti, sample, now_ns); + + pthread_rwlock_unlock(&frcti->lock); +} + +/* Honours piggybacked ACK on the KA. */ +static void frcti_ka_rcv(struct frcti * frcti, + const struct frct_pci * pci, + uint64_t now_ns, + uint16_t flags) +{ + uint32_t ka_ackno; + + STORE_RELEASE(&frcti->t_ka_rcv, now_ns); + STAT_BUMP(frcti, ka_rcv); + + if (!(flags & FRCT_ACK)) + return; + + ka_ackno = ntoh32(pci->ackno); + + pthread_rwlock_wrlock(&frcti->lock); + + if (within(ka_ackno, frcti->snd_cr.lwe, frcti->snd_cr.seqno)) + STORE_RELEASE(&frcti->snd_cr.lwe, ka_ackno); + + pthread_rwlock_unlock(&frcti->lock); +} + +/* + * Additive HoL re-emit (carries DRF); runs before rcv_cr->act + * refresh so it doesn't pre-empt peer's first DRF. + */ +__attribute__((cold)) +static void frcti_nack_rcv(struct frcti * frcti) +{ + struct timespec now; + uint64_t now_ns; + size_t hp; + struct rxm_entry * rxm; + void * pkt_copy = NULL; + size_t pkt_len = 0; clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + pthread_rwlock_wrlock(&frcti->lock); + + STAT_BUMP(frcti, nack_rcv); + + if (frcti->snd_cr.seqno == frcti->snd_cr.lwe) { + pthread_rwlock_unlock(&frcti->lock); + return; + } + + hp = RQ_SLOT(frcti->snd_cr.lwe); + rxm = LOAD_ACQUIRE(&frcti->snd_slots[hp].rxm); + if (rxm == NULL || RXM_AGED_OUT(rxm->t0, now_ns, frcti->t_r)) { + pthread_rwlock_unlock(&frcti->lock); + return; + } + + pkt_copy = malloc(rxm->len); + if (pkt_copy != NULL) { + memcpy(pkt_copy, rxm->pkt, rxm->len); + pkt_len = rxm->len; + /* Karn: suppress RTT sample. NACK supersedes pending TLP. */ + frcti->snd_slots[hp].flags = + (frcti->snd_slots[hp].flags & ~SND_TLP) + | SND_RTX | SND_FAST_RXM; + frcti->rtt_lwe = frcti->snd_cr.lwe + 1; + STAT_BUMP(frcti, rxm_nack); + } + + pthread_rwlock_unlock(&frcti->lock); + + if (pkt_copy != NULL) { + int ret = fast_rxm_send(frcti, pkt_copy, pkt_len); + if (ret == -EFLOWDOWN || ret == -ENOTALLOC) + STAT_BUMP(frcti, rxm_tx_dead); + free(pkt_copy); + } +} + +__attribute__((cold)) +static void frcti_rdv_rcv(struct frcti * frcti) +{ + uint32_t rwe; pthread_rwlock_rdlock(&frcti->lock); - ackno = frcti->rcv_cr.lwe; - if (frcti->rcv_cr.lwe != frcti->rcv_cr.seqno) - fd = frcti->fd; + rwe = frcti_advert_rwe(frcti); - wait = MAX(frcti->rcv_cr.inact - now.tv_sec + frcti->rcv_cr.act.tv_sec, - frcti->snd_cr.inact - now.tv_sec + frcti->snd_cr.act.tv_sec); + pthread_rwlock_unlock(&frcti->lock); - if (frcti->snd_cr.cflags & FRCTFLINGER - && before(frcti->snd_cr.lwe, frcti->snd_cr.seqno)) - wait = -wait; + STAT_BUMP(frcti, rdv_rcv); + frcti_pkt_snd(frcti, FRCT_FC, 0, rwe); +} + +/* §7.2: PTO = 2*SRTT + max delayed-ACK delay; fallback when unseeded. */ +static __inline__ uint64_t tlp_pto(const struct frcti * frcti) +{ + if (frcti->srtt > 0) + return 2ULL * (uint64_t) frcti->srtt + ACK_DELAY_NS; + + return NACK_COOLDOWN_NS; +} + +/* + * RFC 8985 §7: lazy probe. Re-evaluate on fire — if sender was active + * within PTO, re-post; else probe HoL once and hand off to RTO. + */ +__attribute__((cold)) +static void tlp_due(void * arg) +{ + struct frcti * frcti = arg; + struct timespec now; + uint64_t now_ns; + uint64_t pto; + uint64_t rto_at; + size_t hp; + struct rxm_entry * rxm; + void * pkt_copy = NULL; + size_t pkt_len = 0; + bool re_post = false; + uint64_t deadline = 0; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + pthread_rwlock_wrlock(&frcti->lock); + + if (frcti->snd_cr.seqno == frcti->snd_cr.lwe) + goto unlock; + + if (!before(frcti->snd_cr.seqno, frcti->snd_cr.rwe)) + goto unlock; /* FC-blocked: RDV handles it. */ + + /* RFC 8985 §7.3: one outstanding probe, MAX_TLP_PER_EP per ep. */ + if (frcti->tlp_high_seq != 0) + goto unlock; + + if (frcti->tlp_count >= MAX_TLP_PER_EP) + goto unlock; + + pto = tlp_pto(frcti); + + /* §7.2: anchor PTO on most recent send; defer if still active. */ + if (now_ns < frcti->snd_cr.act + pto) { + deadline = frcti->snd_cr.act + pto; + re_post = true; + goto unlock; + } + + hp = RQ_SLOT(frcti->snd_cr.lwe); + rxm = LOAD_ACQUIRE(&frcti->snd_slots[hp].rxm); + if (rxm == NULL || RXM_AGED_OUT(rxm->t0, now_ns, frcti->t_r)) + goto unlock; + + /* Cap: if HoL RTO is due, let rxm_due fire instead. */ + rto_at = rxm->t0 + ((uint64_t) frcti->rto + << LOAD_RELAXED(&frcti->rto_mul)); + if (rto_at <= now_ns) + goto unlock; + + pkt_copy = malloc(rxm->len); + if (pkt_copy != NULL) { + memcpy(pkt_copy, rxm->pkt, rxm->len); + pkt_len = rxm->len; + frcti->snd_slots[hp].time = now_ns; + frcti->snd_slots[hp].flags |= SND_TLP | SND_FAST_RXM; + frcti->rtt_lwe = frcti->snd_cr.lwe + 1; + /* §7.3 outstanding-probe marker; ack_rcv/rxm_snd clear. */ + frcti->tlp_high_seq = frcti->snd_cr.seqno; + frcti->tlp_count++; + STAT_BUMP(frcti, tlp_snd); + } + + unlock: pthread_rwlock_unlock(&frcti->lock); - if (fd != -1) - __send_frct_pkt(fd, FRCT_ACK, ackno, 0); + if (pkt_copy != NULL) { + fast_rxm_send(frcti, pkt_copy, pkt_len); + free(pkt_copy); + } + + if (re_post) + tw_post(&frcti->tlp_tw, deadline, tlp_due, frcti); + else + __atomic_clear(&frcti->tlp_pending, __ATOMIC_RELAXED); +} + +/* §7.2 lazy: post once per quiet period. tlp_due re-evaluates on fire. */ +static int tlp_arm(struct frcti * frcti) +{ + struct timespec now; + uint64_t now_ns; + uint64_t pto; + uint64_t deadline; + + /* §7.3: one outstanding probe, MAX_TLP_PER_EP per recovery ep. */ + if (LOAD_RELAXED(&frcti->tlp_high_seq) != 0) + return 0; + if (LOAD_RELAXED(&frcti->tlp_count) >= MAX_TLP_PER_EP) + return 0; + if (__atomic_test_and_set(&frcti->tlp_pending, __ATOMIC_RELAXED)) + return 0; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + pto = tlp_pto(frcti); + + deadline = LOAD_RELAXED(&frcti->snd_cr.act) + pto; + if (deadline <= now_ns) + deadline = now_ns + pto; + + tw_post(&frcti->tlp_tw, deadline, tlp_due, frcti); + + return 0; +} + +/* + * FC window advert from any flag-bearing packet. Caps at lwe + RQ_SIZE, + * rejects backward shrink (forged/stale FC), marks window open. + * Caller wrlock. + */ +static __inline__ void frcti_fc_rcv(struct frcti * frcti, + const struct frct_pci * pci) +{ + struct frct_cr * snd_cr; + uint32_t rwe; + uint32_t rwe_max; + + snd_cr = &frcti->snd_cr; + rwe = ntoh32(pci->window); + rwe_max = snd_cr->lwe + RQ_SIZE; + + if (after(rwe, rwe_max)) + rwe = rwe_max; + + /* Reject backward shrink (forged/stale FC). */ + if (before(rwe, snd_cr->rwe)) + rwe = snd_cr->rwe; + + STORE_RELAXED(&snd_cr->rwe, rwe); + frcti->open = true; +} + +/* Packet copies captured under frcti->lock; emitted after release. */ +struct pending { + buffer_t fast_rxm; + buffer_t sack_rxm[SACK_RXM_MAX]; + size_t sack_rxm_cnt; +}; + +/* RFC 6582 §3.2: seal recovery_high on entry; do not extend on new gaps. */ +static void recovery_enter(struct frcti * frcti) +{ + if (frcti->in_recovery) + return; + + frcti->in_recovery = true; + frcti->recovery_high = frcti->snd_cr.seqno + RTT_QUARANTINE; +} + +/* True when cum-ACK clears recovery_high or all in-flight ACKed. */ +static bool recovery_exit_reached(struct frcti * frcti, + uint32_t ackno) +{ + if (!frcti->in_recovery) + return false; + + if (!before(ackno, frcti->recovery_high)) + return true; + + return ackno == frcti->snd_cr.seqno; +} + +/* RTT sample gate: Karn + SACK-consume + don't-seed. */ +static bool rtt_sample_eligible(struct frcti * frcti, + size_t p, + uint16_t flags, + uint32_t lwe) +{ + if (flags & FRCT_RXM) + return false; + if (frcti->snd_slots[p].flags & (SND_RTX | SND_TLP)) + return false; + if (LOAD_ACQUIRE(&frcti->snd_slots[p].rxm) == NULL) + return false; + if (before(lwe, frcti->rtt_lwe)) + return false; + /* Don't seed srtt from a cum-ACK; let probes seed. */ + if (frcti->srtt == 0) + return false; + return true; +} + +#define RXM_SLOT_EMPTY(rxm) ((rxm) == NULL) +#define FAST_RXM_STAGED(pending) ((pending)->fast_rxm.data != NULL) +#define RXM_FAST_DONE(flags) (((flags) & SND_FAST_RXM) != 0) + +/* RACK fast retransmit on cum-ACK: HoL aged past R, not yet retransmitted. */ +static void fast_rxm_consider(struct frcti * frcti, + uint64_t now_ns, + struct pending * pending) +{ + struct rxm_entry * rxm; + struct snd_slot * slot; + size_t hp; + uint64_t R; + bool rack_ok; + + hp = RQ_SLOT(frcti->snd_cr.lwe); + slot = &frcti->snd_slots[hp]; + rxm = LOAD_ACQUIRE(&slot->rxm); + R = rack_reorder_window(frcti); + + if (RXM_SLOT_EMPTY(rxm)) + return; + + /* RFC 8985 §6.2: time-based RACK OR DupThresh count. */ + rack_ok = (int64_t)(frcti->t_latest_ack - slot->time) > (int64_t) R; + if (!rack_ok && frcti->dup_thresh < DUP_THRESH) + return; + + /* HoL aged past t_r; let rxm_due tear the flow down. */ + if (RXM_AGED_OUT(rxm->t0, now_ns, frcti->t_r)) + return; + + /* Already on it. */ + if (FAST_RXM_STAGED(pending) || RXM_FAST_DONE(slot->flags)) + return; + + recovery_enter(frcti); + + pending->fast_rxm.data = malloc(rxm->len); + if (pending->fast_rxm.data == NULL) + return; + + pending->fast_rxm.len = rxm->len; + memcpy(pending->fast_rxm.data, rxm->pkt, rxm->len); + slot->flags |= SND_RTX | SND_FAST_RXM; + frcti->rtt_lwe = frcti->snd_cr.lwe + 1; + if (rack_ok) + STAT_BUMP(frcti, rxm_rack); + else + STAT_BUMP(frcti, rxm_dupthresh); +} + +/* Caller holds wrlock; RACK fast retransmit queued in pending. */ +__attribute__((hot)) +static void frcti_ack_rcv(struct frcti * frcti, + const struct frct_pci * pci, + uint16_t flags, + uint64_t now_ns, + struct pending * pending) +{ + uint32_t ackno; + uint32_t lwe; + size_t p; + size_t fresh; + + if (!(flags & FRCT_DATA)) + STAT_BUMP(frcti, ack_rcv); + + ackno = ntoh32(pci->ackno); + if (ackno == frcti->snd_cr.lwe) { + /* RFC 8985 §6.2: only on scoreboard change. */ + if (frcti->snd_cr.lwe != frcti->rack_fired_lwe) { + fast_rxm_consider(frcti, now_ns, pending); + frcti->rack_fired_lwe = frcti->snd_cr.lwe; + } + return; + } + + if (!within(ackno, frcti->snd_cr.lwe, frcti->snd_cr.seqno)) + return; + + lwe = frcti->snd_cr.lwe; + p = RQ_SLOT(lwe); + + STORE_RELEASE(&frcti->snd_cr.lwe, ackno); + + /* §7.3: cum-ACK past the probed seqno resolves the TLP. */ + if (frcti->tlp_high_seq != 0 + && !before(ackno, frcti->tlp_high_seq)) + frcti->tlp_high_seq = 0; + + /* §7.3: end the probe episode once inflight drains. */ + if (ackno == frcti->snd_cr.seqno) + frcti->tlp_count = 0; + + /* RFC 8985 §7.2: halve mult per REO_DECAY_PKTS fresh-ACK'd seqnos. */ + fresh = ackno - frcti->dsack_lwe_snap; + if (frcti->reo_wnd_mult > 1 && fresh >= REO_DECAY_PKTS) { + uint8_t half = frcti->reo_wnd_mult >> 1; + frcti->reo_wnd_mult = half < 1 ? 1 : half; + frcti->dsack_lwe_snap = ackno; + } + + /* RFC 8985: latest cum-ACKed send-time (slot of ackno-1). */ + frcti->t_latest_ack = frcti->snd_slots[RQ_SLOT(ackno - 1)].time; + + /* RFC 8985: SACK-above-lwe count is per-recovery-episode. */ + frcti->dup_thresh = 0; + + /* Karn-skip on retx; TLP ACK clears rto_mul (no CC backoff). */ + if ((frcti->snd_slots[p].flags & SND_RTX) == 0 + || (frcti->snd_slots[p].flags & SND_TLP) != 0) + STORE_RELEASE(&frcti->rto_mul, 0); + + if (recovery_exit_reached(frcti, ackno)) + frcti->in_recovery = false; + + if (rtt_sample_eligible(frcti, p, flags, lwe)) { + int64_t mrtt = ts_age_ns(now_ns, frcti->snd_slots[p].time); + if (mrtt > 0) { + if (!(flags & FRCT_DATA)) + STAT_BUMP(frcti, ack_rtt); + rtt_update(frcti, (time_t) mrtt, now_ns); + frcti->t_rcv_rtt = now_ns; + } + } +} + +/* Skip k == lwe under clamp: NULLing HoL from a stale SACK wedges it. */ +static uint32_t sack_mark_blocks(struct frcti * frcti, + const uint8_t * payload, + uint16_t n, + uint32_t * newly_marked) +{ + uint32_t hi_sacked = frcti->snd_cr.lwe; + uint32_t marked = 0; + uint16_t i; + + for (i = 0; i < n; ++i) { + uint32_t s; + uint32_t e; + uint32_t k; + bool clamped; + + sack_block_get(payload, i, &s, &e); + + if (!before(s, e)) + continue; + + clamped = before(s, frcti->snd_cr.lwe); + if (clamped) + s = frcti->snd_cr.lwe; + if (after(e, frcti->snd_cr.seqno)) + e = frcti->snd_cr.seqno; + + for (k = s; before(k, e); ++k) { + size_t kp = RQ_SLOT(k); + uint64_t t_k; + if (clamped && k == frcti->snd_cr.lwe) + continue; + if (LOAD_ACQUIRE(&frcti->snd_slots[kp].rxm) == NULL) + continue; + STORE_RELEASE(&frcti->snd_slots[kp].rxm, NULL); + frcti->snd_slots[kp].flags = 0; + marked++; + /* RACK.fack: latest SACK-confirmed send-time. */ + t_k = frcti->snd_slots[kp].time; + if (t_k > frcti->t_latest_ack) + frcti->t_latest_ack = t_k; + } + + if (after(e, hi_sacked)) + hi_sacked = e; + } + + *newly_marked = marked; + return hi_sacked; +} + +/* Queue once per loss event (SND_FAST_RXM gates). Emit after unlock. */ +static void sack_queue_rxm(struct frcti * frcti, + uint32_t hi_sacked, + uint64_t now_ns, + struct pending * pending) +{ + uint64_t R = rack_reorder_window(frcti); + uint32_t k; + bool rack_ok; + + for (k = frcti->snd_cr.lwe; before(k, hi_sacked); ++k) { + struct rxm_entry * rxm; + size_t kp = RQ_SLOT(k); + size_t cnt = pending->sack_rxm_cnt; + size_t rack_age; + + rxm = LOAD_ACQUIRE(&frcti->snd_slots[kp].rxm); + + if (cnt >= SACK_RXM_MAX) + break; + + if (rxm == NULL) + continue; + + if (frcti->snd_slots[kp].flags & SND_FAST_RXM) + continue; + + if (RXM_AGED_OUT(rxm->t0, now_ns, frcti->t_r)) + continue; + + rack_age = frcti->t_latest_ack - frcti->snd_slots[kp].time; + /* RFC 8985 §6.2: time-based RACK OR DupThresh count. */ + rack_ok = (int64_t) rack_age > (int64_t) R; + if (!rack_ok && frcti->dup_thresh < DUP_THRESH) + continue; + + if (rack_ok) + STAT_BUMP(frcti, rxm_rack); + else + STAT_BUMP(frcti, rxm_dupthresh); + + pending->sack_rxm[cnt].data = malloc(rxm->len); + if (pending->sack_rxm[cnt].data == NULL) + break; + + pending->sack_rxm[cnt].len = rxm->len; + memcpy(pending->sack_rxm[cnt].data, rxm->pkt, rxm->len); + pending->sack_rxm_cnt++; + /* NULL slot so the original timer self-cleans. */ + STORE_RELEASE(&frcti->snd_slots[kp].rxm, NULL); + frcti->snd_slots[kp].time = now_ns; + frcti->snd_slots[kp].flags |= SND_RTX | SND_FAST_RXM; + frcti->rtt_lwe = k + 1; + } +} + +/* + * RFC 2883 D-SACK detector. Returns true iff block[0] is a D-SACK + * report: + * case 1: blocks[0].start < pkt_ackno (strictly below cum-ACK). + * case 2: blocks[0] is a strict sub-range of some blocks[i>0]. + * MAX_DSACK_LAG bounds case-1 distance to one rcv window (sanity). + */ +static bool sack_is_dsack(struct frcti * frcti, + const uint8_t * payload, + uint16_t n, + uint32_t pkt_ackno) +{ + uint32_t s0; + uint32_t e0; + uint16_t i; + + if (n == 0) + return false; + + sack_block_get(payload, 0, &s0, &e0); + if (!before(s0, e0)) + return false; + + if (before(s0, pkt_ackno)) { + if ((pkt_ackno - s0) <= (uint32_t) MAX_DSACK_LAG) + return true; + STAT_BUMP(frcti, dsack_drop); + return false; + } + + for (i = 1; i < n; ++i) { + uint32_t si; + uint32_t ei; + + sack_block_get(payload, i, &si, &ei); + if (!before(si, ei)) + continue; + if (!before(s0, si) && !after(e0, ei) + && (s0 != si || e0 != ei)) + return true; + } + + return false; +} + +/* RFC 8985 §7.2: grow reo_wnd_mult on DSACK; at most once per RTT. */ +static __inline__ void reo_wnd_on_dsack(struct frcti * frcti, + uint64_t now_ns) +{ + time_t srtt = frcti->srtt; + + /* Snap is unconditional: feeds the per-D-SACK decay clock. */ + frcti->dsack_lwe_snap = frcti->snd_cr.lwe; + + if (srtt > 0 + && now_ns - frcti->t_last_reo_widen <= (uint64_t) srtt) + return; + + if (frcti->reo_wnd_mult < REO_WND_MULT_MAX) + frcti->reo_wnd_mult++; + + frcti->t_last_reo_widen = now_ns; +} + +/* Caller holds wrlock; retransmits queued for post-unlock emission. */ +static void frcti_sack_rcv(struct frcti * frcti, + buffer_t pkt, + uint32_t pkt_ackno, + uint64_t now_ns, + struct pending * pending) +{ + uint32_t hi_sacked; + uint32_t marked; + uint16_t n; + bool dsack; + uint16_t n_real; + + if (pkt.len < SACK_HDR_SIZE) + return; + + n = ntoh16(*(const uint16_t *) pkt.data); + if (n > SACK_MAX_BLOCKS) + return; + + if (pkt.len < SACK_HDR_SIZE + (size_t) n * SACK_BLOCK_SIZE) + return; + + STAT_BUMP(frcti, sack_rcv); + + dsack = sack_is_dsack(frcti, pkt.data, n, pkt_ackno); + n_real = n - (dsack ? 1 : 0); + + if (dsack) { + STAT_BUMP(frcti, dsack_rcv); + reo_wnd_on_dsack(frcti, now_ns); + } + + /* DSACK-only carries no new gap; don't enter recovery. */ + if (n_real > 0) + recovery_enter(frcti); + + marked = 0; + hi_sacked = sack_mark_blocks(frcti, pkt.data, n, &marked); + frcti->dup_thresh += marked; + + if (after(hi_sacked, frcti->snd_cr.lwe)) + sack_queue_rxm(frcti, hi_sacked, now_ns, pending); +} + +/* Emit and free queued packet copies. */ +static void pending_flush(struct frcti * frcti, + struct pending * pending) +{ + size_t i; + + for (i = 0; i < pending->sack_rxm_cnt; ++i) { + sack_rxm_snd(frcti, pending->sack_rxm[i].data, + pending->sack_rxm[i].len); + free(pending->sack_rxm[i].data); + } + + if (pending->fast_rxm.data != NULL) { + int ret = fast_rxm_send(frcti, pending->fast_rxm.data, + pending->fast_rxm.len); + if (ret == -EFLOWDOWN || ret == -ENOTALLOC) + STAT_BUMP(frcti, rxm_tx_dead); + free(pending->fast_rxm.data); + } +} + +/* Pre-DRF NACK: ask peer to retransmit HoL; seqno is informational. */ +static void frcti_nack_snd(struct frcti * frcti, + uint32_t seqno_unseen) +{ + struct ssm_pk_buff * spb; + struct frct_pci * pci; + + if (frct_ctrl_alloc(&spb, &pci, 0) < 0) + return; + + pci->flags = hton16(FRCT_NACK); + pci->seqno = hton32(seqno_unseen); + + frct_hcs_set(pci, false); + + frct_tx(frcti, spb); +} + +enum frct_act { + FRCT_ACTIVE, + FRCT_INACT_NEED_NACK, + FRCT_INACT_DROP, +}; + +/* On rcv inactivity: rebase on DRF, or arm pre-DRF NACK. Caller wrlock. */ +static enum frct_act rcv_inact_check(struct frcti * frcti, + uint16_t flags, + uint32_t seqno, + uint64_t now_ns) +{ + struct frct_cr * rcv_cr = &frcti->rcv_cr; + uint64_t cd; + + if (!ts_aged_ns(now_ns, rcv_cr->act, rcv_cr->inact)) + return FRCT_ACTIVE; + + if (flags & FRCT_DRF) { + if (same_epoch_drf(seqno, flags, rcv_cr)) + return FRCT_ACTIVE; + + /* Bootstrap or fresh epoch: rebase. */ + STAT_BUMP(frcti, drf_rebase); + release_rq(frcti); + STORE_RELEASE(&rcv_cr->lwe, seqno); + rcv_cr->rwe = seqno + RQ_SIZE; + rcv_cr->seqno = seqno; + return FRCT_ACTIVE; + } + + if (!(flags & FRCT_DATA)) + return FRCT_ACTIVE; + + /* Pre-DRF: nudge sender with NACK (rate-limited). */ + cd = frcti->srtt > 0 ? (uint64_t) frcti->srtt : NACK_COOLDOWN_NS; + if (!ts_aged_ns(now_ns, frcti->t_nack, cd)) + return FRCT_INACT_DROP; - return wait; + frcti->t_nack = now_ns; + STAT_BUMP(frcti, nack_snd); + + return FRCT_INACT_NEED_NACK; } -static int __frcti_snd(struct frcti * frcti, - struct shm_du_buff * sdb) +/* Both modes: bounded accept into rq[seqno]. Caller wrlock. */ +__attribute__((hot)) +static bool rq_accept(struct frcti * frcti, + uint32_t seqno, + size_t pos, + uint16_t flags) { - struct frct_pci * pci; - struct timespec now; - struct frct_cr * snd_cr; - struct frct_cr * rcv_cr; - uint32_t seqno; - bool rtx; + struct frct_cr * rcv_cr = &frcti->rcv_cr; + + if (!before(seqno, rcv_cr->rwe)) { + STAT_BUMP(frcti, out_rcv); + return false; + } + + if (!before(seqno, rcv_cr->lwe + RQ_SIZE)) { + STAT_BUMP(frcti, rqo_rcv); + return false; + } + + if (frcti->rcv_slots[pos].idx != -1) { + if (flags & FRCT_RXM) + STAT_BUMP(frcti, rxm_dup_rcv); + else + STAT_BUMP(frcti, dup_rcv); + /* RFC 2883 §4 case 2: in-window dup; sub-range marker. */ + frcti->dsack_seqno = seqno; + frcti->dsack_valid = true; + return false; + } + + return true; +} + +/* OOO arrival; throttle by min_gap + scoreboard dedup. */ +static bool sack_check(struct frcti * frcti, + uint32_t seqno, + uint64_t now_ns, + struct sack_args * out) +{ + struct frct_cr * rcv_cr = &frcti->rcv_cr; + uint64_t min_gap; + uint16_t n; + + if (!after(seqno, rcv_cr->lwe)) + return false; + + STAT_BUMP(frcti, ooo_rcv); + + /* SACK carries cum-ACK; bound by t_a like any other ACK. */ + if (ACK_AGED_OUT(rcv_cr->act, now_ns, frcti->t_a)) + return false; + + /* srtt/8 gate starved recovery under burst loss; floor to save CPU. */ + min_gap = (uint64_t) SACK_MIN_GAP_NS; + + if (!ts_aged_ns(now_ns, frcti->t_snd_sack, min_gap)) + return false; + + out->dsack = false; + n = dsack_consume(frcti, out->blocks); + if (n == 1) + out->dsack = true; + n += sack_blocks_build(frcti, out->blocks + n, + frcti->sack_n_max - n); + + if (!out->dsack + && rcv_cr->lwe == frcti->sack_lwe && n == frcti->sack_n) + return false; + + out->n = n; + out->ack = rcv_cr->lwe; + out->rwe = frcti_advert_rwe(frcti); + frcti->t_snd_sack = now_ns; + frcti->sack_lwe = rcv_cr->lwe; + frcti->sack_n = n; + + return true; +} + +/* Wire-dup of fresh DATA at an already-ACKed seqno. */ +static __inline__ bool is_dup_data(uint16_t flags, + uint32_t seqno, + uint32_t lwe) +{ + if (!(flags & FRCT_DATA)) + return false; + + if (flags & FRCT_RXM) + return false; + + return before(seqno, lwe); +} + +/* + * Wire-dup ACK packet: same seqno as the previous emission. Updates + * the dedup ackno on a fresh ACK; caller drops on true. + */ +static __inline__ bool is_dup_ack(struct frcti * frcti, + uint16_t flags, + uint32_t seqno) +{ + if (flags & FRCT_DATA) + return false; + + if (!(flags & FRCT_ACK)) + return false; + + if (seqno == frcti->rcv_cr.ackno) + return true; + + frcti->rcv_cr.ackno = seqno; + + return false; +} + +/* Caller wrlock. */ +__attribute__((cold)) +static void seqno_rotate(struct frcti * frcti, + uint64_t now_ns) +{ + struct frct_cr * snd_cr = &frcti->snd_cr; + + if (!ts_aged_ns(now_ns, snd_cr->act, snd_cr->inact)) + return; + /* Idle-on-wire ≠ idle e2e: don't orphan in-flight rxm. */ + if (snd_cr->seqno != snd_cr->lwe) + return; + + /* Avoid colliding with peer's current rcv window. */ + do { + random_buffer(&snd_cr->seqno, sizeof(snd_cr->seqno)); + } while (in_window(snd_cr->seqno, snd_cr)); + STORE_RELEASE(&snd_cr->lwe, snd_cr->seqno); + STORE_RELAXED(&snd_cr->rwe, snd_cr->lwe + START_WINDOW); + frcti->rtt_lwe = snd_cr->seqno; + frcti->in_recovery = false; + frcti->recovery_high = snd_cr->seqno; +} + +__attribute__((hot)) +static int frcti_snd(struct frcti * frcti, + struct ssm_pk_buff * spb, + uint16_t flags) +{ + struct frct_pci * pci; + struct frct_pci_stream * spci = NULL; + struct timespec now; + struct frct_cr * snd_cr; + struct frct_cr * rcv_cr; + uint32_t seqno; + uint16_t pci_flags = 0; + bool rtx; + uint64_t now_ns; + int64_t rcv_idle; + uint32_t probe_id = 0; + uint8_t probe_nonce[RTTP_NONCE_LEN] = { 0 }; + bool probe; + size_t payload_len = 0; assert(frcti); + /* Stream mode permits 0-byte sends for the EOS marker. */ + assert(ssm_pk_buff_len(spb) != 0 || frcti->stream); snd_cr = &frcti->snd_cr; rcv_cr = &frcti->rcv_cr; - timerwheel_move(); + tw_move_safe(); + + if (frcti->stream) + payload_len = ssm_pk_buff_len(spb); - pci = (struct frct_pci *) shm_du_buff_head_alloc(sdb, FRCT_PCILEN); + pci = FRCT_HDR_PUSH(spb, frcti); if (pci == NULL) return -ENOMEM; - memset(pci, 0, sizeof(*pci)); + memset(pci, 0, FRCT_PCILEN); + + if (frcti->stream) + spci = FRCT_SPCI(pci); clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); pthread_rwlock_wrlock(&frcti->lock); rtx = snd_cr->cflags & FRCTFRTX; - pci->flags |= FRCT_DATA; + pci_flags |= FRCT_DATA; + if (!frcti->stream) + pci_flags |= (flags & FRCT_FR_MASK); - /* Set DRF if there are no unacknowledged packets. */ - if (snd_cr->seqno == snd_cr->lwe) - pci->flags |= FRCT_DRF; + if (!frcti->stream && (flags & FRCT_FR_MASK) != FRCT_FR_SOLE) + STAT_BUMP(frcti, frag_snd); - /* Choose a new sequence number if sender inactivity expired. */ - if (now.tv_sec - snd_cr->act.tv_sec > snd_cr->inact) { - /* There are no unacknowledged packets. */ - assert(snd_cr->seqno == snd_cr->lwe); - random_buffer(&snd_cr->seqno, sizeof(snd_cr->seqno)); - snd_cr->lwe = snd_cr->seqno - 1; - snd_cr->rwe = snd_cr->lwe + START_WINDOW; + if (frcti->stream) { + if (flags & FRCT_FIN) + pci_flags |= FRCT_FIN; + + spci->start = hton32(frcti->snd_byte_next); + frcti->snd_byte_next += (uint32_t) payload_len; + spci->end = hton32(frcti->snd_byte_next); + STAT_ADD(frcti, strm_snd_byte, payload_len); } + if (snd_cr->seqno == snd_cr->lwe) + pci_flags |= FRCT_DRF; + + seqno_rotate(frcti, now_ns); + seqno = snd_cr->seqno; pci->seqno = hton32(seqno); - if (now.tv_sec - rcv_cr->act.tv_sec < rcv_cr->inact) { - pci->flags |= FRCT_FC; - *((uint32_t *) pci) |= hton32(rcv_cr->rwe & 0x00FFFFFF); + rcv_idle = ts_age_ns(now_ns, rcv_cr->act); + + if (rcv_idle < (int64_t) rcv_cr->inact) { + pci_flags |= FRCT_FC; + pci->window = hton32(frcti_advert_rwe(frcti)); } if (!rtx) { - snd_cr->lwe++; + STORE_RELEASE(&snd_cr->lwe, snd_cr->lwe + 1); + STORE_RELEASE(&snd_cr->rwe, snd_cr->lwe + RQ_SIZE); } else { - if (!frcti->probe) { - frcti->rttseq = snd_cr->seqno; - frcti->t_probe = now; - frcti->probe = true; - } - - if (now.tv_sec - rcv_cr->act.tv_sec <= frcti->a) { - pci->flags |= FRCT_ACK; + size_t p = RQ_SLOT(seqno); + frcti->snd_slots[p].time = now_ns; + /* Fresh send clears RTX bits. */ + frcti->snd_slots[p].flags = 0; + if (rcv_idle <= (int64_t) frcti->t_a) { + pci_flags |= FRCT_ACK; pci->ackno = hton32(rcv_cr->lwe); rcv_cr->seqno = rcv_cr->lwe; } } + pci->flags = hton16(pci_flags); + + frct_hcs_set(pci, frcti->stream); + snd_cr->seqno++; - snd_cr->act = now; + STORE_RELEASE(&snd_cr->act, now_ns); + + probe = rtt_probe_arm(frcti, now_ns, &probe_id, probe_nonce); pthread_rwlock_unlock(&frcti->lock); - if (rtx) - timerwheel_rxm(frcti, seqno, sdb); + if (probe) + frcti_rttp_snd(frcti, probe_id, 0, probe_nonce); + + if (rtx) { + rxm_arm(frcti, seqno, spb); + tlp_arm(frcti); + } return 0; } -static void rtt_estimator(struct frcti * frcti, - time_t mrtt) +/* + * Stream: 0-byte FRCT_FIN DATA so peer's flow_read returns 0 at this + * byte. Msg: control packet with FRCT_FIN flag, snd_cr.seqno carried + * in pci->ackno (sender packs via frcti_pkt_snd's ackno parameter). + */ +static void frcti_fin_snd(struct frcti * frcti) { - time_t srtt = frcti->srtt; - time_t rttvar = frcti->mdev; + struct ssm_pk_buff * spb; + bool already; + uint32_t fin_seqno; - if (srtt == 0) { /* first measurement */ - srtt = mrtt; - rttvar = mrtt >> 1; - } else { - time_t delta = mrtt - srtt; - srtt += (delta >> 3); - rttvar += (ABS(delta) - rttvar) >> 2; + if (!(frcti->snd_cr.cflags & FRCTFLINGER)) + return; + + pthread_rwlock_wrlock(&frcti->lock); + + already = frcti->snd_fin_sent; + frcti->snd_fin_sent = true; + fin_seqno = frcti->snd_cr.seqno; + + if (!already && !frcti->stream) + frcti->snd_fin_seqno = fin_seqno; + + pthread_rwlock_unlock(&frcti->lock); + + if (already) + return; + + if (!frcti->stream) { + frcti_pkt_snd(frcti, FRCT_FIN, fin_seqno, 0); + return; } - frcti->srtt = MAX(1000U, srtt); - frcti->mdev = MAX(100U, rttvar); - frcti->rto = MAX(RTO_MIN, frcti->srtt + (frcti->mdev << 1)); + if (frct_spb_reserve(frcti_data_hdr_len(frcti), &spb) < 0) + return; + + /* Reset spb to 0-len so frcti_snd's head_alloc populates PCI. */ + ssm_pk_buff_truncate(spb, 0); + + if (frcti_snd(frcti, spb, FRCT_FIN) < 0) { + frct_spb_release(spb); + return; + } + + if (frct_tx(frcti, spb) < 0) + return; + + pthread_rwlock_wrlock(&frcti->lock); + + frcti->snd_fin_seqno = frcti->snd_cr.seqno - 1; + + pthread_rwlock_unlock(&frcti->lock); } -static void __frcti_tick(void) +static bool final_ack_due(struct frcti * frcti, + struct frct_cr * rcv_cr, + uint64_t now_ns) { - timerwheel_move(); + if (rcv_cr->lwe == rcv_cr->seqno) + return false; + + if (ACK_AGED_OUT(rcv_cr->act, now_ns, frcti->t_a)) + return false; + + return true; } -/* Always queues the next application packet on the RQ. */ -static void __frcti_rcv(struct frcti * frcti, - struct shm_du_buff * sdb) +/* Snd-side has FLINGER cflag and unACK'd data below the FIN/seqno. */ +static __inline__ bool snd_drain_pending(struct frct_cr * snd_cr, + uint32_t edge) { - ssize_t idx; - size_t pos; - struct frct_pci * pci; - struct timespec now; - struct frct_cr * rcv_cr; - struct frct_cr * snd_cr; - uint32_t seqno; - uint32_t ackno; - uint32_t rwe; - int fd = -1; + if (!(snd_cr->cflags & FRCTFLINGER)) + return false; - assert(frcti); + return before(snd_cr->lwe, edge); +} + +/* Peer is still active and we haven't seen their FIN yet. */ +static __inline__ bool rcv_drain_pending(struct frcti * frcti, + struct frct_cr * rcv_cr, + uint64_t now_ns) +{ + if (frcti->rcv_fin_seen) + return false; + + return !ts_aged_ns(now_ns, rcv_cr->act, rcv_cr->inact); +} +/* Drain-loop predicate: snd-side unACK'd data OR peer still active. */ +static bool frcti_lingering(struct frcti * frcti) +{ + struct timespec now; + struct frct_cr * snd_cr; + struct frct_cr * rcv_cr; + uint32_t edge; + uint64_t now_ns; + bool snd_linger; + bool rcv_linger; + + /* Idempotent; emits FIN once per side, both stream and msg. */ + frcti_fin_snd(frcti); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + pthread_rwlock_rdlock(&frcti->lock); + + snd_cr = &frcti->snd_cr; rcv_cr = &frcti->rcv_cr; + + if (frcti->snd_fin_sent) + edge = frcti->snd_fin_seqno; + else + edge = snd_cr->seqno; + + snd_linger = snd_drain_pending(snd_cr, edge); + rcv_linger = rcv_drain_pending(frcti, rcv_cr, now_ns); + + pthread_rwlock_unlock(&frcti->lock); + + return snd_linger || rcv_linger; +} + +static time_t frcti_dealloc(struct frcti * frcti) +{ + struct timespec now; + struct frct_cr * snd_cr; + struct frct_cr * rcv_cr; + int ackno; + bool due; + int64_t now_ns; + int64_t rcv; + int64_t snd; + snd_cr = &frcti->snd_cr; + rcv_cr = &frcti->rcv_cr; + + /* Idempotent; usually already sent by frcti_lingering. */ + frcti_fin_snd(frcti); clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); - pci = (struct frct_pci *) shm_du_buff_head_release(sdb, FRCT_PCILEN); + pthread_rwlock_rdlock(&frcti->lock); - idx = shm_du_buff_get_idx(sdb); - seqno = ntoh32(pci->seqno); - pos = seqno & (RQ_SIZE - 1); + ackno = rcv_cr->lwe; + rcv = (int64_t)(rcv_cr->act + rcv_cr->inact) - now_ns; + snd = (int64_t)(snd_cr->act + snd_cr->inact) - now_ns; + due = final_ack_due(frcti, rcv_cr, now_ns); - pthread_rwlock_wrlock(&frcti->lock); + pthread_rwlock_unlock(&frcti->lock); - if (now.tv_sec - rcv_cr->act.tv_sec > rcv_cr->inact) { - if (pci->flags & FRCT_DRF) { /* New run. */ - rcv_cr->lwe = seqno; - rcv_cr->rwe = seqno + RQ_SIZE; - } else { - goto drop_packet; - } - } + if (due) + frcti_pkt_snd(frcti, FRCT_ACK, ackno, 0); - /* For now, just send an immediate window update. */ - if (pci->flags & FRCT_RDVS) { - fd = frcti->fd; - rwe = rcv_cr->rwe; - pthread_rwlock_unlock(&frcti->lock); + return (time_t) MAX((MAX(rcv, snd) / BILLION), 0); +} - __send_frct_pkt(fd, FRCT_FC, 0, rwe); +__attribute__((hot)) +static void frcti_rcv(struct frcti * frcti, + struct ssm_pk_buff * spb) +{ + ssize_t idx; + size_t pos; + struct frct_pci * pci; + struct timespec now; + uint64_t now_ns; + struct frct_cr * rcv_cr; + uint32_t seqno; + uint16_t flags; + buffer_t pkt; + struct pending pending = { 0 }; + bool in_order; + struct sack_args * sa = NULL; + bool send_sack = false; - shm_rdrbuff_remove(ai.rdrb, idx); + assert(frcti); + + rcv_cr = &frcti->rcv_cr; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + if (ssm_pk_buff_len(spb) < FRCT_PCILEN) { + frct_spb_release(spb); return; } - if (pci->flags & FRCT_ACK) { - ackno = ntoh32(pci->ackno); - if (after(ackno, frcti->snd_cr.lwe)) - frcti->snd_cr.lwe = ackno; + pci = FRCT_HDR_POP(spb, frct_pci); - if (frcti->probe && after(ackno, frcti->rttseq)) { - rtt_estimator(frcti, ts_diff_ns(&frcti->t_probe, &now)); - frcti->probe = false; - } + idx = ssm_pk_buff_get_off(spb); + seqno = ntoh32(pci->seqno); + pos = RQ_SLOT(seqno); + + flags = ntoh16(pci->flags); + + pkt.data = ssm_pk_buff_head(spb); + pkt.len = ssm_pk_buff_len(spb); + + if (flags & FRCT_RXM) + STAT_BUMP(frcti, rxm_rcv); + + /* Stateless / lock-free dispatches. spb released via ctrl_done. */ + if (flags & FRCT_KA) { + frcti_ka_rcv(frcti, pci, now_ns, flags); + goto ctrl_done; } - if (pci->flags & FRCT_FC) { - uint32_t rwe; + if (flags & FRCT_RTTP) { + frcti_rttp_rcv(frcti, pkt, now_ns); + goto ctrl_done; + } - rwe = ntoh32(*((uint32_t *)pci) & hton32(0x00FFFFFF)); - rwe |= snd_cr->rwe & 0xFF000000; + if (flags & FRCT_NACK) { + frcti_nack_rcv(frcti); + goto ctrl_done; + } - /* Rollover for 24 bit */ - if (before(rwe, snd_cr->rwe) && snd_cr->rwe - rwe > 0x007FFFFF) - rwe += 0x01000000; + if (flags & FRCT_RDVS) { + frcti_rdv_rcv(frcti); + goto ctrl_done; + } - snd_cr->rwe = rwe; + /* Msg-mode FIN: control packet, FIN seqno carried in pci->ackno. */ + if ((flags & FRCT_FIN) && !(flags & FRCT_DATA)) { + pthread_rwlock_wrlock(&frcti->lock); + if (!frcti->rcv_fin_seen) { + frcti->rcv_fin_seen = true; + frcti->rcv_byte_fin = ntoh32(pci->ackno); + } + pthread_rwlock_unlock(&frcti->lock); + goto ctrl_done; + } - pthread_mutex_lock(&frcti->mtx); - if (!frcti->open) { - frcti->open = true; - pthread_cond_broadcast(&frcti->cond); + pthread_rwlock_wrlock(&frcti->lock); + + /* rcv_inact_check is a no-op for non-DATA non-DRF packets. */ + if (flags & (FRCT_DATA | FRCT_DRF)) { + switch (rcv_inact_check(frcti, flags, seqno, now_ns)) { + case FRCT_INACT_NEED_NACK: + pthread_rwlock_unlock(&frcti->lock); + frcti_nack_snd(frcti, seqno - 1); + frct_spb_release(spb); + return; + case FRCT_INACT_DROP: + STAT_BUMP(frcti, inact_drop); + goto drop_packet; + case FRCT_ACTIVE: + /* FALLTHRU */ + default: + break; } - pthread_mutex_unlock(&frcti->mtx); } - if (!(pci->flags & FRCT_DATA)) + /* DATA-only act refresh: non-DATA would lock out DRF rebase. */ + if (flags & FRCT_DATA) + STORE_RELEASE(&rcv_cr->act, now_ns); + + /* Wire-dup ACK packet: same seqno as the previous emission. */ + if (is_dup_ack(frcti, flags, seqno)) { + STAT_BUMP(frcti, ack_dup_rcv); goto drop_packet; + } - if (before(seqno, rcv_cr->lwe)) { - rcv_cr->seqno = seqno; /* Ensures we send a new ACK. */ + /* Wire-dup of DATA: piggybacked ACK info already processed. */ + if (is_dup_data(flags, seqno, rcv_cr->lwe)) { + rcv_cr->seqno = seqno; + STAT_BUMP(frcti, dup_rcv); + /* RFC 2883 §4 case 1: dup below cum-ACK. */ + frcti->dsack_seqno = seqno; + frcti->dsack_valid = true; goto drop_packet; } - if (rcv_cr->cflags & FRCTFRTX) { + if (flags & FRCT_ACK) + frcti_ack_rcv(frcti, pci, flags, now_ns, &pending); - if (!before(seqno, rcv_cr->rwe)) /* Out of window. */ - goto drop_packet; + if (flags & FRCT_SACK) + frcti_sack_rcv(frcti, pkt, ntoh32(pci->ackno), + now_ns, &pending); - if (!before(seqno, rcv_cr->lwe + RQ_SIZE)) - goto drop_packet; /* Out of rq. */ + if (flags & FRCT_FC) + frcti_fc_rcv(frcti, pci); - if (frcti->rq[pos] != -1) - goto drop_packet; /* Duplicate in rq. */ + if (!(flags & FRCT_DATA)) + goto drop_packet; + + if (before(seqno, rcv_cr->lwe)) { + /* Bump rcv_cr.seqno to force ack_snd to fire on the dup. */ + rcv_cr->seqno = seqno; + if (flags & FRCT_RXM) + STAT_BUMP(frcti, rxm_dup_rcv); + else + STAT_BUMP(frcti, dup_rcv); + /* RFC 2883 §4 case 1: dup below cum-ACK. */ + frcti->dsack_seqno = seqno; + frcti->dsack_valid = true; + goto drop_packet; + } - fd = frcti->fd; + if (!rq_accept(frcti, seqno, pos, flags)) + goto drop_packet; + + if (frcti->stream) { + if (frcti_stream_data_rcv(frcti, spb, pos, flags) < 0) { + STAT_BUMP(frcti, strm_drop); + goto drop_packet; + } + /* spb consumed by stash; do not release in drop path. */ + spb = NULL; } else { - rcv_cr->lwe = seqno; + frcti_data_stash(frcti, idx, pos, flags); } - frcti->rq[pos] = idx; + /* Lazy alloc: only OOO arrivals can trigger a SACK send. */ + if (after(seqno, rcv_cr->lwe) && frcti->sack_n_max > 0) { + size_t sa_sz = sizeof(*sa) + + frcti->sack_n_max * sizeof(sa->blocks[0]); + sa = malloc(sa_sz); + /* If alloc fails, sack_check sees NULL and we skip SACK. */ + } - rcv_cr->act = now; + send_sack = sa != NULL && sack_check(frcti, seqno, now_ns, sa); + in_order = !after(seqno, rcv_cr->lwe); pthread_rwlock_unlock(&frcti->lock); - if (fd != -1) - timerwheel_ack(fd, frcti); + if (send_sack) { + STAT_BUMP(frcti, sack_snd); + if (sa->dsack) + STAT_BUMP(frcti, dsack_snd); + frcti_sack_snd(frcti, sa); + } else if (in_order) { + ack_arm(frcti); + } + + if ((flags & FRCT_ACK) && frcti->snd_cr.seqno != frcti->snd_cr.lwe) + tlp_arm(frcti); + + pending_flush(frcti, &pending); + + frcti_rcv_probe(frcti, now_ns); + free(sa); + return; + + ctrl_done: + frct_spb_release(spb); return; drop_packet: pthread_rwlock_unlock(&frcti->lock); + frct_spb_release(spb); + /* with_sack=true: ack_snd no-ops if neither dsack nor SACK is due. */ + ack_snd(frcti, true); - send_frct_pkt(frcti); - - shm_rdrbuff_remove(ai.rdrb, idx); - return; + pending_flush(frcti, &pending); + free(sa); } -/* Filter fqueue events for non-data packets */ -int frcti_filter(struct fqueue * fq) -{ - struct shm_du_buff * sdb; - int fd; - ssize_t idx; - struct frcti * frcti; - struct shm_rbuff * rb; +/* NULL-shim macros for the no-FRCT case. */ - while (fq->next < fq->fqsize) { - if (fq->fqueue[fq->next + 1] != FLOW_PKT) - return 1; +#define FRCTI_SND(frcti, spb, flags) \ + ((frcti) == NULL ? 0 : frcti_snd((frcti), (spb), (flags))) - pthread_rwlock_rdlock(&ai.lock); +#define FRCTI_RCV(frcti, spb) \ + do { \ + if ((frcti) != NULL) \ + frcti_rcv((frcti), (spb)); \ + } while (0) - fd = ai.ports[fq->fqueue[fq->next]].fd; - rb = ai.flows[fd].rx_rb; - frcti = ai.flows[fd].frcti; +#define FRCTI_PDU_READY(frcti) \ + ((frcti) != NULL && frcti_pdu_ready(frcti)) - if (frcti == NULL) { - pthread_rwlock_unlock(&ai.lock); - return 1; - } +#define FRCTI_CONSUME(frcti, buf, count) \ + ((frcti) == NULL ? (ssize_t) -EAGAIN \ + : (frcti)->stream \ + ? frcti_consume_stream((frcti), (buf), (count)) \ + : frcti_consume((frcti), (buf), (count))) - if (__frcti_pdu_ready(frcti) >= 0) { - pthread_rwlock_unlock(&ai.lock); - return 1; - } +#define FRCTI_IS_FRTX(frcti) \ + ((frcti) != NULL && ((frcti)->rcv_cr.cflags & FRCTFRTX)) - idx = shm_rbuff_read(rb); - if (idx < 0) { - pthread_rwlock_unlock(&ai.lock); - return 0; - } +#define FRCTI_IS_STREAM(frcti) ((frcti) != NULL && (frcti)->stream) - sdb = shm_rdrbuff_get(ai.rdrb, idx); +#define FRCTI_PAYLOAD_CAP(frcti) \ + ((frcti)->frag_mtu - frcti_data_hdr_len(frcti)) - __frcti_rcv(frcti, sdb); +#define FRCTI_NEEDS_FRAG(frcti, count) \ + ((frcti) != NULL && (count) > FRCTI_PAYLOAD_CAP(frcti)) - if (__frcti_pdu_ready(frcti) >= 0) { - pthread_rwlock_unlock(&ai.lock); - return 1; - } +#define FRCTI_IS_WINDOW_OPEN(frcti) \ + ((frcti) == NULL ? true : frcti_is_window_open(frcti)) - pthread_rwlock_unlock(&ai.lock); +#define FRCTI_IS_WINDOW_OPEN_N(frcti, n) \ + ((frcti) == NULL ? true : frcti_is_window_open_n((frcti), (n))) - fq->next += 2; - } +#define FRCTI_LINGERING(frcti) \ + ((frcti) == NULL ? false : frcti_lingering(frcti)) + +#define FRCTI_DEALLOC(frcti) \ + ((frcti) == NULL ? (time_t) 0 : frcti_dealloc(frcti)) - return fq->next < fq->fqsize; -} diff --git a/src/lib/hash.c b/src/lib/hash.c index e5f90ee0..62bbf2b8 100644 --- a/src/lib/hash.c +++ b/src/lib/hash.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Hashing * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This implementation is adapted and redistributed from the RHASH * project @@ -29,43 +29,59 @@ #include "config.h" +#include <ouroboros/endian.h> #include <ouroboros/hash.h> -#ifndef HAVE_LIBGCRYPT +#ifdef HAVE_LIBGCRYPT +#include <gcrypt.h> +#else #include <ouroboros/crc32.h> #include <ouroboros/md5.h> #include <ouroboros/sha3.h> -#else -#include <gcrypt.h> #endif +#include <ouroboros/crc8.h> +#include <ouroboros/crc16.h> +#include <ouroboros/crc64.h> #include <string.h> #include <assert.h> #include <stdbool.h> +#ifdef HAVE_LIBGCRYPT +int gcry_algo_tbl [] = { + /* DIR_HASH policies first */ + GCRY_MD_SHA3_224, + GCRY_MD_SHA3_256, + GCRY_MD_SHA3_384, + GCRY_MD_SHA3_512, + /* Below for internal use only */ + GCRY_MD_CRC32, + GCRY_MD_MD5, +}; +#else +int hash_len_tbl [] = { + /* DIR_HASH policies first */ + SHA3_224_HASH_LEN, + SHA3_256_HASH_LEN, + SHA3_384_HASH_LEN, + SHA3_512_HASH_LEN, + /* Below for internal use only */ + CRC32_HASH_LEN, + MD5_HASH_LEN +}; +#endif + uint16_t hash_len(enum hash_algo algo) { + if (algo == HASH_CRC8) + return CRC8_HASH_LEN; + if (algo == HASH_CRC16) + return CRC16_HASH_LEN; + if (algo == HASH_CRC64) + return CRC64_HASH_LEN; #ifdef HAVE_LIBGCRYPT - return (uint16_t) gcry_md_get_algo_dlen(algo); + return (uint16_t) gcry_md_get_algo_dlen(gcry_algo_tbl[algo]); #else - switch (algo) { - case HASH_CRC32: - return CRC32_HASH_LEN; - case HASH_MD5: - return MD5_HASH_LEN; - case HASH_SHA3_224: - return SHA3_224_HASH_LEN; - case HASH_SHA3_256: - return SHA3_256_HASH_LEN; - case HASH_SHA3_384: - return SHA3_384_HASH_LEN; - case HASH_SHA3_512: - return SHA3_512_HASH_LEN; - default: - assert(false); - break; - } - - return 0; + return hash_len_tbl[algo]; #endif } @@ -74,16 +90,39 @@ void mem_hash(enum hash_algo algo, const uint8_t * buf, size_t len) { -#ifdef HAVE_LIBGCRYPT - gcry_md_hash_buffer(algo, dst, buf, len); -#else +#ifndef HAVE_LIBGCRYPT struct sha3_ctx sha3_ctx; struct md5_ctx md5_ctx; +#endif + if (algo == HASH_CRC8) { + uint8_t crc = 0; + + crc8_autosar(&crc, buf, len); + *(uint8_t *) dst = crc; + return; + } + if (algo == HASH_CRC16) { + uint16_t crc = 0; + + crc16_ccitt_false(&crc, buf, len); + *(uint16_t *) dst = htobe16(crc); + return; + } + if (algo == HASH_CRC64) { + uint64_t crc = 0; + crc64_nvme(&crc, buf, len); + *(uint64_t *) dst = htobe64(crc); + return; + } +#ifdef HAVE_LIBGCRYPT + gcry_md_hash_buffer(gcry_algo_tbl[algo], dst, buf, len); +#else switch (algo) { case HASH_CRC32: memset(dst, 0, CRC32_HASH_LEN); crc32((uint32_t *) dst, buf, len); + *(uint32_t *) dst = htobe32(*(uint32_t *) dst); break; case HASH_MD5: rhash_md5_init(&md5_ctx); diff --git a/src/lib/ipcp_config.proto b/src/lib/ipcp_config.proto deleted file mode 100644 index 7bf5329e..00000000 --- a/src/lib/ipcp_config.proto +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * Layer configuration message - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -syntax = "proto2"; - -message layer_info_msg { - required string layer_name = 1; - required uint32 dir_hash_algo = 2; -} - -message ipcp_config_msg { - required layer_info_msg layer_info = 1; - required int32 ipcp_type = 2; - // Config for unicast IPCP - optional uint32 addr_size = 3; - optional uint32 eid_size = 4; - optional uint32 max_ttl = 5; - optional uint32 addr_auth_type = 6; - optional uint32 routing_type = 7; - optional uint32 cong_avoid = 8; - // Config for UDP - optional uint32 ip_addr = 9; - optional uint32 dns_addr = 10; - optional uint32 clt_port = 11; - optional uint32 srv_port = 12; - // Config for the Ethernet - optional string dev = 13; - // Config for DIX Ethernet - optional uint32 ethertype = 14; -} - -enum enroll_code { - ENROLL_REQ = 1; - ENROLL_BOOT = 2; - ENROLL_DONE = 4; -}; - -message enroll_msg { - required enroll_code code = 1; - optional ipcp_config_msg conf = 2; - optional int64 t_sec = 3; - optional uint32 t_nsec = 4; - optional int32 result = 5; -};
\ No newline at end of file diff --git a/src/lib/irm.c b/src/lib/irm.c index 42ad74fa..c62701aa 100644 --- a/src/lib/irm.c +++ b/src/lib/irm.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * The API to instruct the IRM * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -31,28 +31,37 @@ #include <ouroboros/irm.h> #include <ouroboros/utils.h> #include <ouroboros/sockets.h> +#include <ouroboros/protobuf.h> #include <stdbool.h> #include <string.h> #include <stdlib.h> #include <sys/stat.h> -pid_t irm_create_ipcp(const char * name, - enum ipcp_type type) +int irm_create_ipcp(const char * name, + enum ipcp_type type) { - irm_msg_t msg = IRM_MSG__INIT; - irm_msg_t * recv_msg = NULL; - int ret = -1; + irm_msg_t msg = IRM_MSG__INIT; + irm_msg_t * recv_msg; + int ret; + struct ipcp_info info; - if (name == NULL) + if (name == NULL || strlen(name) > IPCP_NAME_SIZE) return -EINVAL; - msg.code = IRM_MSG_CODE__IRM_CREATE_IPCP; - msg.name = (char *) name; - msg.has_ipcp_type = true; - msg.ipcp_type = type; + memset(&info, 0, sizeof(info)); + + info.type = type; + strcpy(info.name, name); + + msg.code = IRM_MSG_CODE__IRM_CREATE_IPCP; + msg.ipcp_info = ipcp_info_s_to_msg(&info); + if (msg.ipcp_info == NULL) + return -ENOMEM; recv_msg = send_recv_irm_msg(&msg); + ipcp_info_msg__free_unpacked(msg.ipcp_info, NULL); + if (recv_msg == NULL) return -EIRMD; @@ -98,11 +107,9 @@ int irm_destroy_ipcp(pid_t pid) int irm_bootstrap_ipcp(pid_t pid, const struct ipcp_config * conf) { - irm_msg_t msg = IRM_MSG__INIT; - ipcp_config_msg_t config = IPCP_CONFIG_MSG__INIT; - layer_info_msg_t layer_info = LAYER_INFO_MSG__INIT; - irm_msg_t * recv_msg = NULL; - int ret = -1; + irm_msg_t msg = IRM_MSG__INIT; + irm_msg_t * recv_msg; + int ret; if (pid == -1 || conf == NULL) return -EINVAL; @@ -110,58 +117,10 @@ int irm_bootstrap_ipcp(pid_t pid, msg.code = IRM_MSG_CODE__IRM_BOOTSTRAP_IPCP; msg.has_pid = true; msg.pid = pid; - - config.layer_info = &layer_info; - msg.conf = &config; - layer_info.layer_name = (char *) conf->layer_info.layer_name; - - config.ipcp_type = conf->type; - - if (conf->type != IPCP_UDP) - layer_info.dir_hash_algo = conf->layer_info.dir_hash_algo; - - switch (conf->type) { - case IPCP_UNICAST: - config.has_addr_size = true; - config.addr_size = conf->addr_size; - config.has_eid_size = true; - config.eid_size = conf->eid_size; - config.has_max_ttl = true; - config.max_ttl = conf->max_ttl; - config.has_addr_auth_type = true; - config.addr_auth_type = conf->addr_auth_type; - config.has_routing_type = true; - config.routing_type = conf->routing_type; - config.has_cong_avoid = true; - config.cong_avoid = conf->cong_avoid; - break; - case IPCP_UDP: - config.has_ip_addr = true; - config.ip_addr = conf->ip_addr; - config.has_dns_addr = true; - config.dns_addr = conf->dns_addr; - config.has_srv_port = true; - config.srv_port = conf->srv_port; - config.has_clt_port = true; - config.clt_port = conf->clt_port; - break; - case IPCP_LOCAL: - case IPCP_BROADCAST: - case IPCP_RAPTOR: - break; - case IPCP_ETH_LLC: - config.dev = conf->dev; - break; - case IPCP_ETH_DIX: - config.dev = conf->dev; - config.has_ethertype = true; - config.ethertype = conf->ethertype; - break; - default: - return -EIPCPTYPE; - } + msg.conf = ipcp_config_s_to_msg(conf); recv_msg = send_recv_irm_msg(&msg); + ipcp_config_msg__free_unpacked(msg.conf, NULL); if (recv_msg == NULL) return -EIRMD; @@ -181,8 +140,7 @@ int irm_connect_ipcp(pid_t pid, const char * component, qosspec_t qs) { - irm_msg_t msg = IRM_MSG__INIT; - qosspec_msg_t qs_msg = QOSSPEC_MSG__INIT; + irm_msg_t msg = IRM_MSG__INIT; irm_msg_t * recv_msg; int ret; @@ -192,10 +150,11 @@ int irm_connect_ipcp(pid_t pid, msg.comp = (char *) component; msg.has_pid = true; msg.pid = pid; - qs_msg = spec_to_msg(&qs); - msg.qosspec = &qs_msg; + msg.qosspec = qos_spec_s_to_msg(&qs); recv_msg = send_recv_irm_msg(&msg); + qosspec_msg__free_unpacked(msg.qosspec, NULL); + if (recv_msg == NULL) return -EIRMD; @@ -239,7 +198,7 @@ int irm_disconnect_ipcp(pid_t pid, return ret; } -ssize_t irm_list_ipcps(struct ipcp_info ** ipcps) +ssize_t irm_list_ipcps(struct ipcp_list_info ** ipcps) { irm_msg_t msg = IRM_MSG__INIT; irm_msg_t * recv_msg; @@ -409,50 +368,69 @@ int irm_bind_program(const char * prog, int argc, char ** argv) { - irm_msg_t msg = IRM_MSG__INIT; - irm_msg_t * recv_msg = NULL; - int ret = -1; - char * full_name; + irm_msg_t msg = IRM_MSG__INIT; + irm_msg_t * recv_msg; + char ** exec; + int ret; + int i; if (prog == NULL || name == NULL) return -EINVAL; - full_name = strdup(prog); - if (full_name == NULL) - return -ENOMEM; + exec = malloc((argc + 2) * sizeof(*exec)); + if (exec== NULL) { + ret = -ENOMEM; + goto fail_exec; + } - if ((ret = check_prog_path(&full_name)) < 0) { - free(full_name); - return ret; + exec[0] = strdup(prog); + if (exec[0] == NULL) { + ret = -ENOMEM; + goto fail_exec0; } + ret = check_prog_path(&exec[0]); + if (ret < 0) + goto fail; + + for (i = 0; i < argc; i++) + exec[i + 1] = argv[i]; + + exec[argc + 1] = ""; + msg.code = IRM_MSG_CODE__IRM_BIND_PROGRAM; msg.name = (char *) name; - msg.prog = full_name; - if (argv != NULL) { - msg.n_args = argc; - msg.args = (char **) argv; - } + msg.n_exec = argc + 2; + msg.exec = exec; msg.has_opts = true; msg.opts = opts; recv_msg = send_recv_irm_msg(&msg); - - free(full_name); - - if (recv_msg == NULL) - return -EIRMD; + if (recv_msg == NULL) { + ret = -EIRMD; + goto fail; + } if (!recv_msg->has_result) { irm_msg__free_unpacked(recv_msg, NULL); - return -1; + ret = -EPERM; + goto fail; } ret = recv_msg->result; irm_msg__free_unpacked(recv_msg, NULL); + free(exec[0]); + free(exec); + + return ret; + fail: + free(exec[0]); + fail_exec0: + free(exec); + fail_exec: return ret; } @@ -545,32 +523,23 @@ int irm_unbind_process(pid_t pid, return ret; } -int irm_create_name(const char * name, - enum pol_balance pol) +int irm_create_name(struct name_info * info) { irm_msg_t msg = IRM_MSG__INIT; - name_info_msg_t ni_msg = NAME_INFO_MSG__INIT; irm_msg_t * recv_msg; int ret; - if (name == NULL) + if (info == NULL) return -EINVAL; - msg.code = IRM_MSG_CODE__IRM_CREATE_NAME; - ni_msg.name = (char *) name; - ni_msg.pol_lb = pol; - msg.n_names = 1; - - msg.names = malloc(sizeof(*msg.names)); - if (msg.names == NULL) { - return -ENOMEM; - } - - msg.names[0] = &ni_msg; + msg.code = IRM_MSG_CODE__IRM_CREATE_NAME; + msg.name_info = name_info_s_to_msg(info); + if (msg.name_info == NULL) + goto fail_info_msg; recv_msg = send_recv_irm_msg(&msg); - free(msg.names); + name_info_msg__free_unpacked(msg.name_info, NULL); if (recv_msg == NULL) return -EIRMD; @@ -584,6 +553,9 @@ int irm_create_name(const char * name, irm_msg__free_unpacked(recv_msg, NULL); return ret; + + fail_info_msg: + return -ENOMEM; } int irm_destroy_name(const char * name) @@ -642,7 +614,7 @@ ssize_t irm_list_names(struct name_info ** names) return 0; } - *names = malloc(nr * sizeof(**names)); + *names = calloc(nr, sizeof(**names)); if (*names == NULL) { irm_msg__free_unpacked(recv_msg, NULL); return -ENOMEM; diff --git a/src/lib/irmd_messages.proto b/src/lib/irmd_messages.proto deleted file mode 100644 index 5b23ee9d..00000000 --- a/src/lib/irmd_messages.proto +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * IRMd message - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -syntax = "proto2"; - -import "ipcp_config.proto"; -import "qosspec.proto"; - -enum irm_msg_code { - IRM_CREATE_IPCP = 1; - IPCP_CREATE_R = 2; - IRM_DESTROY_IPCP = 3; - IRM_LIST_IPCPS = 4; - IRM_BOOTSTRAP_IPCP = 5; - IRM_ENROLL_IPCP = 6; - IRM_CONNECT_IPCP = 7; - IRM_DISCONNECT_IPCP = 8; - IRM_BIND_PROGRAM = 9; - IRM_UNBIND_PROGRAM = 10; - IRM_PROC_ANNOUNCE = 11; - IRM_BIND_PROCESS = 12; - IRM_UNBIND_PROCESS = 13; - IRM_CREATE_NAME = 14; - IRM_DESTROY_NAME = 15; - IRM_LIST_NAMES = 16; - IRM_REG_NAME = 17; - IRM_UNREG_NAME = 18; - IRM_FLOW_ALLOC = 19; - IRM_FLOW_ACCEPT = 20; - IRM_FLOW_JOIN = 21; - IRM_FLOW_DEALLOC = 22; - IPCP_FLOW_REQ_ARR = 23; - IPCP_FLOW_ALLOC_REPLY = 24; - IRM_REPLY = 25; -}; - -message ipcp_info_msg { - required uint32 pid = 1; - required uint32 type = 2; - required string name = 3; - required string layer = 4; -}; - -message name_info_msg { - required string name = 1; - required uint32 pol_lb = 2; -}; - -message irm_msg { - required irm_msg_code code = 1; - optional string prog = 2; - optional sint32 pid = 3; - optional string name = 4; - optional uint32 ipcp_type = 5; - optional string layer = 6; - repeated string args = 7; - optional sint32 response = 8; - optional string dst = 9; - optional bytes hash = 10; - optional sint32 flow_id = 11; - optional qosspec_msg qosspec = 12; - optional ipcp_config_msg conf = 13; - optional uint32 opts = 14; - repeated ipcp_info_msg ipcps = 15; - repeated name_info_msg names = 16; - optional uint32 timeo_sec = 17; - optional uint32 timeo_nsec = 18; - optional string comp = 19; - optional bytes pk = 20; /* piggyback */ - optional sint32 result = 21; -}; diff --git a/src/lib/list.c b/src/lib/list.c deleted file mode 100644 index 83950a5c..00000000 --- a/src/lib/list.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * Simple doubly linked list implementation. - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -#include <ouroboros/list.h> - -#include <stddef.h> - -void list_head_init(struct list_head * h) -{ - h->nxt = h; - h->prv = h; -} - -static void add_list(struct list_head * n, - struct list_head * prv, - struct list_head * nxt) -{ - nxt->prv = n; - n->nxt = nxt; - n->prv = prv; - prv->nxt = n; -} - -static void del_list(struct list_head * prv, - struct list_head * nxt) -{ - nxt->prv = prv; - prv->nxt = nxt; -} - -void list_add(struct list_head * n, - struct list_head * h) -{ - add_list(n, h, h->nxt); -} - -void list_add_tail(struct list_head * n, - struct list_head * h) -{ - add_list(n, h->prv, h); -} - -void list_del(struct list_head * e) -{ - del_list(e->prv, e->nxt); - e->nxt = e->prv = NULL; -} - -bool list_is_empty(struct list_head * h) -{ - return h->nxt == h; -} - -void list_move(struct list_head * n, - struct list_head * h) -{ - del_list(n->prv, n->nxt); - add_list(n, h, h->nxt); -} diff --git a/src/lib/lockfile.c b/src/lib/lockfile.c index ac3c3062..0c18dfc1 100644 --- a/src/lib/lockfile.c +++ b/src/lib/lockfile.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Lockfile for Ouroboros * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -36,81 +36,69 @@ #include <sys/stat.h> #define LF_SIZE (sizeof(pid_t)) +#define LF_PROT (PROT_READ | PROT_WRITE) struct lockfile { pid_t * pid; }; -struct lockfile * lockfile_create() { - int fd; - mode_t mask; - struct lockfile * lf = malloc(sizeof(*lf)); - if (lf == NULL) - return NULL; - - mask = umask(0); +static struct lockfile * __lockfile_open(int oflag) +{ + int fd; + struct lockfile * lf; - fd = shm_open(SHM_LOCKFILE_NAME, O_CREAT | O_EXCL | O_RDWR, 0666); - if (fd == -1) { - free(lf); - return NULL; - } + lf = malloc(sizeof(*lf)); + if (lf == NULL) + goto fail_lockfile; - umask(mask); + fd = shm_open(SHM_LOCKFILE_NAME, oflag, 0666); + if (fd == -1) + goto fail_shm_open; - if (ftruncate(fd, LF_SIZE - 1) < 0) { - free(lf); - return NULL; - } + if ((oflag & O_CREAT) && ftruncate(fd, LF_SIZE) < 0) + goto fail_truncate; - lf->pid = mmap(NULL, - LF_SIZE, PROT_READ | PROT_WRITE, - MAP_SHARED, - fd, - 0); + lf->pid = mmap(NULL, LF_SIZE, LF_PROT, MAP_SHARED, fd, 0); + if (lf->pid == MAP_FAILED) + goto fail_mmap; close (fd); - if (lf->pid == MAP_FAILED) { - shm_unlink(SHM_LOCKFILE_NAME); - free(lf); - return NULL; - } - - *lf->pid = getpid(); - return lf; + + fail_mmap: + shm_unlink(SHM_LOCKFILE_NAME); + fail_truncate: + close(fd); + fail_shm_open: + free(lf); + fail_lockfile: + return NULL; } -struct lockfile * lockfile_open() { - int fd; - struct lockfile * lf = malloc(sizeof(*lf)); - if (lf == NULL) - return NULL; +struct lockfile * lockfile_create(void) +{ + struct lockfile * lf; + mode_t mask; - fd = shm_open(SHM_LOCKFILE_NAME, O_RDWR, 0666); - if (fd < 0) { - free(lf); - return NULL; - } + mask = umask(0); - lf->pid = mmap(NULL, - LF_SIZE, PROT_READ | PROT_WRITE, - MAP_SHARED, - fd, - 0); + lf = __lockfile_open(O_CREAT | O_EXCL | O_RDWR); + if (lf == NULL) + return NULL; - close(fd); + umask(mask); - if (lf->pid == MAP_FAILED) { - shm_unlink(SHM_LOCKFILE_NAME); - free(lf); - return NULL; - } + *lf->pid = getpid(); return lf; } +struct lockfile * lockfile_open(void) +{ + return __lockfile_open(O_RDWR); +} + void lockfile_close(struct lockfile * lf) { assert(lf); @@ -127,11 +115,9 @@ void lockfile_destroy(struct lockfile * lf) if (getpid() != *lf->pid && kill(*lf->pid, 0) == 0) return; - munmap(lf->pid, LF_SIZE); + lockfile_close(lf); shm_unlink(SHM_LOCKFILE_NAME); - - free(lf); } pid_t lockfile_owner(struct lockfile * lf) diff --git a/src/lib/logs.c b/src/lib/logs.c index 986b01ec..af03de10 100644 --- a/src/lib/logs.c +++ b/src/lib/logs.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Logging facilities * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License diff --git a/src/lib/md5.c b/src/lib/md5.c index 4dfa8f66..2412b909 100644 --- a/src/lib/md5.c +++ b/src/lib/md5.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * MD5 algorithm * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This implementation is adapted and redistributed from the RHASH * project diff --git a/src/lib/notifier.c b/src/lib/notifier.c index dd211dd3..10bdb28d 100644 --- a/src/lib/notifier.c +++ b/src/lib/notifier.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Notifier event system using callbacks * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -25,8 +25,9 @@ #include <ouroboros/errno.h> #include <ouroboros/notifier.h> #include <ouroboros/list.h> +#include <ouroboros/utils.h> +#include <ouroboros/pthread.h> -#include <pthread.h> #include <stdlib.h> struct listener { @@ -75,8 +76,7 @@ void notifier_event(int event, pthread_rwlock_rdlock(¬ifier.lock); - pthread_cleanup_push((void (*) (void *)) pthread_rwlock_unlock, - (void *) ¬ifier.lock) + pthread_cleanup_push(__cleanup_rwlock_unlock, ¬ifier.lock) list_for_each(p, ¬ifier.listeners) { struct listener * l = list_entry(p, struct listener, next); @@ -95,18 +95,14 @@ int notifier_reg(notifier_fn_t callback, pthread_rwlock_wrlock(¬ifier.lock); list_for_each(p, ¬ifier.listeners) { - struct listener * l = list_entry(p, struct listener, next); - if (l->callback == callback) { - pthread_rwlock_unlock(¬ifier.lock); - return -EPERM; - } + l = list_entry(p, struct listener, next); + if (l->callback == callback) + goto fail; } l = malloc(sizeof(*l)); - if (l == NULL) { - pthread_rwlock_unlock(¬ifier.lock); - return -ENOMEM; - } + if (l == NULL) + goto fail; l->callback = callback; l->obj = obj; @@ -116,6 +112,10 @@ int notifier_reg(notifier_fn_t callback, pthread_rwlock_unlock(¬ifier.lock); return 0; + fail: + pthread_rwlock_unlock(¬ifier.lock); + return -1; + } void notifier_unreg(notifier_fn_t callback) diff --git a/src/lib/cacep.proto b/src/lib/pb/cep.proto index 4f4ae9df..14a85cfa 100644 --- a/src/lib/cacep.proto +++ b/src/lib/pb/cep.proto @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * - * Message for Connection Information in CACEP + * Message for Connection Information in OCEP * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -27,7 +27,7 @@ message fixed_conc_syntax_msg { repeated uint32 lens = 2; } -message cacep_msg { +message cep_msg { required string comp_name = 1; required string protocol = 2; required int32 pref_version = 3; @@ -36,4 +36,4 @@ message cacep_msg { repeated int32 supp_syntax = 6; optional fixed_conc_syntax_msg syntax_spec = 7; required uint64 address = 8; -}
\ No newline at end of file +} diff --git a/src/lib/pb/enroll.proto b/src/lib/pb/enroll.proto new file mode 100644 index 00000000..37226b4d --- /dev/null +++ b/src/lib/pb/enroll.proto @@ -0,0 +1,41 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Enrollment protocol + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +syntax = "proto2"; +import "ipcp_config.proto"; + +message enroll_req_msg { + required bytes id = 1; +} + +message enroll_resp_msg { + required bytes id = 1; + required int64 t_sec = 2; + required int32 t_nsec = 3; + required int32 response = 4; + optional ipcp_config_msg conf = 5; +} + +message enroll_ack_msg { + required bytes id = 1; + required int32 result = 2; +} diff --git a/src/lib/ipcpd_messages.proto b/src/lib/pb/ipcp.proto index 809117b8..406b8d9c 100644 --- a/src/lib/ipcpd_messages.proto +++ b/src/lib/pb/ipcp.proto @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * - * IPCPd message + * Messages sent to IPCPds * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -23,7 +23,8 @@ syntax = "proto2"; import "ipcp_config.proto"; -import "qosspec.proto"; +import "model.proto"; + enum ipcp_msg_code { IPCP_BOOTSTRAP = 1; @@ -38,7 +39,7 @@ enum ipcp_msg_code { IPCP_CONNECT = 10; IPCP_DISCONNECT = 11; IPCP_REPLY = 12; -}; +} message ipcp_msg { required ipcp_msg_code code = 1; @@ -53,5 +54,7 @@ message ipcp_msg { optional int32 response = 10; optional string comp = 11; optional uint32 timeo_sec = 12; - optional int32 result = 13; -}; + optional sint32 mpl = 13; /* MPL in ms. */ + optional int32 result = 14; + optional uint32 uid = 15; /* 0 = GSPP, >0 = PUP uid */ +} diff --git a/src/lib/pb/ipcp_config.proto b/src/lib/pb/ipcp_config.proto new file mode 100644 index 00000000..eac4da37 --- /dev/null +++ b/src/lib/pb/ipcp_config.proto @@ -0,0 +1,93 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Layer configuration message + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +syntax = "proto2"; + +import "model.proto"; + + +message ls_config_msg { + required uint32 pol = 1; + required uint32 t_recalc = 2; + required uint32 t_update = 3; + required uint32 t_timeo = 4; +} + +message routing_config_msg { + required uint32 pol = 1; + optional ls_config_msg ls = 2; +} + +message dt_config_msg { + required uint32 addr_size = 1; + required uint32 eid_size = 2; + required uint32 max_ttl = 3; + required routing_config_msg routing = 4; +} + +message dir_dht_config_msg { + required uint32 alpha = 1; + required uint32 k = 2; + required uint32 t_expire = 3; + required uint32 t_refresh = 4; + required uint32 t_replicate = 5; + required uint64 peer = 6; +} + +message dir_config_msg { + required uint32 pol = 1; + optional dir_dht_config_msg dht = 2; +} + +message uni_config_msg { + required dt_config_msg dt = 1; + required dir_config_msg dir = 2; + required uint32 addr_auth_type = 3; + required uint32 cong_avoid = 4; +} + +message eth_config_msg { + required string dev = 1; + required uint32 ethertype = 2; +} + +message udp4_config_msg { + required uint32 ip_addr = 1; + required uint32 port = 2; + required uint32 dns_addr = 3; /* set to 0 if unused */ +} + +message udp6_config_msg { + required bytes ip_addr = 1; + required uint32 port = 2; + required bytes dns_addr = 3; /* set to NULL if unused */ +} + + +message ipcp_config_msg { + required layer_info_msg layer_info = 1; + required uint32 ipcp_type = 2; + optional uni_config_msg unicast = 3; + optional udp4_config_msg udp4 = 4; + optional udp6_config_msg udp6 = 5; + optional eth_config_msg eth = 6; +} diff --git a/src/lib/pb/irm.proto b/src/lib/pb/irm.proto new file mode 100644 index 00000000..5de860a5 --- /dev/null +++ b/src/lib/pb/irm.proto @@ -0,0 +1,99 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Messages sent to IRMd + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +syntax = "proto2"; + +import "ipcp_config.proto"; +import "model.proto"; + +enum irm_msg_code { + IRM_CREATE_IPCP = 1; + IPCP_CREATE_R = 2; + IRM_DESTROY_IPCP = 3; + IRM_LIST_IPCPS = 4; + IRM_BOOTSTRAP_IPCP = 5; + IRM_ENROLL_IPCP = 6; + IRM_CONNECT_IPCP = 7; + IRM_DISCONNECT_IPCP = 8; + IRM_BIND_PROGRAM = 9; + IRM_UNBIND_PROGRAM = 10; + IRM_PROC_ANNOUNCE = 11; + IRM_PROC_EXIT = 12; + IRM_BIND_PROCESS = 13; + IRM_UNBIND_PROCESS = 14; + IRM_CREATE_NAME = 15; + IRM_DESTROY_NAME = 16; + IRM_LIST_NAMES = 17; + IRM_REG_NAME = 18; + IRM_UNREG_NAME = 19; + IRM_FLOW_ALLOC = 20; + IRM_FLOW_ACCEPT = 21; + IRM_FLOW_JOIN = 22; + IRM_FLOW_DEALLOC = 23; + IPCP_FLOW_DEALLOC = 24; + IPCP_FLOW_REQ_ARR = 25; + IPCP_FLOW_ALLOC_REPLY = 26; + IRM_REPLY = 27; +} + +message timespec_msg { + required uint64 tv_sec = 1; + required uint32 tv_nsec = 2; +} + +message ipcp_list_msg { + required uint32 pid = 1; + required uint32 type = 2; + required string name = 3; + required string layer = 4; + required uint32 hash_algo = 5; +} + +message irm_msg { + required irm_msg_code code = 1; + optional string prog = 2; + optional sint32 pid = 3; + optional string name = 4; + optional flow_info_msg flow_info = 5; + optional ipcp_info_msg ipcp_info = 6; + optional name_info_msg name_info = 7; + optional string layer = 8; + repeated string exec = 9; + optional sint32 response = 10; + optional string dst = 11; + optional bytes hash = 12; + optional sint32 flow_id = 13; + optional qosspec_msg qosspec = 14; + optional ipcp_config_msg conf = 15; + optional uint32 opts = 16; + repeated ipcp_list_msg ipcps = 17; + repeated name_info_msg names = 18; + optional timespec_msg timeo = 19; + optional sint32 mpl = 20; /* MPL in ms. */ + optional string comp = 21; + optional bytes pk = 22; /* piggyback */ + optional uint32 timeo_sec = 23; + optional uint32 timeo_nsec = 24; + optional sint32 result = 25; + optional bytes sym_key = 26; /* symmetric encryption key */ + optional sint32 cipher_nid = 27; /* cipher NID */ +} diff --git a/src/lib/pb/model.proto b/src/lib/pb/model.proto new file mode 100644 index 00000000..4c1564a5 --- /dev/null +++ b/src/lib/pb/model.proto @@ -0,0 +1,66 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Model description messages + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +syntax = "proto2"; + +message qosspec_msg { + required uint32 delay = 1; /* In ms. */ + required uint64 bandwidth = 2; /* In bits/s. */ + required uint32 availability = 3; /* Class of 9s. */ + required uint32 loss = 4; /* Packet loss. */ + required uint32 ber = 5; /* Bit error rate, ppb. */ + required uint32 service = 6; /* enum qos_service. */ + required uint32 max_gap = 7; /* In ms. */ + required uint32 timeout = 8; /* Timeout in ms. */ +} + +message flow_info_msg { + required uint32 id = 1; + required uint32 n_pid = 2; + required uint32 n_1_pid = 3; + required uint32 mpl = 4; /* MPL in ms. */ + required uint32 state = 5; + required qosspec_msg qos = 6; + required uint32 uid = 7; + required uint32 mtu = 8; /* Layer MTU (bytes). */ +} + +message name_info_msg { + required string name = 1; + required uint32 pol_lb = 2; + required string skey = 3; + required string scrt = 4; + required string ckey = 5; + required string ccrt = 6; +} + +message layer_info_msg { + required string name = 1; + required uint32 dir_hash_algo = 2; +} + +message ipcp_info_msg { + required uint32 type = 1; + required string name = 2; + required uint32 pid = 3; + required uint32 state = 4; +} diff --git a/src/lib/protobuf.c b/src/lib/protobuf.c new file mode 100644 index 00000000..a824d357 --- /dev/null +++ b/src/lib/protobuf.c @@ -0,0 +1,925 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Protobuf syntax conversion + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _DEFAULT_SOURCE + +#include <ouroboros/protobuf.h> +#include <ouroboros/crypt.h> +#include <ouroboros/proc.h> + +#include <stdlib.h> +#include <string.h> +#include <time.h> + +timespec_msg_t * timespec_s_to_msg(const struct timespec * s) +{ + timespec_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + timespec_msg__init(msg); + + msg->tv_sec = s->tv_sec; + msg->tv_nsec = s->tv_nsec; + + return msg; + + fail_malloc: + return NULL; +} + +struct timespec timespec_msg_to_s(timespec_msg_t * msg) +{ + struct timespec s; + + assert(msg != NULL); + + s.tv_sec = msg->tv_sec; + s.tv_nsec = msg->tv_nsec; + + return s; +} + +flow_info_msg_t * flow_info_s_to_msg(const struct flow_info * s) +{ + flow_info_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + flow_info_msg__init(msg); + + msg->id = s->id; + msg->n_pid = s->n_pid; + msg->n_1_pid = s->n_1_pid; + msg->mpl = s->mpl; + msg->state = s->state; + msg->uid = s->uid; + msg->mtu = s->mtu; + msg->qos = qos_spec_s_to_msg(&s->qs); + if (msg->qos == NULL) + goto fail_msg; + + return msg; + + fail_msg: + flow_info_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct flow_info flow_info_msg_to_s(const flow_info_msg_t * msg) +{ + struct flow_info s; + + assert(msg != NULL); + + memset(&s, 0, sizeof(s)); + + s.id = msg->id; + s.n_pid = msg->n_pid; + s.n_1_pid = msg->n_1_pid; + s.mpl = msg->mpl; + s.state = msg->state; + s.uid = msg->uid; + s.mtu = msg->mtu; + s.qs = qos_spec_msg_to_s(msg->qos); + + return s; +} + +name_info_msg_t * name_info_s_to_msg(const struct name_info * info) +{ + name_info_msg_t * msg; + + assert(info != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + name_info_msg__init(msg); + + msg->name = strdup(info->name); + if (msg->name == NULL) + goto fail_msg; + + msg->skey = strdup(info->s.key); + if (msg->skey == NULL) + goto fail_msg; + + msg->scrt = strdup(info->s.crt); + if (msg->scrt == NULL) + goto fail_msg; + + msg->ckey = strdup(info->c.key); + if (msg->ckey == NULL) + goto fail_msg; + + msg->ccrt = strdup(info->c.crt); + if (msg->ccrt == NULL) + goto fail_msg; + + msg->pol_lb = info->pol_lb; + + return msg; + + fail_msg: + name_info_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct name_info name_info_msg_to_s(const name_info_msg_t * msg) +{ + struct name_info s; + + assert(msg != NULL); + assert(strlen(msg->name) <= NAME_SIZE); + + memset(&s, 0, sizeof(s)); + + strcpy(s.name, msg->name); + strcpy(s.s.key, msg->skey); + strcpy(s.s.crt, msg->scrt); + strcpy(s.c.key, msg->ckey); + strcpy(s.c.crt, msg->ccrt); + + s.pol_lb = msg->pol_lb; + + return s; +} + +layer_info_msg_t * layer_info_s_to_msg(const struct layer_info * s) +{ + layer_info_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + layer_info_msg__init(msg); + + msg->name = strdup(s->name); + if (msg->name == NULL) + goto fail_msg; + + msg->dir_hash_algo = s->dir_hash_algo; + + return msg; + + fail_msg: + layer_info_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct layer_info layer_info_msg_to_s(const layer_info_msg_t * msg) +{ + struct layer_info s; + + assert(msg != NULL); + assert(strlen(msg->name) <= LAYER_NAME_SIZE); + + s.dir_hash_algo = msg->dir_hash_algo; + strcpy(s.name, msg->name); + + return s; +} + +ipcp_info_msg_t * ipcp_info_s_to_msg(const struct ipcp_info * s) +{ + ipcp_info_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + ipcp_info_msg__init(msg); + + msg->name = strdup(s->name); + if (msg->name == NULL) + goto fail_msg; + + msg->type = s->type; + msg->pid = s->pid; + msg->state = s->state; + + return msg; + fail_msg: + ipcp_info_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct ipcp_info ipcp_info_msg_to_s(const ipcp_info_msg_t * msg) +{ + struct ipcp_info s; + + assert(msg != NULL); + assert(msg->name != NULL); + assert(strlen(msg->name) <= NAME_SIZE); + + strcpy(s.name, msg->name); + s.type = msg->type; + s.pid = msg->pid; + s.state = msg->state; + + return s; +} + +ls_config_msg_t * ls_config_s_to_msg(const struct ls_config * s) +{ + ls_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + ls_config_msg__init(msg); + + msg->pol = s->pol; + msg->t_recalc = s->t_recalc; + msg->t_update = s->t_update; + msg->t_timeo = s->t_timeo; + + return msg; + + fail_malloc: + return NULL; +} + +struct ls_config ls_config_msg_to_s(const ls_config_msg_t * msg) +{ + struct ls_config s; + + assert(msg != NULL); + + s.pol = msg->pol; + s.t_recalc = msg->t_recalc; + s.t_update = msg->t_update; + s.t_timeo = msg->t_timeo; + + return s; +} + +routing_config_msg_t * routing_config_s_to_msg(const struct routing_config * s) +{ + routing_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + return NULL; + + routing_config_msg__init(msg); + + switch (s->pol) { + case ROUTING_LINK_STATE: + msg->ls = ls_config_s_to_msg(&s->ls); + if (msg->ls == NULL) + goto fail_ls; + break; + default: + /* No checks here */ + break; + } + + msg->pol = s->pol; + + return msg; + + fail_ls: + routing_config_msg__free_unpacked(msg, NULL); + return NULL; +} + +struct routing_config routing_config_msg_to_s(const routing_config_msg_t * msg) +{ + struct routing_config s; + + assert(msg != NULL); + + switch (msg->pol) { + case ROUTING_LINK_STATE: + s.ls = ls_config_msg_to_s(msg->ls); + break; + default: + /* No checks here */ + break; + } + + s.pol = msg->pol; + + return s; +} + +dt_config_msg_t * dt_config_s_to_msg(const struct dt_config * s) +{ + dt_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + return NULL; + + dt_config_msg__init(msg); + + msg->addr_size = s->addr_size; + msg->eid_size = s->eid_size; + msg->max_ttl = s->max_ttl; + msg->routing = routing_config_s_to_msg(&s->routing); + if (msg->routing == NULL) + goto fail_routing; + + return msg; + fail_routing: + dt_config_msg__free_unpacked(msg, NULL); + return NULL; +} + +struct dt_config dt_config_msg_to_s(const dt_config_msg_t * msg) +{ + struct dt_config s; + + assert(msg != NULL); + + s.addr_size = msg->addr_size; + s.eid_size = msg->eid_size; + s.max_ttl = msg->max_ttl; + s.routing = routing_config_msg_to_s(msg->routing); + + return s; +} + +struct dir_dht_config dir_dht_config_msg_to_s(const dir_dht_config_msg_t * msg) +{ + struct dir_dht_config s; + + assert(msg != NULL); + + s.params.alpha = msg->alpha; + s.params.k = msg->k; + s.params.t_expire = msg->t_expire; + s.params.t_refresh = msg->t_refresh; + s.params.t_replicate = msg->t_replicate; + s.peer = msg->peer; + + return s; +} + +dir_dht_config_msg_t * dir_dht_config_s_to_msg(const struct dir_dht_config * s) +{ + dir_dht_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + return NULL; + + dir_dht_config_msg__init(msg); + + msg->alpha = s->params.alpha; + msg->k = s->params.k; + msg->t_expire = s->params.t_expire; + msg->t_refresh = s->params.t_refresh; + msg->t_replicate = s->params.t_replicate; + msg->peer = s->peer; + + return msg; +} + +struct dir_config dir_config_msg_to_s(const dir_config_msg_t * msg) +{ + struct dir_config s; + + assert(msg != NULL); + + switch (msg->pol) { + case DIR_DHT: + s.dht = dir_dht_config_msg_to_s(msg->dht); + break; + default: + /* No checks here */ + break; + } + + s.pol = msg->pol; + + return s; +} + +dir_config_msg_t * dir_config_s_to_msg(const struct dir_config * s) +{ + dir_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + return NULL; + + dir_config_msg__init(msg); + + switch (s->pol) { + case DIR_DHT: + msg->dht = dir_dht_config_s_to_msg(&s->dht); + if (msg->dht == NULL) + goto fail_msg; + break; + default: + /* No checks here */ + break; + } + + msg->pol = s->pol; + + return msg; + + fail_msg: + dir_config_msg__free_unpacked(msg, NULL); + return NULL; +} + + +uni_config_msg_t * uni_config_s_to_msg(const struct uni_config * s) +{ + uni_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + uni_config_msg__init(msg); + + msg->dt = dt_config_s_to_msg(&s->dt); + if (msg->dt == NULL) + goto fail_msg; + + msg->dir = dir_config_s_to_msg(&s->dir); + if (msg->dir == NULL) + goto fail_msg; + + + msg->addr_auth_type = s->addr_auth_type; + msg->cong_avoid = s->cong_avoid; + + return msg; + + fail_msg: + uni_config_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct uni_config uni_config_msg_to_s(const uni_config_msg_t * msg) +{ + struct uni_config s; + + s.dt = dt_config_msg_to_s(msg->dt); + s.dir = dir_config_msg_to_s(msg->dir); + + s.addr_auth_type = msg->addr_auth_type; + s.cong_avoid = msg->cong_avoid; + + return s; +} + +udp4_config_msg_t * udp4_config_s_to_msg(const struct udp4_config * s) +{ + udp4_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + return NULL; + + udp4_config_msg__init(msg); + + msg->ip_addr = s->ip_addr.s_addr; + msg->dns_addr = s->dns_addr.s_addr; + msg->port = s->port; + + return msg; +} + +struct udp4_config udp4_config_msg_to_s(const udp4_config_msg_t * msg) +{ + struct udp4_config s; + + assert(msg != NULL); + + s.ip_addr.s_addr = msg->ip_addr; + s.dns_addr.s_addr = msg->dns_addr; + s.port = msg->port; + + return s; +} + +#define IN6_LEN (size_t) sizeof(struct in6_addr) +udp6_config_msg_t * udp6_config_s_to_msg(const struct udp6_config * s) +{ + udp6_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + udp6_config_msg__init(msg); + + msg->ip_addr.data = malloc(IN6_LEN); + if (msg->ip_addr.data == NULL) + goto fail_msg; + + msg->ip_addr.len = IN6_LEN; + memcpy(msg->ip_addr.data, &s->ip_addr.s6_addr, IN6_LEN); + + msg->dns_addr.data = malloc(IN6_LEN); + if (msg->dns_addr.data == NULL) + goto fail_msg; + + msg->dns_addr.len = IN6_LEN; + memcpy(msg->dns_addr.data, &s->dns_addr.s6_addr, IN6_LEN); + + msg->port = s->port; + + return msg; + + fail_msg: + udp6_config_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct udp6_config udp6_config_msg_to_s(const udp6_config_msg_t * msg) +{ + struct udp6_config s; + + assert(msg != NULL); + + assert(msg->ip_addr.len == IN6_LEN); + assert(msg->dns_addr.len == IN6_LEN); + + memcpy(&s.ip_addr.s6_addr, msg->ip_addr.data, IN6_LEN); + memcpy(&s.dns_addr.s6_addr, msg->dns_addr.data, IN6_LEN); + s.port = msg->port; + + return s; +} + +eth_config_msg_t * eth_config_s_to_msg(const struct eth_config * s) +{ + eth_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + eth_config_msg__init(msg); + + msg->dev = strdup(s->dev); + if (msg->dev == NULL) + goto fail_msg; + + msg->ethertype = s->ethertype; + + return msg; + + fail_msg: + eth_config_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct eth_config eth_config_msg_to_s(const eth_config_msg_t * msg) +{ + struct eth_config s; + + assert(msg != NULL); + assert(strlen(msg->dev) <= DEV_NAME_SIZE); + + strcpy(s.dev, msg->dev); + s.ethertype = msg->ethertype; + + return s; +} + + +ipcp_config_msg_t * ipcp_config_s_to_msg(const struct ipcp_config * s) +{ + ipcp_config_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + ipcp_config_msg__init(msg); + + switch (s->type) { + case IPCP_LOCAL: + break; + case IPCP_UNICAST: + msg->unicast = uni_config_s_to_msg(&s->unicast); + if (msg->unicast == NULL) + goto fail_msg; + break; + case IPCP_BROADCAST: + break; + case IPCP_ETH_LLC: + /* FALLTHRU */ + case IPCP_ETH_DIX: + msg->eth = eth_config_s_to_msg(&s->eth); + if (msg->eth == NULL) + goto fail_msg; + break; + case IPCP_UDP4: + msg->udp4 = udp4_config_s_to_msg(&s->udp4); + if (msg->udp4 == NULL) + goto fail_msg; + break; + case IPCP_UDP6: + msg->udp6 = udp6_config_s_to_msg(&s->udp6); + if (msg->udp6 == NULL) + goto fail_msg; + break; + default: + /* No checks here */ + break; + } + + msg->ipcp_type = s->type; + + msg->layer_info = layer_info_s_to_msg(&s->layer_info); + if (msg->layer_info == NULL) + goto fail_msg; + + return msg; + + fail_msg: + ipcp_config_msg__free_unpacked(msg, NULL); + fail_malloc: + return NULL; +} + +struct ipcp_config ipcp_config_msg_to_s(const ipcp_config_msg_t * msg) +{ + struct ipcp_config s; + + assert(msg != NULL); + + s.type = msg->ipcp_type; + + s.layer_info = layer_info_msg_to_s(msg->layer_info); + + switch(msg->ipcp_type) { + case IPCP_LOCAL: + break; + case IPCP_UNICAST: + s.unicast = uni_config_msg_to_s(msg->unicast); + break; + case IPCP_ETH_LLC: + /* FALLTHRU */ + case IPCP_ETH_DIX: + s.eth = eth_config_msg_to_s(msg->eth); + break; + case IPCP_UDP4: + s.udp4 = udp4_config_msg_to_s(msg->udp4); + break; + case IPCP_UDP6: + s.udp6 = udp6_config_msg_to_s(msg->udp6); + break; + case IPCP_BROADCAST: + break; + default: + /* No checks here */ + break; + } + + return s; +} + +qosspec_msg_t * qos_spec_s_to_msg(const struct qos_spec * s) +{ + qosspec_msg_t * msg; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + return NULL; + + qosspec_msg__init(msg); + + msg->delay = s->delay; + msg->bandwidth = s->bandwidth; + msg->availability = s->availability; + msg->loss = s->loss; + msg->ber = s->ber; + msg->service = s->service; + msg->max_gap = s->max_gap; + msg->timeout = s->timeout; + + return msg; +} + +struct qos_spec qos_spec_msg_to_s(const qosspec_msg_t * msg) +{ + struct qos_spec s; + + assert(msg != NULL); + + s.delay = msg->delay; + s.bandwidth = msg->bandwidth; + s.availability = msg->availability; + s.loss = msg->loss; + s.ber = msg->ber; + s.service = msg->service; + s.max_gap = msg->max_gap; + s.timeout = msg->timeout; + + return s; +} + +enroll_req_msg_t * enroll_req_s_to_msg(const struct enroll_req * s) +{ + enroll_req_msg_t * msg; + uint8_t * id; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_msg; + + id = malloc(ENROLL_ID_LEN); + if (id == NULL) + goto fail_id; + + memcpy(id, s->id, ENROLL_ID_LEN); + + enroll_req_msg__init(msg); + + msg->id.len = ENROLL_ID_LEN; + msg->id.data = id; + + return msg; + + fail_id: + free(msg); + fail_msg: + return NULL; +} + +struct enroll_req enroll_req_msg_to_s(const enroll_req_msg_t * msg) +{ + struct enroll_req s; + + assert(msg != NULL); + + memcpy(s.id, msg->id.data, ENROLL_ID_LEN); + + return s; +} + +enroll_resp_msg_t * enroll_resp_s_to_msg(const struct enroll_resp * s) +{ + enroll_resp_msg_t * msg; + uint8_t * id; + + assert(s != NULL); + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_msg; + + id = malloc(ENROLL_ID_LEN); + if (id == NULL) + goto fail_id; + + memcpy(id, s->id, ENROLL_ID_LEN); + + enroll_resp_msg__init(msg); + + msg->id.len = ENROLL_ID_LEN; + msg->id.data = id; + + msg->t_sec = s->t.tv_sec; + msg->t_nsec = s->t.tv_nsec; + msg->response = s->response; + if (msg->response < 0) + return msg; + + msg->conf = ipcp_config_s_to_msg(&s->conf); + if (msg->conf == NULL) + goto fail_id; + + return msg; + + fail_id: + enroll_resp_msg__free_unpacked(msg, NULL); + fail_msg: + return NULL; +} + +struct enroll_resp enroll_resp_msg_to_s(const enroll_resp_msg_t * msg) +{ + struct enroll_resp s; + + assert (msg != NULL); + + s.response = msg->response; + if (s.response >= 0) + s.conf = ipcp_config_msg_to_s(msg->conf); + + s.t.tv_sec = msg->t_sec; + s.t.tv_nsec = msg->t_nsec; + + memcpy(s.id, msg->id.data, ENROLL_ID_LEN); + + return s; +} + +enroll_ack_msg_t * enroll_ack_s_to_msg(const struct enroll_ack * s) +{ + enroll_ack_msg_t * msg; + uint8_t * id; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_msg; + + id = malloc(ENROLL_ID_LEN); + if (id == NULL) + goto fail_id; + + memcpy(id, s->id, ENROLL_ID_LEN); + + enroll_ack_msg__init(msg); + + msg->id.len = ENROLL_ID_LEN; + msg->id.data = id; + + msg->result = s->result; + + return msg; + + fail_id: + enroll_ack_msg__free_unpacked(msg, NULL); + fail_msg: + return NULL; +} + +struct enroll_ack enroll_ack_msg_to_s(const enroll_ack_msg_t * msg) +{ + struct enroll_ack s; + + assert(msg != NULL); + + memcpy(s.id, msg->id.data, ENROLL_ID_LEN); + + s.result = msg->result; + + return s; +} diff --git a/src/lib/qoscube.c b/src/lib/qoscube.c index 3e6f884d..5d7ae17d 100644 --- a/src/lib/qoscube.c +++ b/src/lib/qoscube.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Quality of Service cube * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -29,15 +29,11 @@ qoscube_t qos_spec_to_cube(qosspec_t qs) { - if (qs.delay <= qos_voice.delay && - qs.bandwidth <= qos_voice.bandwidth && - qs.availability >= qos_voice.availability && - qs.max_gap <= qos_voice.max_gap) + if (qs.delay <= 50 && qs.bandwidth <= 100000 + && qs.availability >= 5 && qs.max_gap <= 50) return QOS_CUBE_VOICE; - else if (qs.delay <= qos_video.delay && - qs.bandwidth <= qos_video.bandwidth && - qs.availability >= qos_video.availability && - qs.max_gap <= qos_video.max_gap) + else if (qs.delay <= 100 && qs.availability >= 3 + && qs.max_gap <= 100) return QOS_CUBE_VIDEO; else return QOS_CUBE_BE; diff --git a/src/lib/qosspec.proto b/src/lib/qosspec.proto deleted file mode 100644 index 5d8abb79..00000000 --- a/src/lib/qosspec.proto +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * QoS specification message - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -syntax = "proto2"; - -message qosspec_msg { - required uint32 delay = 1; /* In ms */ - required uint64 bandwidth = 2; /* In bits/s */ - required uint32 availability = 3; /* Class of 9s */ - required uint32 loss = 4; /* Packet loss */ - required uint32 ber = 5; /* Bit error rate, ppb */ - required uint32 in_order = 6; /* In-order delivery */ - required uint32 max_gap = 7; /* In ms */ - required uint32 cypher_s = 8; /* Crypto strength in bits */ -}; diff --git a/src/lib/random.c b/src/lib/random.c index b2cdabbc..2c9a6c0d 100644 --- a/src/lib/random.c +++ b/src/lib/random.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Pseudo random generator * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -24,22 +24,12 @@ #include <ouroboros/random.h> -#if defined(__APPLE__) /* Barf */ -#undef __OSX_AVAILABLE -#define __OSX_AVAILABLE(arg) -#undef __IOS_AVAILABLE -#define __IOS_AVAILABLE(arg) -#undef __TVOS_AVAILABLE -#define __TVOS_AVAILABLE(arg) -#undef __WATCHOS_AVAILABLE -#define __WATCHOS_AVAILABLE(arg) -#include <sys/random.h> +#if defined(__APPLE__) || defined(__FreeBSD__) +#include <stdlib.h> #elif defined(HAVE_SYS_RANDOM) #include <sys/random.h> #elif defined(HAVE_LIBGCRYPT) #include <gcrypt.h> -#elif defined(__FreeBSD__) -#include <stdlib.h> #elif defined(HAVE_OPENSSL_RNG) #include <openssl/rand.h> #include <limits.h> @@ -48,19 +38,18 @@ int random_buffer(void * buf, size_t len) { -#if defined(__APPLE__) - return getentropy(buf, len); -#elif defined(__FreeBSD__) +#if defined(__APPLE__) || defined(__FreeBSD__) arc4random_buf(buf, len); return 0; #elif defined(HAVE_SYS_RANDOM) - return getrandom(buf, len, GRND_NONBLOCK); /* glibc 2.25 */ + return getrandom(buf, len, GRND_NONBLOCK); #elif defined(HAVE_LIBGCRYPT) gcry_randomize(buf, len, GCRY_STRONG_RANDOM); return 0; #elif defined(HAVE_OPENSSL_RNG) - if (len > 0 && len < INT_MAX) - return RAND_bytes((unsigned char *) buf, (int) len); - return -1; + if (len == 0 || len >= INT_MAX) + return -1; + + return RAND_bytes((unsigned char *) buf, (int) len) == 1 ? 0 : -1; #endif } diff --git a/src/lib/rib.c b/src/lib/rib.c index 684c5dcd..6e421397 100644 --- a/src/lib/rib.c +++ b/src/lib/rib.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * RIB export using FUSE * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -43,15 +43,14 @@ #define __USE_XOPEN #elif defined (__FreeBSD__) #define __XSI_VISIBLE 500 -#endif +#endif /* __linux__ */ +#include <sys/stat.h> #include <fuse.h> #ifndef CLOCK_REALTIME_COARSE #define CLOCK_REALTIME_COARSE CLOCK_REALTIME #endif -#define RT "/" - struct reg_comp { struct list_head next; @@ -106,22 +105,21 @@ static int rib_read(const char * path, strcpy(comp, path + 1); - c = strstr(comp, "/"); - + c = strstr(comp, RIB_SEPARATOR); if (c != NULL) *c = '\0'; (void) info; (void) offset; - pthread_rwlock_wrlock(&rib.lock); + pthread_rwlock_rdlock(&rib.lock); list_for_each(p, &rib.reg_comps) { struct reg_comp * r = list_entry(p, struct reg_comp, next); if (strcmp(comp, r->path) == 0) { - int ret = r->ops->read(c + 1, buf, size); + struct rib_ops * ops = r->ops; pthread_rwlock_unlock(&rib.lock); - return ret; + return ops->read(path + 1, buf, size); } } @@ -141,12 +139,16 @@ static int rib_readdir(const char * path, (void) offset; (void) info; + /* Fix ls calling readdir in an infinite loop on raspbian. */ + if (info != NULL && info->nonseekable != 0) + return -ENOENT; + filler(buf, ".", NULL, 0); filler(buf, "..", NULL, 0); pthread_rwlock_rdlock(&rib.lock); - if (strcmp(path, RT) == 0) { + if (strcmp(path, RIB_SEPARATOR) == 0) { list_for_each(p, &rib.reg_comps) { struct reg_comp * c; c = list_entry(p, struct reg_comp, next); @@ -158,19 +160,25 @@ static int rib_readdir(const char * path, ssize_t len; ssize_t i; struct reg_comp * c; + struct rib_ops * ops; c = list_entry(p, struct reg_comp, next); if (strcmp(path + 1, c->path) != 0) continue; - assert(c->ops->readdir != NULL); + ops = c->ops; - len = c->ops->readdir(&dir_entries); + assert(ops->readdir != NULL); + + pthread_rwlock_unlock(&rib.lock); + + len = ops->readdir(&dir_entries); if (len < 0) - break; + return 0; for (i = 0; i < len; ++i) filler(buf, dir_entries[i], NULL, 0); freepp(char, dir_entries, len); + return 0; } } @@ -185,24 +193,35 @@ static size_t __getattr(const char * path, struct list_head * p; char comp[RIB_PATH_LEN + 1]; char * c; + struct rib_attr attr; if (strlen(path) > RIB_PATH_LEN) return -1; strcpy(comp, path + 1); - c = strstr(comp, "/"); - + c = strstr(comp, RIB_SEPARATOR); if (c != NULL) *c = '\0'; + memset(&attr, 0, sizeof(attr)); + pthread_rwlock_rdlock(&rib.lock); list_for_each(p, &rib.reg_comps) { struct reg_comp * r = list_entry(p, struct reg_comp, next); if (strcmp(comp, r->path) == 0) { - size_t ret = r->ops->getattr(c + 1, st); + size_t ret = r->ops->getattr(path + 1, &attr); pthread_rwlock_unlock(&rib.lock); + st->st_mode = S_IFREG | 0644; + st->st_blocks = 1; + st->st_nlink = 1; + st->st_uid = getuid(); + st->st_gid = getgid(); + st->st_size = attr.size; + st->st_atime = attr.mtime; + st->st_mtime = attr.mtime; + st->st_ctime = attr.mtime; return ret; } } @@ -220,7 +239,7 @@ static int rib_getattr(const char * path, memset(st, 0, sizeof(*st)); - if (strcmp(path, RT) == 0) + if (strcmp(path, RIB_SEPARATOR) == 0) goto finish_dir; pthread_rwlock_rdlock(&rib.lock); @@ -246,6 +265,8 @@ static int rib_getattr(const char * path, st->st_uid = getuid(); st->st_gid = getgid(); st->st_mtime = now.tv_sec; + st->st_atime = now.tv_sec; + st->st_ctime = now.tv_sec; return 0; } @@ -266,7 +287,6 @@ static void * fuse_thr(void * o) } #endif /* HAVE_FUSE */ - int rib_init(const char * mountpt) { #ifdef HAVE_FUSE @@ -280,8 +300,11 @@ int rib_init(const char * mountpt) NULL}; struct fuse_args args = FUSE_ARGS_INIT(3, argv); + if (access("/dev/fuse", R_OK)) + goto fail; + if (stat(FUSE_PREFIX, &st) == -1) - return -1; + goto fail; sprintf(rib.mnt, FUSE_PREFIX "/%s", mountpt); @@ -289,13 +312,13 @@ int rib_init(const char * mountpt) switch(errno) { case ENOENT: if (mkdir(rib.mnt, 0777)) - return -1; + goto fail_mnt; break; case ENOTCONN: fuse_unmount(rib.mnt, rib.ch); break; default: - return -1; + goto fail_mnt; } fuse_opt_parse(&args, NULL, NULL, NULL); @@ -329,6 +352,9 @@ int rib_init(const char * mountpt) fail_mount: fuse_opt_free_args(&args); rmdir(rib.mnt); + fail_mnt: + memset(rib.mnt, 0, sizeof(rib.mnt)); + fail: return -1; #else (void) mountpt; @@ -342,6 +368,9 @@ void rib_fini(void) struct list_head * p; struct list_head * h; + if (strlen(rib.mnt) == 0) + return; + fuse_exit(rib.fuse); fuse_unmount(rib.mnt, rib.ch); @@ -363,6 +392,18 @@ void rib_fini(void) pthread_rwlock_unlock(&rib.lock); pthread_rwlock_destroy(&rib.lock); + + memset(rib.mnt, 0, sizeof(rib.mnt)); +#endif +} + +void rib_cleanup(const char * mnt) +{ +#ifdef HAVE_FUSE + fuse_unmount(mnt, NULL); + rmdir(mnt); +#else + (void) mnt; #endif } @@ -373,6 +414,9 @@ int rib_reg(const char * path, struct reg_comp * rc; struct list_head * p; + if (strlen(rib.mnt) == 0) + return 0; + pthread_rwlock_wrlock(&rib.lock); list_for_each(p, &rib.reg_comps) { @@ -417,6 +461,9 @@ void rib_unreg(const char * path) struct list_head * p; struct list_head * h; + if (strlen(rib.mnt) == 0) + return; + pthread_rwlock_wrlock(&rib.lock); list_for_each_safe(p, h, &rib.reg_comps) { diff --git a/src/lib/serdes-irm.c b/src/lib/serdes-irm.c new file mode 100644 index 00000000..65f2c02d --- /dev/null +++ b/src/lib/serdes-irm.c @@ -0,0 +1,473 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Ouroboros IRM Protocol - serialization/deserialization + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include "config.h" + +#include <ouroboros/crypt.h> +#include <ouroboros/errno.h> +#include <ouroboros/serdes-irm.h> +#include <ouroboros/protobuf.h> + +#include <stdlib.h> +#include <string.h> + +int flow_accept__irm_req_ser(buffer_t * buf, + const struct flow_info * flow, + const struct timespec * timeo) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IRM_FLOW_ACCEPT; + msg->flow_info = flow_info_s_to_msg(flow); + if (msg->flow_info == NULL) + goto fail_msg; + + msg->timeo = timeo == NULL ? NULL : timespec_s_to_msg(timeo); + if (timeo != NULL && msg->timeo == NULL) + goto fail_msg; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + irm_msg__free_unpacked(msg, NULL); + + return 0; + + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + +static int __flow_alloc_ser(buffer_t * buf, + const struct flow_info * flow, + const char * dst, + const struct timespec * timeo, + int msg_code) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = msg_code; + msg->flow_info = flow_info_s_to_msg(flow); + if (msg->flow_info == NULL) + goto fail_msg; + + msg->dst = strdup(dst); + if (msg->dst == NULL) + goto fail_msg; + + msg->timeo = timeo == NULL ? NULL : timespec_s_to_msg(timeo); + if (timeo != NULL && msg->timeo == NULL) + goto fail_msg; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + irm_msg__free_unpacked(msg, NULL); + + return 0; + + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + +int flow_alloc__irm_req_ser(buffer_t * buf, + const struct flow_info * flow, + const char * dst, + const struct timespec * timeo) +{ + return __flow_alloc_ser(buf, flow, dst, timeo, + IRM_MSG_CODE__IRM_FLOW_ALLOC); +} + +int flow_join__irm_req_ser(buffer_t * buf, + const struct flow_info * flow, + const char * dst, + const struct timespec * timeo) +{ + return __flow_alloc_ser(buf, flow, dst, timeo, + IRM_MSG_CODE__IRM_FLOW_JOIN); +} + +int flow__irm_result_des(buffer_t * buf, + struct flow_info * flow, + struct crypt_sk * sk) +{ + irm_msg_t * msg; + int err; + + msg = irm_msg__unpack(NULL, buf->len, buf->data); + if (msg == NULL) { + err = -EIRMD; + goto fail_msg; + } + + if (!msg->has_result) { + err = -EIRMD; + goto fail; + } + + if (msg->result < 0) { + err = msg->result; + goto fail; + } + + if (msg->flow_info == NULL) { + err = -EBADF; + goto fail; + } + + *flow = flow_info_msg_to_s(msg->flow_info); + + if (msg->has_cipher_nid) + sk->nid = msg->cipher_nid; + else + sk->nid = NID_undef; + + if (msg->sym_key.len == SYMMKEYSZ) + memcpy(sk->key, msg->sym_key.data, SYMMKEYSZ); + else + memset(sk->key, 0, SYMMKEYSZ); + + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail: + irm_msg__free_unpacked(msg, NULL); + fail_msg: + return err; +} + +int flow_dealloc__irm_req_ser(buffer_t * buf, + const struct flow_info * flow, + const struct timespec * timeo) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IRM_FLOW_DEALLOC; + msg->flow_info = flow_info_s_to_msg(flow); + if (msg->flow_info == NULL) + goto fail_msg; + + msg->timeo = timespec_s_to_msg(timeo); + if (msg->timeo == NULL) + goto fail_msg; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + irm_msg__free_unpacked(msg, NULL); + + return 0; + + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + +int ipcp_flow_dealloc__irm_req_ser(buffer_t * buf, + const struct flow_info * flow) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IPCP_FLOW_DEALLOC; + msg->flow_info = flow_info_s_to_msg(flow); + if (msg->flow_info == NULL) + goto fail_msg; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + + +int ipcp_create_r__irm_req_ser(buffer_t * buf, + const struct ipcp_info * ipcp) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IPCP_CREATE_R; + msg->ipcp_info = ipcp_info_s_to_msg(ipcp); + if (msg->ipcp_info == NULL) + goto fail_msg; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + +int proc_announce__irm_req_ser(buffer_t * buf, + const struct proc_info * proc) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IRM_PROC_ANNOUNCE; + msg->has_pid = true; + msg->pid = proc->pid; + msg->prog = strdup(proc->prog); + if (msg->prog == NULL) + goto fail_msg; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + +int proc_exit__irm_req_ser(buffer_t * buf) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IRM_PROC_EXIT; + msg->has_pid = true; + msg->pid = getpid(); + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + +int ipcp_flow_req_arr__irm_req_ser(buffer_t * buf, + const buffer_t * dst, + const struct flow_info * flow, + const buffer_t * data) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IPCP_FLOW_REQ_ARR; + msg->flow_info = flow_info_s_to_msg(flow); + if (msg->flow_info == NULL) + goto fail_msg; + + msg->has_hash = true; + msg->hash.len = dst->len; + msg->hash.data = dst->data; + msg->has_pk = true; + msg->pk.len = data->len; + msg->pk.data = data->data; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + + /* Don't free * dst or data! */ + msg->hash.len = 0; + msg->hash.data = NULL; + msg->pk.len = 0; + msg->pk.data = NULL; + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + +int ipcp_flow_alloc_reply__irm_msg_ser(buffer_t * buf, + const struct flow_info * flow, + int response, + const buffer_t * data) +{ + irm_msg_t * msg; + size_t len; + + msg = malloc(sizeof(*msg)); + if (msg == NULL) + goto fail_malloc; + + irm_msg__init(msg); + + msg->code = IRM_MSG_CODE__IPCP_FLOW_ALLOC_REPLY; + msg->flow_info = flow_info_s_to_msg(flow); + if (msg->flow_info == NULL) + goto fail_msg; + + msg->has_pk = true; + msg->pk.len = data->len; + msg->pk.data = data->data; + msg->has_response = true; + msg->response = response; + + len = irm_msg__get_packed_size(msg); + if (len == 0 || len > buf->len) + goto fail_msg; + + buf->len = len; + + irm_msg__pack(msg, buf->data); + + /* Don't free * data! */ + msg->pk.len = 0; + msg->pk.data = NULL; + + irm_msg__free_unpacked(msg, NULL); + + return 0; + fail_msg: + irm_msg__free_unpacked(msg, NULL); + fail_malloc: + return -ENOMEM; +} + +int irm__irm_result_des(buffer_t * buf) +{ + irm_msg_t * msg; + int err; + + msg = irm_msg__unpack(NULL, buf->len, buf->data); + if (msg == NULL) { + err = -EIRMD; + goto fail_msg; + } + + if (!msg->has_result) { + err = -EIRMD; + goto fail; + } + + err = msg->result; + fail: + irm_msg__free_unpacked(msg, NULL); + fail_msg: + return err; +} diff --git a/src/lib/serdes-oep.c b/src/lib/serdes-oep.c new file mode 100644 index 00000000..3d191494 --- /dev/null +++ b/src/lib/serdes-oep.c @@ -0,0 +1,161 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Ouroboros Enrollment Protocol - serialization/deserialization + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200112L + +#include <ouroboros/protobuf.h> +#include <ouroboros/serdes-oep.h> + +ssize_t enroll_req_ser(const struct enroll_req * req, + buffer_t buf) +{ + enroll_req_msg_t * msg; + ssize_t sz; + + msg = enroll_req_s_to_msg(req); + if (msg == NULL) + goto fail_msg; + + sz = enroll_req_msg__get_packed_size(msg); + if (sz < 0 || (size_t) sz > buf.len) + goto fail_pack; + + enroll_req_msg__pack(msg, buf.data); + + enroll_req_msg__free_unpacked(msg, NULL); + + return sz; + + fail_pack: + enroll_req_msg__free_unpacked(msg, NULL); + fail_msg: + return -1; +} + +int enroll_req_des(struct enroll_req * req, + const buffer_t buf) +{ + enroll_req_msg_t * msg; + + msg = enroll_req_msg__unpack(NULL, buf.len, buf.data); + if (msg == NULL) + goto fail_unpack; + + if (msg->id.len != ENROLL_ID_LEN) + goto fail_id; + + *req = enroll_req_msg_to_s(msg); + + enroll_req_msg__free_unpacked(msg, NULL); + + return 0; + + fail_id: + enroll_req_msg__free_unpacked(msg, NULL); + fail_unpack: + return -1; +} + +ssize_t enroll_resp_ser(const struct enroll_resp * resp, + buffer_t buf) +{ + enroll_resp_msg_t * msg; + ssize_t sz; + + msg = enroll_resp_s_to_msg(resp); + if (msg == NULL) + goto fail_msg; + + sz = enroll_resp_msg__get_packed_size(msg); + if (sz < 0 || (size_t) sz > buf.len) + goto fail_pack; + + enroll_resp_msg__pack(msg, buf.data); + + enroll_resp_msg__free_unpacked(msg, NULL); + + return sz; + + fail_pack: + enroll_resp_msg__free_unpacked(msg, NULL); + fail_msg: + return -1; +} + +int enroll_resp_des(struct enroll_resp * resp, + const buffer_t buf) +{ + enroll_resp_msg_t * msg; + + msg = enroll_resp_msg__unpack(NULL, buf.len, buf.data); + if (msg == NULL) + return -1; + + *resp = enroll_resp_msg_to_s(msg); + + enroll_resp_msg__free_unpacked(msg, NULL); + + return 0; +} + +ssize_t enroll_ack_ser(const struct enroll_ack * ack, + buffer_t buf) +{ + enroll_ack_msg_t * msg; + ssize_t sz; + + msg = enroll_ack_s_to_msg(ack); + if (msg == NULL) + goto fail_msg; + + sz = enroll_ack_msg__get_packed_size(msg); + if (sz < 0 || (size_t) sz > buf.len) + goto fail_pack; + + enroll_ack_msg__pack(msg, buf.data); + + enroll_ack_msg__free_unpacked(msg, NULL); + + return sz; + + fail_pack: + enroll_ack_msg__free_unpacked(msg, NULL); + fail_msg: + return -1; + +} + +int enroll_ack_des(struct enroll_ack * ack, + const buffer_t buf) +{ + enroll_ack_msg_t * msg; + + msg = enroll_ack_msg__unpack(NULL, buf.len, buf.data); + if (msg == NULL) + return -1; + + *ack = enroll_ack_msg_to_s(msg); + + enroll_ack_msg__free_unpacked(msg, NULL); + + return 0; +} diff --git a/src/lib/sha3.c b/src/lib/sha3.c index 7bdafcb5..f406124e 100644 --- a/src/lib/sha3.c +++ b/src/lib/sha3.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * SHA3 algorithm * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This implementation is adapted and redistributed from the RHASH * project @@ -52,8 +52,7 @@ #include <assert.h> #include <string.h> -#define IS_ALIGNED_64(p) (0 == (7 & ((const uint8_t *) (p) \ - - (const uint8_t *) 0))) +#define IS_ALIGNED_64(p) (0 == (7 & ((uintptr_t) (p)))) #define I64(x) x##LL #define ROTL64(qword, n) ((qword) << (n) ^ ((qword) >> (64 - (n)))) diff --git a/src/lib/shm_rbuff.c b/src/lib/shm_rbuff.c deleted file mode 100644 index 0a07e799..00000000 --- a/src/lib/shm_rbuff.c +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * Ring buffer implementations for incoming packets - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -#define _POSIX_C_SOURCE 200809L - -#include "config.h" - -#include <ouroboros/shm_rbuff.h> -#include <ouroboros/lockfile.h> -#include <ouroboros/time_utils.h> -#include <ouroboros/errno.h> -#include <ouroboros/fccntl.h> - -#include <pthread.h> -#include <sys/mman.h> -#include <fcntl.h> -#include <stdlib.h> -#include <string.h> -#include <stdio.h> -#include <stdint.h> -#include <unistd.h> -#include <signal.h> -#include <sys/stat.h> -#include <assert.h> -#include <stdbool.h> - -#define FN_MAX_CHARS 255 - -#define SHM_RB_FILE_SIZE ((SHM_RBUFF_SIZE) * sizeof(ssize_t) \ - + 3 * sizeof(size_t) \ - + sizeof(pthread_mutex_t) \ - + 2 * sizeof (pthread_cond_t)) - -#define shm_rbuff_used(rb) ((*rb->head + (SHM_RBUFF_SIZE) - *rb->tail) \ - & ((SHM_RBUFF_SIZE) - 1)) -#define shm_rbuff_free(rb) (shm_rbuff_used(rb) + 1 < (SHM_RBUFF_SIZE)) -#define shm_rbuff_empty(rb) (*rb->head == *rb->tail) -#define head_el_ptr(rb) (rb->shm_base + *rb->head) -#define tail_el_ptr(rb) (rb->shm_base + *rb->tail) - -struct shm_rbuff { - ssize_t * shm_base; /* start of entry */ - size_t * head; /* start of ringbuffer head */ - size_t * tail; /* start of ringbuffer tail */ - size_t * acl; /* access control */ - pthread_mutex_t * lock; /* lock all free space in shm */ - pthread_cond_t * add; /* packet arrived */ - pthread_cond_t * del; /* packet removed */ - pid_t pid; /* pid of the owner */ - int flow_id; /* flow_id of the flow */ -}; - -void shm_rbuff_close(struct shm_rbuff * rb) -{ - assert(rb); - - munmap(rb->shm_base, SHM_RB_FILE_SIZE); - - free(rb); -} - -#define MM_FLAGS (PROT_READ | PROT_WRITE) - -struct shm_rbuff * rbuff_create(pid_t pid, - int flow_id, - int flags) -{ - struct shm_rbuff * rb; - int fd; - ssize_t * shm_base; - char fn[FN_MAX_CHARS]; - - sprintf(fn, SHM_RBUFF_PREFIX "%d.%d", pid, flow_id); - - rb = malloc(sizeof(*rb)); - if (rb == NULL) - goto fail_malloc; - - fd = shm_open(fn, flags, 0666); - if (fd == -1) - goto fail_open; - - if ((flags & O_CREAT) && ftruncate(fd, SHM_RB_FILE_SIZE - 1) < 0) - goto fail_truncate; - - shm_base = mmap(NULL, SHM_RB_FILE_SIZE, MM_FLAGS, MAP_SHARED, fd, 0); - if (shm_base == MAP_FAILED) - goto fail_truncate; - - close(fd); - - rb->shm_base = shm_base; - rb->head = (size_t *) (rb->shm_base + (SHM_RBUFF_SIZE)); - rb->tail = rb->head + 1; - rb->acl = rb->tail + 1; - rb->lock = (pthread_mutex_t *) (rb->acl + 1); - rb->add = (pthread_cond_t *) (rb->lock + 1); - rb->del = rb->add + 1; - rb->pid = pid; - rb->flow_id = flow_id; - - return rb; - - fail_truncate: - close(fd); - if (flags & O_CREAT) - shm_unlink(fn); - fail_open: - free(rb); - fail_malloc: - return NULL; -} - -struct shm_rbuff * shm_rbuff_create(pid_t pid, - int flow_id) -{ - struct shm_rbuff * rb; - pthread_mutexattr_t mattr; - pthread_condattr_t cattr; - mode_t mask; - - mask = umask(0); - - rb = rbuff_create(pid, flow_id, O_CREAT | O_EXCL | O_RDWR); - - umask(mask); - - if (rb == NULL) - goto fail_rb; - - if (pthread_mutexattr_init(&mattr)) - goto fail_mattr; - - pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); -#ifdef HAVE_ROBUST_MUTEX - pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); -#endif - if (pthread_mutex_init(rb->lock, &mattr)) - goto fail_mutex; - - if (pthread_condattr_init(&cattr)) - goto fail_cattr; - - pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); -#ifndef __APPLE__ - pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); -#endif - if (pthread_cond_init(rb->add, &cattr)) - goto fail_add; - - if (pthread_cond_init(rb->del, &cattr)) - goto fail_del; - - *rb->acl = ACL_RDWR; - *rb->head = 0; - *rb->tail = 0; - - rb->pid = pid; - rb->flow_id = flow_id; - - pthread_mutexattr_destroy(&mattr); - pthread_condattr_destroy(&cattr); - - return rb; - - fail_del: - pthread_cond_destroy(rb->add); - fail_add: - pthread_condattr_destroy(&cattr); - fail_cattr: - pthread_mutex_destroy(rb->lock); - fail_mutex: - pthread_mutexattr_destroy(&mattr); - fail_mattr: - shm_rbuff_destroy(rb); - fail_rb: - return NULL; -} - -struct shm_rbuff * shm_rbuff_open(pid_t pid, - int flow_id) -{ - return rbuff_create(pid, flow_id, O_RDWR); -} - -#if (defined(SHM_RBUFF_LOCKLESS) && \ - (defined(__GNUC__) || defined (__clang__))) -#include "shm_rbuff_ll.c" -#else -#include "shm_rbuff_pthr.c" -#endif diff --git a/src/lib/shm_rbuff_ll.c b/src/lib/shm_rbuff_ll.c deleted file mode 100644 index 7c2a7d60..00000000 --- a/src/lib/shm_rbuff_ll.c +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * Lockless ring buffer for incoming packets - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -#define RB_HEAD __sync_fetch_and_add(rb->head, 0) -#define RB_TAIL __sync_fetch_and_add(rb->tail, 0) - -void shm_rbuff_destroy(struct shm_rbuff * rb) -{ - char fn[FN_MAX_CHARS]; - - assert(rb); - - sprintf(fn, SHM_RBUFF_PREFIX "%d.%d", rb->pid, rb->flow_id); - - shm_rbuff_close(rb); - - shm_unlink(fn); -} - -int shm_rbuff_write(struct shm_rbuff * rb, - size_t idx) -{ - size_t ohead; - size_t nhead; - bool was_empty = false; - - assert(rb); - assert(idx < SHM_BUFFER_SIZE); - - if (__sync_fetch_and_add(rb->acl, 0) != ACL_RDWR) { - if (__sync_fetch_and_add(rb->acl, 0) & ACL_FLOWDOWN) - return -EFLOWDOWN; - else if (__sync_fetch_and_add(rb->acl, 0) & ACL_RDONLY) - return -ENOTALLOC; - } - - if (!shm_rbuff_free(rb)) - return -EAGAIN; - - if (shm_rbuff_empty(rb)) - was_empty = true; - - nhead = RB_HEAD; - - *(rb->shm_base + nhead) = (ssize_t) idx; - - do { - ohead = nhead; - nhead = (ohead + 1) & ((SHM_RBUFF_SIZE) - 1); - nhead = __sync_val_compare_and_swap(rb->head, ohead, nhead); - } while (nhead != ohead); - - if (was_empty) - pthread_cond_broadcast(rb->add); - - return 0; -} - -/* FIXME: this is a copy of the pthr implementation */ -int shm_rbuff_write_b(struct shm_rbuff * rb, - size_t idx, - const struct timespec * abstime) -{ - int ret = 0; - - assert(rb); - assert(idx < SHM_BUFFER_SIZE); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - - if (*rb->acl != ACL_RDWR) { - if (*rb->acl & ACL_FLOWDOWN) - ret = -EFLOWDOWN; - else if (*rb->acl & ACL_RDONLY) - ret = -ENOTALLOC; - goto err; - } - - pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock, - (void *) rb->lock); - - while (!shm_rbuff_free(rb) && ret != -ETIMEDOUT) { - if (abstime != NULL) - ret = -pthread_cond_timedwait(rb->add, - rb->lock, - abstime); - else - ret = -pthread_cond_wait(rb->add, rb->lock); -#ifdef HAVE_ROBUST_MUTEX - if (ret == -EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - } - - if (shm_rbuff_empty(rb)) - pthread_cond_broadcast(rb->add); - - if (ret != -ETIMEDOUT) { - *head_el_ptr(rb) = (ssize_t) idx; - *rb->head = (*rb->head + 1) & ((SHM_RBUFF_SIZE) -1); - } - - pthread_cleanup_pop(true); - - return ret; - err: - pthread_mutex_unlock(rb->lock); - return ret; -} - -ssize_t shm_rbuff_read(struct shm_rbuff * rb) -{ - size_t otail; - size_t ntail; - - assert(rb); - - if (shm_rbuff_empty(rb)) - return __sync_fetch_and_add(rb->acl, 0) & ACL_FLOWDOWN ? - -EFLOWDOWN : -EAGAIN; - - ntail = RB_TAIL; - - do { - otail = ntail; - ntail = (otail + 1) & ((SHM_RBUFF_SIZE) - 1); - ntail = __sync_val_compare_and_swap(rb->tail, otail, ntail); - } while (ntail != otail); - - pthread_cond_broadcast(rb->del); - - return *(rb->shm_base + ntail); -} - -ssize_t shm_rbuff_read_b(struct shm_rbuff * rb, - const struct timespec * abstime) -{ - ssize_t idx = -1; - - assert(rb); - - /* try a non-blocking read first */ - idx = shm_rbuff_read(rb); - if (idx != -EAGAIN) - return idx; - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock, - (void *) rb->lock); - - while (shm_rbuff_empty(rb) && (idx != -ETIMEDOUT)) { - if (abstime != NULL) - idx = -pthread_cond_timedwait(rb->add, - rb->lock, - abstime); - else - idx = -pthread_cond_wait(rb->add, rb->lock); -#ifdef HAVE_ROBUST_MUTEX - if (idx == -EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - } - - if (idx != -ETIMEDOUT) { - /* do a nonblocking read */ - idx = shm_rbuff_read(rb); - assert(idx >= 0); - } - - pthread_cleanup_pop(true); - - return idx; -} - -void shm_rbuff_set_acl(struct shm_rbuff * rb, - uint32_t flags) -{ - assert(rb); - - __sync_bool_compare_and_swap(rb->acl, *rb->acl, flags); -} - -uint32_t shm_rbuff_get_acl(struct shm_rbuff * rb) -{ - assert(rb); - - return __sync_fetch_and_add(rb->acl, 0); -} - -void shm_rbuff_fini(struct shm_rbuff * rb) -{ - assert(rb); - - if (shm_rbuff_empty(rb)) - return; - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - - pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock, - (void *) rb->lock); - - while (!shm_rbuff_empty(rb)) -#ifndef HAVE_ROBUST_MUTEX - pthread_cond_wait(rb->del, rb->lock); -#else - if (pthread_cond_wait(rb->del, rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - pthread_cleanup_pop(true); -} - -size_t shm_rbuff_queued(struct shm_rbuff * rb) -{ - assert(rb); - - return shm_rbuff_used(rb); -} diff --git a/src/lib/shm_rbuff_pthr.c b/src/lib/shm_rbuff_pthr.c deleted file mode 100644 index 91eb8b5f..00000000 --- a/src/lib/shm_rbuff_pthr.c +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * Ring buffer for incoming packets - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -void shm_rbuff_destroy(struct shm_rbuff * rb) -{ - char fn[FN_MAX_CHARS]; - - assert(rb); - -#ifdef CONFIG_OUROBOROS_DEBUG - pthread_mutex_lock(rb->lock); - - assert(shm_rbuff_empty(rb)); - - pthread_mutex_unlock(rb->lock); -#endif - sprintf(fn, SHM_RBUFF_PREFIX "%d.%d", rb->pid, rb->flow_id); - - shm_rbuff_close(rb); - - shm_unlink(fn); -} - -int shm_rbuff_write(struct shm_rbuff * rb, - size_t idx) -{ - int ret = 0; - - assert(rb); - assert(idx < SHM_BUFFER_SIZE); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - - if (*rb->acl != ACL_RDWR) { - if (*rb->acl & ACL_FLOWDOWN) - ret = -EFLOWDOWN; - else if (*rb->acl & ACL_RDONLY) - ret = -ENOTALLOC; - goto err; - } - - if (!shm_rbuff_free(rb)) { - ret = -EAGAIN; - goto err; - } - - if (shm_rbuff_empty(rb)) - pthread_cond_broadcast(rb->add); - - *head_el_ptr(rb) = (ssize_t) idx; - *rb->head = (*rb->head + 1) & ((SHM_RBUFF_SIZE) - 1); - - pthread_mutex_unlock(rb->lock); - - return 0; - err: - pthread_mutex_unlock(rb->lock); - return ret; -} - -int shm_rbuff_write_b(struct shm_rbuff * rb, - size_t idx, - const struct timespec * abstime) -{ - int ret = 0; - - assert(rb); - assert(idx < SHM_BUFFER_SIZE); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - - if (*rb->acl != ACL_RDWR) { - if (*rb->acl & ACL_FLOWDOWN) - ret = -EFLOWDOWN; - else if (*rb->acl & ACL_RDONLY) - ret = -ENOTALLOC; - goto err; - } - - pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock, - (void *) rb->lock); - - while (!shm_rbuff_free(rb) - && ret != -ETIMEDOUT - && !(*rb->acl & ACL_FLOWDOWN)) { - if (abstime != NULL) - ret = -pthread_cond_timedwait(rb->del, - rb->lock, - abstime); - else - ret = -pthread_cond_wait(rb->del, rb->lock); -#ifdef HAVE_ROBUST_MUTEX - if (ret == -EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - } - - if (ret != -ETIMEDOUT) { - if (shm_rbuff_empty(rb)) - pthread_cond_broadcast(rb->add); - *head_el_ptr(rb) = (ssize_t) idx; - *rb->head = (*rb->head + 1) & ((SHM_RBUFF_SIZE) - 1); - } - - pthread_cleanup_pop(true); - - return ret; - err: - pthread_mutex_unlock(rb->lock); - return ret; -} - -ssize_t shm_rbuff_read(struct shm_rbuff * rb) -{ - ssize_t ret = 0; - - assert(rb); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - - if (shm_rbuff_empty(rb)) { - ret = *rb->acl & ACL_FLOWDOWN ? -EFLOWDOWN : -EAGAIN; - pthread_mutex_unlock(rb->lock); - return ret; - } - - ret = *tail_el_ptr(rb); - *rb->tail = (*rb->tail + 1) & ((SHM_RBUFF_SIZE) - 1); - pthread_cond_broadcast(rb->del); - - pthread_mutex_unlock(rb->lock); - - return ret; -} - -ssize_t shm_rbuff_read_b(struct shm_rbuff * rb, - const struct timespec * abstime) -{ - ssize_t idx = -1; - - assert(rb); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - - if (shm_rbuff_empty(rb) && (*rb->acl & ACL_FLOWDOWN)) { - pthread_mutex_unlock(rb->lock); - return -EFLOWDOWN; - } - - pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock, - (void *) rb->lock); - - while (shm_rbuff_empty(rb) - && (idx != -ETIMEDOUT) - && !(*rb->acl & ACL_FLOWDOWN)) { - if (abstime != NULL) - idx = -pthread_cond_timedwait(rb->add, - rb->lock, - abstime); - else - idx = -pthread_cond_wait(rb->add, rb->lock); -#ifdef HAVE_ROBUST_MUTEX - if (idx == -EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - } - - if (idx != -ETIMEDOUT) { - idx = *tail_el_ptr(rb); - *rb->tail = (*rb->tail + 1) & ((SHM_RBUFF_SIZE) - 1); - pthread_cond_broadcast(rb->del); - } - - pthread_cleanup_pop(true); - - return idx; -} - -void shm_rbuff_set_acl(struct shm_rbuff * rb, - uint32_t flags) -{ - assert(rb); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - *rb->acl = (size_t) flags; - - pthread_cond_broadcast(rb->del); - pthread_cond_broadcast(rb->add); - - pthread_mutex_unlock(rb->lock); -} - -uint32_t shm_rbuff_get_acl(struct shm_rbuff * rb) -{ - uint32_t flags; - - assert(rb); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - flags = (uint32_t) *rb->acl; - - pthread_mutex_unlock(rb->lock); - - return flags; -} - -void shm_rbuff_fini(struct shm_rbuff * rb) -{ - assert(rb); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock, - (void *) rb->lock); - - while (!shm_rbuff_empty(rb)) -#ifndef HAVE_ROBUST_MUTEX - pthread_cond_wait(rb->del, rb->lock); -#else - if (pthread_cond_wait(rb->del, rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - pthread_cleanup_pop(true); -} - -size_t shm_rbuff_queued(struct shm_rbuff * rb) -{ - size_t ret; - - assert(rb); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rb->lock); -#else - if (pthread_mutex_lock(rb->lock) == EOWNERDEAD) - pthread_mutex_consistent(rb->lock); -#endif - - ret = shm_rbuff_used(rb); - - pthread_mutex_unlock(rb->lock); - - return ret; -} diff --git a/src/lib/shm_rdrbuff.c b/src/lib/shm_rdrbuff.c deleted file mode 100644 index 76943989..00000000 --- a/src/lib/shm_rdrbuff.c +++ /dev/null @@ -1,612 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * Random Deletion Ring Buffer for Data Units - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -#define _POSIX_C_SOURCE 200809L - -#include "config.h" - -#include <ouroboros/errno.h> -#include <ouroboros/shm_rdrbuff.h> -#include <ouroboros/shm_du_buff.h> -#include <ouroboros/time_utils.h> - -#include <pthread.h> -#include <sys/mman.h> -#include <fcntl.h> -#include <unistd.h> -#include <stdlib.h> -#include <string.h> -#include <stdio.h> -#include <signal.h> -#include <sys/stat.h> -#include <stdbool.h> -#include <assert.h> - -#define SHM_BLOCKS_SIZE ((SHM_BUFFER_SIZE) * SHM_RDRB_BLOCK_SIZE) -#define SHM_FILE_SIZE (SHM_BLOCKS_SIZE + 2 * sizeof(size_t) \ - + sizeof(pthread_mutex_t) + 2 * sizeof(pthread_cond_t) \ - + sizeof(pid_t)) -#define DU_BUFF_OVERHEAD (DU_BUFF_HEADSPACE + DU_BUFF_TAILSPACE) - -#define get_head_ptr(rdrb) \ - idx_to_du_buff_ptr(rdrb, *rdrb->head) - -#define get_tail_ptr(rdrb) \ - idx_to_du_buff_ptr(rdrb, *rdrb->tail) - -#define idx_to_du_buff_ptr(rdrb, idx) \ - ((struct shm_du_buff *) (rdrb->shm_base + idx * SHM_RDRB_BLOCK_SIZE)) - -#define shm_rdrb_used(rdrb) \ - (((*rdrb->head + (SHM_BUFFER_SIZE) - *rdrb->tail) + 1) \ - & ((SHM_BUFFER_SIZE) - 1)) - -#define shm_rdrb_free(rdrb, i) \ - (shm_rdrb_used(rdrb) + i < (SHM_BUFFER_SIZE)) - -#define shm_rdrb_empty(rdrb) \ - (*rdrb->tail == *rdrb->head) - -struct shm_du_buff { - size_t size; -#ifdef SHM_RDRB_MULTI_BLOCK - size_t blocks; -#endif - size_t du_head; - size_t du_tail; - size_t refs; - size_t idx; -}; - -struct shm_rdrbuff { - uint8_t * shm_base; /* start of blocks */ - size_t * head; /* start of ringbuffer head */ - size_t * tail; /* start of ringbuffer tail */ - pthread_mutex_t * lock; /* lock all free space in shm */ - pthread_cond_t * healthy; /* flag when packet is read */ - pid_t * pid; /* pid of the irmd owner */ -}; - -static void garbage_collect(struct shm_rdrbuff * rdrb) -{ -#ifdef SHM_RDRB_MULTI_BLOCK - struct shm_du_buff * sdb; - while (!shm_rdrb_empty(rdrb) && - (sdb = get_tail_ptr(rdrb))->refs == 0) - *rdrb->tail = (*rdrb->tail + sdb->blocks) - & ((SHM_BUFFER_SIZE) - 1); -#else - while (!shm_rdrb_empty(rdrb) && get_tail_ptr(rdrb)->refs == 0) - *rdrb->tail = (*rdrb->tail + 1) & ((SHM_BUFFER_SIZE) - 1); -#endif - pthread_cond_broadcast(rdrb->healthy); -} - -#ifdef HAVE_ROBUST_MUTEX -static void sanitize(struct shm_rdrbuff * rdrb) -{ - --get_head_ptr(rdrb)->refs; - garbage_collect(rdrb); - pthread_mutex_consistent(rdrb->lock); -} -#endif - -static char * rdrb_filename(void) -{ - char * str; - - str = malloc(strlen(SHM_RDRB_NAME) + 1); - if (str == NULL) - return NULL; - - sprintf(str, "%s", SHM_RDRB_NAME); - - return str; -} - -void shm_rdrbuff_close(struct shm_rdrbuff * rdrb) -{ - assert(rdrb); - - munmap(rdrb->shm_base, SHM_FILE_SIZE); - free(rdrb); -} - -void shm_rdrbuff_destroy(struct shm_rdrbuff * rdrb) -{ - char * shm_rdrb_fn; - - assert(rdrb); - - if (getpid() != *rdrb->pid && kill(*rdrb->pid, 0) == 0) { - free(rdrb); - return; - } - - shm_rdrbuff_close(rdrb); - - shm_rdrb_fn = rdrb_filename(); - if (shm_rdrb_fn == NULL) - return; - - shm_unlink(shm_rdrb_fn); - free(shm_rdrb_fn); -} - -#define MM_FLAGS (PROT_READ | PROT_WRITE) - -static struct shm_rdrbuff * rdrb_create(int flags) -{ - struct shm_rdrbuff * rdrb; - int fd; - uint8_t * shm_base; - char * shm_rdrb_fn; - - shm_rdrb_fn = rdrb_filename(); - if (shm_rdrb_fn == NULL) - goto fail_fn; - - rdrb = malloc(sizeof *rdrb); - if (rdrb == NULL) - goto fail_rdrb; - - fd = shm_open(shm_rdrb_fn, flags, 0666); - if (fd == -1) - goto fail_open; - - if ((flags & O_CREAT) && ftruncate(fd, SHM_FILE_SIZE - 1) < 0) - goto fail_truncate; - - shm_base = mmap(NULL, SHM_FILE_SIZE, MM_FLAGS, MAP_SHARED, fd, 0); - if (shm_base == MAP_FAILED) - goto fail_truncate; - - close(fd); - - rdrb->shm_base = shm_base; - rdrb->head = (size_t *) ((uint8_t *) rdrb->shm_base + SHM_BLOCKS_SIZE); - rdrb->tail = rdrb->head + 1; - rdrb->lock = (pthread_mutex_t *) (rdrb->tail + 1); - rdrb->healthy = (pthread_cond_t *) (rdrb->lock + 1); - rdrb->pid = (pid_t *) (rdrb->healthy + 1); - - free(shm_rdrb_fn); - - return rdrb; - - fail_truncate: - close(fd); - if (flags & O_CREAT) - shm_unlink(shm_rdrb_fn); - fail_open: - free(rdrb); - fail_rdrb: - free(shm_rdrb_fn); - fail_fn: - return NULL; -} - -struct shm_rdrbuff * shm_rdrbuff_create() -{ - struct shm_rdrbuff * rdrb; - mode_t mask; - pthread_mutexattr_t mattr; - pthread_condattr_t cattr; - - mask = umask(0); - - rdrb = rdrb_create(O_CREAT | O_EXCL | O_RDWR); - - umask(mask); - - if (rdrb == NULL) - goto fail_rdrb; - - if (pthread_mutexattr_init(&mattr)) - goto fail_mattr; - - pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); -#ifdef HAVE_ROBUST_MUTEX - pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); -#endif - if (pthread_mutex_init(rdrb->lock, &mattr)) - goto fail_mutex; - - if (pthread_condattr_init(&cattr)) - goto fail_cattr; - - pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); -#ifndef __APPLE__ - pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); -#endif - if (pthread_cond_init(rdrb->healthy, &cattr)) - goto fail_healthy; - - *rdrb->head = 0; - *rdrb->tail = 0; - - *rdrb->pid = getpid(); - - pthread_mutexattr_destroy(&mattr); - pthread_condattr_destroy(&cattr); - - return rdrb; - - fail_healthy: - pthread_condattr_destroy(&cattr); - fail_cattr: - pthread_mutex_destroy(rdrb->lock); - fail_mutex: - pthread_mutexattr_destroy(&mattr); - fail_mattr: - shm_rdrbuff_destroy(rdrb); - fail_rdrb: - return NULL; -} - -struct shm_rdrbuff * shm_rdrbuff_open() -{ - return rdrb_create(O_RDWR); -} - -void shm_rdrbuff_purge(void) -{ - char * shm_rdrb_fn; - - shm_rdrb_fn = rdrb_filename(); - if (shm_rdrb_fn == NULL) - return; - - shm_unlink(shm_rdrb_fn); - free(shm_rdrb_fn); -} - -ssize_t shm_rdrbuff_alloc(struct shm_rdrbuff * rdrb, - size_t len, - uint8_t ** ptr, - struct shm_du_buff ** psdb) -{ - struct shm_du_buff * sdb; - size_t size = DU_BUFF_OVERHEAD + len; -#ifdef SHM_RDRB_MULTI_BLOCK - size_t blocks = 0; - size_t padblocks = 0; -#endif - ssize_t sz = size + sizeof(*sdb); - - assert(rdrb); - assert(psdb); - -#ifndef SHM_RDRB_MULTI_BLOCK - if (sz > SHM_RDRB_BLOCK_SIZE) - return -EMSGSIZE; -#else - while (sz > 0) { - sz -= SHM_RDRB_BLOCK_SIZE; - ++blocks; - } -#endif -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rdrb->lock); -#else - if (pthread_mutex_lock(rdrb->lock) == EOWNERDEAD) - sanitize(rdrb); -#endif -#ifdef SHM_RDRB_MULTI_BLOCK - if (blocks + *rdrb->head > (SHM_BUFFER_SIZE)) - padblocks = (SHM_BUFFER_SIZE) - *rdrb->head; - - if (!shm_rdrb_free(rdrb, blocks + padblocks)) { -#else - if (!shm_rdrb_free(rdrb, 1)) { -#endif - pthread_mutex_unlock(rdrb->lock); - return -EAGAIN; - } - -#ifdef SHM_RDRB_MULTI_BLOCK - if (padblocks) { - sdb = get_head_ptr(rdrb); - sdb->size = 0; - sdb->blocks = padblocks; - sdb->refs = 0; - sdb->du_head = 0; - sdb->du_tail = 0; - sdb->idx = *rdrb->head; - - *rdrb->head = 0; - } -#endif - sdb = get_head_ptr(rdrb); - sdb->refs = 1; - sdb->idx = *rdrb->head; -#ifdef SHM_RDRB_MULTI_BLOCK - sdb->blocks = blocks; - - *rdrb->head = (*rdrb->head + blocks) & ((SHM_BUFFER_SIZE) - 1); -#else - *rdrb->head = (*rdrb->head + 1) & ((SHM_BUFFER_SIZE) - 1); -#endif - pthread_mutex_unlock(rdrb->lock); - - sdb->size = size; - sdb->du_head = DU_BUFF_HEADSPACE; - sdb->du_tail = sdb->du_head + len; - - *psdb = sdb; - if (ptr != NULL) - *ptr = (uint8_t *) (sdb + 1) + sdb->du_head; - - return sdb->idx; -} - -ssize_t shm_rdrbuff_alloc_b(struct shm_rdrbuff * rdrb, - size_t len, - uint8_t ** ptr, - struct shm_du_buff ** psdb, - const struct timespec * abstime) -{ - struct shm_du_buff * sdb; - size_t size = DU_BUFF_OVERHEAD + len; -#ifdef SHM_RDRB_MULTI_BLOCK - size_t blocks = 0; - size_t padblocks = 0; -#endif - ssize_t sz = size + sizeof(*sdb); - int ret = 0; - - assert(rdrb); - assert(psdb); - -#ifndef SHM_RDRB_MULTI_BLOCK - if (sz > SHM_RDRB_BLOCK_SIZE) - return -EMSGSIZE; -#else - while (sz > 0) { - sz -= SHM_RDRB_BLOCK_SIZE; - ++blocks; - } -#endif -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rdrb->lock); -#else - if (pthread_mutex_lock(rdrb->lock) == EOWNERDEAD) - sanitize(rdrb); -#endif - pthread_cleanup_push((void (*) (void *)) pthread_mutex_unlock, - (void *) rdrb->lock); - -#ifdef SHM_RDRB_MULTI_BLOCK - if (blocks + *rdrb->head > (SHM_BUFFER_SIZE)) - padblocks = (SHM_BUFFER_SIZE) - *rdrb->head; - - while (!shm_rdrb_free(rdrb, blocks + padblocks) && ret != ETIMEDOUT) { -#else - while (!shm_rdrb_free(rdrb, 1) && ret != ETIMEDOUT) { -#endif - if (abstime != NULL) - ret = pthread_cond_timedwait(rdrb->healthy, - rdrb->lock, - abstime); - else - ret = pthread_cond_wait(rdrb->healthy, rdrb->lock); - -#ifdef SHM_RDRB_MULTI_BLOCK - if (blocks + *rdrb->head > (SHM_BUFFER_SIZE)) - padblocks = (SHM_BUFFER_SIZE) - *rdrb->head; -#endif - } - - if (ret != ETIMEDOUT) { -#ifdef SHM_RDRB_MULTI_BLOCK - if (padblocks) { - sdb = get_head_ptr(rdrb); - sdb->size = 0; - sdb->blocks = padblocks; - sdb->refs = 0; - sdb->du_head = 0; - sdb->du_tail = 0; - sdb->idx = *rdrb->head; - - *rdrb->head = 0; - } -#endif - sdb = get_head_ptr(rdrb); - sdb->refs = 1; - sdb->idx = *rdrb->head; -#ifdef SHM_RDRB_MULTI_BLOCK - sdb->blocks = blocks; - - *rdrb->head = (*rdrb->head + blocks) & ((SHM_BUFFER_SIZE) - 1); -#else - *rdrb->head = (*rdrb->head + 1) & ((SHM_BUFFER_SIZE) - 1); -#endif - } - - pthread_cleanup_pop(true); - - if (ret == ETIMEDOUT) - return -ETIMEDOUT; - - sdb->size = size; - sdb->du_head = DU_BUFF_HEADSPACE; - sdb->du_tail = sdb->du_head + len; - - *psdb = sdb; - if (ptr != NULL) - *ptr = (uint8_t *) (sdb + 1) + sdb->du_head; - - return sdb->idx; -} - -ssize_t shm_rdrbuff_read(uint8_t ** dst, - struct shm_rdrbuff * rdrb, - size_t idx) -{ - struct shm_du_buff * sdb; - - assert(dst); - assert(rdrb); - assert(idx < (SHM_BUFFER_SIZE)); - - sdb = idx_to_du_buff_ptr(rdrb, idx); - *dst = ((uint8_t *) (sdb + 1)) + sdb->du_head; - - return (ssize_t) (sdb->du_tail - sdb->du_head); -} - -struct shm_du_buff * shm_rdrbuff_get(struct shm_rdrbuff * rdrb, - size_t idx) -{ - assert(rdrb); - assert(idx < (SHM_BUFFER_SIZE)); - - return idx_to_du_buff_ptr(rdrb, idx); -} - -int shm_rdrbuff_remove(struct shm_rdrbuff * rdrb, - size_t idx) -{ - struct shm_du_buff * sdb; - - assert(rdrb); - assert(idx < (SHM_BUFFER_SIZE)); - -#ifndef HAVE_ROBUST_MUTEX - pthread_mutex_lock(rdrb->lock); -#else - if (pthread_mutex_lock(rdrb->lock) == EOWNERDEAD) - sanitize(rdrb); -#endif - assert(!shm_rdrb_empty(rdrb)); - - sdb = idx_to_du_buff_ptr(rdrb, idx); - - if (sdb->refs == 1) { /* only stack needs it, can be removed */ - sdb->refs = 0; - if (idx == *rdrb->tail) - garbage_collect(rdrb); - } - - pthread_mutex_unlock(rdrb->lock); - - return 0; -} - -size_t shm_du_buff_get_idx(struct shm_du_buff * sdb) -{ - assert(sdb); - - return sdb->idx; -} - -uint8_t * shm_du_buff_head(struct shm_du_buff * sdb) -{ - assert(sdb); - - return (uint8_t *) (sdb + 1) + sdb->du_head; -} - -uint8_t * shm_du_buff_tail(struct shm_du_buff * sdb) -{ - assert(sdb); - - return (uint8_t *) (sdb + 1) + sdb->du_tail; -} - -uint8_t * shm_du_buff_head_alloc(struct shm_du_buff * sdb, - size_t size) -{ - assert(sdb); - - if (sdb->du_head < size) - return NULL; - - sdb->du_head -= size; - - return (uint8_t *) (sdb + 1) + sdb->du_head; -} - -uint8_t * shm_du_buff_tail_alloc(struct shm_du_buff * sdb, - size_t size) -{ - uint8_t * buf; - - assert(sdb); - - if (sdb->du_tail + size >= sdb->size) - return NULL; - - buf = (uint8_t *) (sdb + 1) + sdb->du_tail; - - sdb->du_tail += size; - - return buf; -} - -uint8_t * shm_du_buff_head_release(struct shm_du_buff * sdb, - size_t size) -{ - uint8_t * buf; - - assert(sdb); - assert(!(size > sdb->du_tail - sdb->du_head)); - - buf = (uint8_t *) (sdb + 1) + sdb->du_head; - - sdb->du_head += size; - - return buf; -} - -uint8_t * shm_du_buff_tail_release(struct shm_du_buff * sdb, - size_t size) -{ - assert(sdb); - assert(!(size > sdb->du_tail - sdb->du_head)); - - sdb->du_tail -= size; - - return (uint8_t *) (sdb + 1) + sdb->du_tail; -} - -void shm_du_buff_truncate(struct shm_du_buff * sdb, - size_t len) -{ - assert(sdb); - assert(len <= sdb->size); - - sdb->du_tail = sdb->du_head + len; -} - -int shm_du_buff_wait_ack(struct shm_du_buff * sdb) -{ - __sync_add_and_fetch(&sdb->refs, 1); - - return 0; -} - -int shm_du_buff_ack(struct shm_du_buff * sdb) -{ - __sync_sub_and_fetch(&sdb->refs, 1); - return 0; -} diff --git a/src/lib/sockets.c b/src/lib/sockets.c index f25ee2ce..46586911 100644 --- a/src/lib/sockets.c +++ b/src/lib/sockets.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * The sockets layer to communicate between daemons * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -21,6 +21,7 @@ */ #include <ouroboros/errno.h> +#include <ouroboros/pthread.h> #include <ouroboros/sockets.h> #include <ouroboros/utils.h> @@ -29,9 +30,7 @@ #include <string.h> #include <stdio.h> #include <stdlib.h> -#include <pthread.h> #include <stdbool.h> -#include <sys/time.h> /* Apple doesn't support SEQPACKET. */ #ifdef __APPLE__ @@ -40,6 +39,11 @@ #define SOCK_TYPE SOCK_SEQPACKET #endif +void __cleanup_close_ptr(void * fd) +{ + close(*(int *) fd); +} + int client_socket_open(char * file_name) { int sockfd; @@ -52,8 +56,7 @@ int client_socket_open(char * file_name) serv_addr.sun_family = AF_UNIX; sprintf(serv_addr.sun_path, "%s", file_name); - if (connect(sockfd, - (struct sockaddr *) &serv_addr, + if (connect(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr))) { close(sockfd); return -1; @@ -95,17 +98,11 @@ int server_socket_open(char * file_name) return sockfd; } -static void close_ptr(void * o) -{ - close(*(int *) o); -} - irm_msg_t * send_recv_irm_msg(irm_msg_t * msg) { - int sockfd; - uint8_t buf[SOCK_BUF_SIZE]; - ssize_t len; - irm_msg_t * recv_msg = NULL; + int sockfd; + uint8_t buf[SOCK_BUF_SIZE]; + ssize_t len; sockfd = client_socket_open(IRM_SOCK_PATH); if (sockfd < 0) @@ -117,27 +114,53 @@ irm_msg_t * send_recv_irm_msg(irm_msg_t * msg) return NULL; } - pthread_cleanup_push(close_ptr, &sockfd); - irm_msg__pack(msg, buf); - if (write(sockfd, buf, len) != -1) + pthread_cleanup_push(__cleanup_close_ptr, &sockfd); + + len = write(sockfd, buf, len); + if (len >= 0) len = read(sockfd, buf, SOCK_BUF_SIZE); - if (len > 0) - recv_msg = irm_msg__unpack(NULL, len, buf); + pthread_cleanup_pop(true); + + if (len < 0) + goto fail; + + return irm_msg__unpack(NULL, len, buf); + fail: + return NULL; +} + +int send_recv_msg(buffer_t * msg) +{ + int sockfd; + ssize_t len = 0; + + sockfd = client_socket_open(IRM_SOCK_PATH); + if (sockfd < 0) + return -1; + + pthread_cleanup_push(__cleanup_close_ptr, &sockfd); + + len = write(sockfd, msg->data, msg->len); + if (len >= 0) + len = read(sockfd, msg->data, SOCK_BUF_SIZE); pthread_cleanup_pop(true); - return recv_msg; + msg->len = (size_t) len; + + return len < 0 ? -1 : 0; } -char * ipcp_sock_path(pid_t pid) +static char * __sock_path(pid_t pid, + const char * prefix, + const char * suffix) { char * full_name = NULL; char * pid_string = NULL; size_t len = 0; - char * delim = "_"; len = n_digits(pid); pid_string = malloc(len + 1); @@ -146,9 +169,9 @@ char * ipcp_sock_path(pid_t pid) sprintf(pid_string, "%d", pid); - len += strlen(IPCP_SOCK_PATH_PREFIX); - len += strlen(delim); - len += strlen(SOCK_PATH_SUFFIX); + len += strlen(prefix); + len += strlen(pid_string); + len += strlen(suffix); full_name = malloc(len + 1); if (full_name == NULL) { @@ -156,49 +179,17 @@ char * ipcp_sock_path(pid_t pid) return NULL; } - strcpy(full_name, IPCP_SOCK_PATH_PREFIX); - strcat(full_name, delim); + strcpy(full_name, prefix); strcat(full_name, pid_string); - strcat(full_name, SOCK_PATH_SUFFIX); + strcat(full_name, suffix); free(pid_string); return full_name; } -qosspec_msg_t spec_to_msg(const qosspec_t * qs) +char * sock_path(pid_t pid, + const char * prefix) { - qosspec_t spec; - qosspec_msg_t msg = QOSSPEC_MSG__INIT; - - spec = (qs == NULL ? qos_raw : *qs); - - msg.delay = spec.delay; - msg.bandwidth = spec.bandwidth; - msg.availability = spec.availability; - msg.loss = spec.loss; - msg.ber = spec.ber; - msg.in_order = spec.in_order; - msg.max_gap = spec.max_gap; - msg.cypher_s = spec.cypher_s; - - return msg; -} - -qosspec_t msg_to_spec(const qosspec_msg_t * msg) -{ - qosspec_t spec; - - assert(msg); - - spec.delay = msg->delay; - spec.bandwidth = msg->bandwidth; - spec.availability = msg->availability; - spec.loss = msg->loss; - spec.ber = msg->ber; - spec.in_order = msg->in_order; - spec.max_gap = msg->max_gap; - spec.cypher_s = msg->cypher_s; - - return spec; + return __sock_path(pid, prefix, SOCK_PATH_SUFFIX); } diff --git a/src/lib/shm_flow_set.c b/src/lib/ssm/flow_set.c index 0f701b63..cb38e6fd 100644 --- a/src/lib/shm_flow_set.c +++ b/src/lib/ssm/flow_set.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Management of flow_sets for fqueue * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -23,22 +23,23 @@ #define _POSIX_C_SOURCE 200809L #include "config.h" +#include "ssm.h" -#include <ouroboros/lockfile.h> -#include <ouroboros/time_utils.h> -#include <ouroboros/shm_flow_set.h> #include <ouroboros/errno.h> +#include <ouroboros/lockfile.h> +#include <ouroboros/pthread.h> +#include <ouroboros/ssm_flow_set.h> +#include <ouroboros/time.h> -#include <pthread.h> -#include <sys/mman.h> +#include <assert.h> #include <fcntl.h> +#include <signal.h> #include <stdlib.h> #include <stdio.h> +#include <string.h> #include <unistd.h> -#include <signal.h> +#include <sys/mman.h> #include <sys/stat.h> -#include <string.h> -#include <assert.h> /* * pthread_cond_timedwait has a WONTFIX bug as of glibc 2.25 where it @@ -52,88 +53,78 @@ #endif #define FN_MAX_CHARS 255 +#define FS_PROT (PROT_READ | PROT_WRITE) -#define QUEUESIZE ((SHM_BUFFER_SIZE) * sizeof(struct portevent)) +#define QUEUESIZE ((SSM_RBUFF_SIZE) * sizeof(struct flowevent)) -#define SHM_FLOW_SET_FILE_SIZE (SYS_MAX_FLOWS * sizeof(ssize_t) \ - + PROG_MAX_FQUEUES * sizeof(size_t) \ - + PROG_MAX_FQUEUES * sizeof(pthread_cond_t) \ - + PROG_MAX_FQUEUES * QUEUESIZE \ - + sizeof(pthread_mutex_t)) +#define SSM_FSET_FILE_SIZE (SYS_MAX_FLOWS * sizeof(ssize_t) \ + + PROC_MAX_FQUEUES * sizeof(size_t) \ + + PROC_MAX_FQUEUES * sizeof(pthread_cond_t) \ + + PROC_MAX_FQUEUES * QUEUESIZE \ + + sizeof(pthread_mutex_t)) -#define fqueue_ptr(fs, idx) (fs->fqueues + (SHM_BUFFER_SIZE) * idx) +#define fqueue_ptr(fs, idx) (fs->fqueues + (SSM_RBUFF_SIZE) * idx) -struct portevent { - int flow_id; - int event; -}; - -struct shm_flow_set { +struct ssm_flow_set { ssize_t * mtable; size_t * heads; pthread_cond_t * conds; - struct portevent * fqueues; + struct flowevent * fqueues; pthread_mutex_t * lock; pid_t pid; }; -static struct shm_flow_set * flow_set_create(pid_t pid, - int flags) +static struct ssm_flow_set * flow_set_create(pid_t pid, + int oflags) { - struct shm_flow_set * set; + struct ssm_flow_set * set; ssize_t * shm_base; char fn[FN_MAX_CHARS]; - int shm_fd; + int fd; - sprintf(fn, SHM_FLOW_SET_PREFIX "%d", pid); + sprintf(fn, SSM_FLOW_SET_PREFIX "%d", pid); set = malloc(sizeof(*set)); if (set == NULL) goto fail_malloc; - shm_fd = shm_open(fn, flags, 0666); - if (shm_fd == -1) + fd = shm_open(fn, oflags, 0666); + if (fd == -1) goto fail_shm_open; - if (ftruncate(shm_fd, SHM_FLOW_SET_FILE_SIZE - 1) < 0) { - close(shm_fd); - goto fail_shm_open; - } - - shm_base = mmap(NULL, - SHM_FLOW_SET_FILE_SIZE, - PROT_READ | PROT_WRITE, - MAP_SHARED, - shm_fd, - 0); - - close(shm_fd); + if ((oflags & O_CREAT) && ftruncate(fd, SSM_FSET_FILE_SIZE) < 0) + goto fail_truncate; + shm_base = mmap(NULL, SSM_FSET_FILE_SIZE, FS_PROT, MAP_SHARED, fd, 0); if (shm_base == MAP_FAILED) goto fail_mmap; + close(fd); + set->mtable = shm_base; set->heads = (size_t *) (set->mtable + SYS_MAX_FLOWS); - set->conds = (pthread_cond_t *)(set->heads + PROG_MAX_FQUEUES); - set->fqueues = (struct portevent *) (set->conds + PROG_MAX_FQUEUES); + set->conds = (pthread_cond_t *)(set->heads + PROC_MAX_FQUEUES); + set->fqueues = (struct flowevent *) (set->conds + PROC_MAX_FQUEUES); set->lock = (pthread_mutex_t *) - (set->fqueues + PROG_MAX_FQUEUES * (SHM_BUFFER_SIZE)); + (set->fqueues + PROC_MAX_FQUEUES * (SSM_RBUFF_SIZE)); return set; fail_mmap: - if (flags & O_CREAT) + if (oflags & O_CREAT) shm_unlink(fn); + fail_truncate: + close(fd); fail_shm_open: free(set); fail_malloc: return NULL; } -struct shm_flow_set * shm_flow_set_create(pid_t pid) +struct ssm_flow_set * ssm_flow_set_create(pid_t pid) { - struct shm_flow_set * set; + struct ssm_flow_set * set; pthread_mutexattr_t mattr; pthread_condattr_t cattr; mode_t mask; @@ -173,7 +164,7 @@ struct shm_flow_set * shm_flow_set_create(pid_t pid) if (pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK)) goto fail_condattr_set; #endif - for (i = 0; i < PROG_MAX_FQUEUES; ++i) { + for (i = 0; i < PROC_MAX_FQUEUES; ++i) { set->heads[i] = 0; if (pthread_cond_init(&set->conds[i], &cattr)) goto fail_init; @@ -194,44 +185,44 @@ struct shm_flow_set * shm_flow_set_create(pid_t pid) fail_mattr_set: pthread_mutexattr_destroy(&mattr); fail_mutexattr_init: - shm_flow_set_destroy(set); + ssm_flow_set_destroy(set); fail_set: return NULL; } -struct shm_flow_set * shm_flow_set_open(pid_t pid) +struct ssm_flow_set * ssm_flow_set_open(pid_t pid) { return flow_set_create(pid, O_RDWR); } -void shm_flow_set_destroy(struct shm_flow_set * set) +void ssm_flow_set_destroy(struct ssm_flow_set * set) { - char fn[25]; + char fn[FN_MAX_CHARS]; assert(set); - sprintf(fn, SHM_FLOW_SET_PREFIX "%d", set->pid); + sprintf(fn, SSM_FLOW_SET_PREFIX "%d", set->pid); - shm_flow_set_close(set); + ssm_flow_set_close(set); shm_unlink(fn); } -void shm_flow_set_close(struct shm_flow_set * set) +void ssm_flow_set_close(struct ssm_flow_set * set) { assert(set); - munmap(set->mtable, SHM_FLOW_SET_FILE_SIZE); + munmap(set->mtable, SSM_FSET_FILE_SIZE); free(set); } -void shm_flow_set_zero(struct shm_flow_set * set, +void ssm_flow_set_zero(struct ssm_flow_set * set, size_t idx) { ssize_t i = 0; assert(set); - assert(idx < PROG_MAX_FQUEUES); + assert(idx < PROC_MAX_FQUEUES); pthread_mutex_lock(set->lock); @@ -245,13 +236,13 @@ void shm_flow_set_zero(struct shm_flow_set * set, } -int shm_flow_set_add(struct shm_flow_set * set, +int ssm_flow_set_add(struct ssm_flow_set * set, size_t idx, int flow_id) { assert(set); assert(!(flow_id < 0) && flow_id < SYS_MAX_FLOWS); - assert(idx < PROG_MAX_FQUEUES); + assert(idx < PROC_MAX_FQUEUES); pthread_mutex_lock(set->lock); @@ -267,13 +258,13 @@ int shm_flow_set_add(struct shm_flow_set * set, return 0; } -void shm_flow_set_del(struct shm_flow_set * set, +void ssm_flow_set_del(struct ssm_flow_set * set, size_t idx, int flow_id) { assert(set); assert(!(flow_id < 0) && flow_id < SYS_MAX_FLOWS); - assert(idx < PROG_MAX_FQUEUES); + assert(idx < PROC_MAX_FQUEUES); pthread_mutex_lock(set->lock); @@ -283,7 +274,7 @@ void shm_flow_set_del(struct shm_flow_set * set, pthread_mutex_unlock(set->lock); } -int shm_flow_set_has(struct shm_flow_set * set, +int ssm_flow_set_has(struct ssm_flow_set * set, size_t idx, int flow_id) { @@ -291,7 +282,7 @@ int shm_flow_set_has(struct shm_flow_set * set, assert(set); assert(!(flow_id < 0) && flow_id < SYS_MAX_FLOWS); - assert(idx < PROG_MAX_FQUEUES); + assert(idx < PROC_MAX_FQUEUES); pthread_mutex_lock(set->lock); @@ -303,10 +294,12 @@ int shm_flow_set_has(struct shm_flow_set * set, return ret; } -void shm_flow_set_notify(struct shm_flow_set * set, +void ssm_flow_set_notify(struct ssm_flow_set * set, int flow_id, int event) { + struct flowevent * e; + assert(set); assert(!(flow_id < 0) && flow_id < SYS_MAX_FLOWS); @@ -317,10 +310,13 @@ void shm_flow_set_notify(struct shm_flow_set * set, return; } - (fqueue_ptr(set, set->mtable[flow_id]) + - (set->heads[set->mtable[flow_id]]))->flow_id = flow_id; - (fqueue_ptr(set, set->mtable[flow_id]) + - (set->heads[set->mtable[flow_id]])++)->event = event; + e = fqueue_ptr(set, set->mtable[flow_id]) + + set->heads[set->mtable[flow_id]]; + + e->flow_id = flow_id; + e->event = event; + + ++set->heads[set->mtable[flow_id]]; pthread_cond_signal(&set->conds[set->mtable[flow_id]]); @@ -328,15 +324,15 @@ void shm_flow_set_notify(struct shm_flow_set * set, } -ssize_t shm_flow_set_wait(const struct shm_flow_set * set, +ssize_t ssm_flow_set_wait(const struct ssm_flow_set * set, size_t idx, - int * fqueue, + struct flowevent * fqueue, const struct timespec * abstime) { ssize_t ret = 0; assert(set); - assert(idx < PROG_MAX_FQUEUES); + assert(idx < PROC_MAX_FQUEUES); assert(fqueue); #ifndef HAVE_ROBUST_MUTEX @@ -346,22 +342,14 @@ ssize_t shm_flow_set_wait(const struct shm_flow_set * set, pthread_mutex_consistent(set->lock); #endif - pthread_cleanup_push((void(*)(void *))pthread_mutex_unlock, - (void *) set->lock); + pthread_cleanup_push(__cleanup_mutex_unlock, set->lock); while (set->heads[idx] == 0 && ret != -ETIMEDOUT) { - if (abstime != NULL) { - ret = -pthread_cond_timedwait(set->conds + idx, - set->lock, - abstime); + ret = -__timedwait(set->conds + idx, set->lock, abstime); #ifdef HAVE_CANCEL_BUG - if (ret == -ETIMEDOUT) - pthread_testcancel(); + if (ret == -ETIMEDOUT) + pthread_testcancel(); #endif - } else { - ret = -pthread_cond_wait(set->conds + idx, - set->lock); - } #ifdef HAVE_ROBUST_MUTEX if (ret == -EOWNERDEAD) pthread_mutex_consistent(set->lock); @@ -371,7 +359,7 @@ ssize_t shm_flow_set_wait(const struct shm_flow_set * set, if (ret != -ETIMEDOUT) { memcpy(fqueue, fqueue_ptr(set, idx), - set->heads[idx] * sizeof(struct portevent)); + set->heads[idx] * sizeof(*fqueue)); ret = set->heads[idx]; set->heads[idx] = 0; } diff --git a/src/lib/ssm/pool.c b/src/lib/ssm/pool.c new file mode 100644 index 00000000..5607a360 --- /dev/null +++ b/src/lib/ssm/pool.c @@ -0,0 +1,895 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Secure Shared Memory Infrastructure (SSMI) Packet Buffer + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include "config.h" + +#include <ouroboros/atomics.h> +#include <ouroboros/errno.h> +#include <ouroboros/pthread.h> +#include <ouroboros/ssm_pool.h> + +#include "ssm.h" + +#include <assert.h> +#include <fcntl.h> +#include <signal.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <sys/mman.h> +#include <sys/stat.h> + +/* Global Shared Packet Pool (GSPP) configuration */ +static const struct ssm_size_class_cfg ssm_gspp_cfg[SSM_POOL_MAX_CLASSES] = { + { (1 << 8), SSM_GSPP_256_BLOCKS }, + { (1 << 9), SSM_GSPP_512_BLOCKS }, + { (1 << 10), SSM_GSPP_1K_BLOCKS }, + { (1 << 11), SSM_GSPP_2K_BLOCKS }, + { (1 << 12), SSM_GSPP_4K_BLOCKS }, + { (1 << 14), SSM_GSPP_16K_BLOCKS }, + { (1 << 16), SSM_GSPP_64K_BLOCKS }, + { (1 << 18), SSM_GSPP_256K_BLOCKS }, + { (1 << 20), SSM_GSPP_1M_BLOCKS }, +}; + +/* Per-User Pool (PUP) configuration */ +static const struct ssm_size_class_cfg ssm_pup_cfg[SSM_POOL_MAX_CLASSES] = { + { (1 << 8), SSM_PUP_256_BLOCKS }, + { (1 << 9), SSM_PUP_512_BLOCKS }, + { (1 << 10), SSM_PUP_1K_BLOCKS }, + { (1 << 11), SSM_PUP_2K_BLOCKS }, + { (1 << 12), SSM_PUP_4K_BLOCKS }, + { (1 << 14), SSM_PUP_16K_BLOCKS }, + { (1 << 16), SSM_PUP_64K_BLOCKS }, + { (1 << 18), SSM_PUP_256K_BLOCKS }, + { (1 << 20), SSM_PUP_1M_BLOCKS }, +}; + +#define PTR_TO_OFFSET(pool_base, ptr) \ + ((uintptr_t)(ptr) - (uintptr_t)(pool_base)) + +#define OFFSET_TO_PTR(pool_base, offset) \ + ((offset == 0) ? NULL : (void *)((uintptr_t)(pool_base) + offset)) + +#define GET_SHARD_FOR_PID(pid) ((int)((pid) % SSM_POOL_SHARDS)) + + +#define SSM_FILE_SIZE (SSM_POOL_TOTAL_SIZE + sizeof(struct _ssm_pool_hdr)) +#define SSM_GSPP_FILE_SIZE (SSM_GSPP_TOTAL_SIZE + sizeof(struct _ssm_pool_hdr)) +#define SSM_PUP_FILE_SIZE (SSM_PUP_TOTAL_SIZE + sizeof(struct _ssm_pool_hdr)) + +#define IS_GSPP(uid) ((uid) == SSM_GSPP_UID) +#define GET_POOL_TOTAL_SIZE(uid) (IS_GSPP(uid) ? SSM_GSPP_TOTAL_SIZE \ + : SSM_PUP_TOTAL_SIZE) +#define GET_POOL_FILE_SIZE(uid) (IS_GSPP(uid) ? SSM_GSPP_FILE_SIZE \ + : SSM_PUP_FILE_SIZE) +#define GET_POOL_CFG(uid) (IS_GSPP(uid) ? ssm_gspp_cfg : ssm_pup_cfg) + +#define NEEDS_CHOWN(uid, gid) ((uid) != geteuid() || (gid) != getegid()) + +struct ssm_pool { + uint8_t * shm_base; /* start of blocks */ + struct _ssm_pool_hdr * hdr; /* shared memory header */ + void * pool_base; /* base of the memory pool */ + uid_t uid; /* user owner (0 = GSPP) */ + size_t total_size; /* total data size */ +}; + +static __inline__ +struct ssm_pk_buff * list_remove_head(struct _ssm_list_head * head, + void * base) +{ + uint32_t off; + uint32_t next_off; + struct ssm_pk_buff * blk; + + assert(head != NULL); + assert(base != NULL); + + off = LOAD(&head->head_offset); + if (off == 0) + return NULL; + + /* Validate offset is within pool bounds */ + if (off >= SSM_POOL_TOTAL_SIZE) + return NULL; + + blk = OFFSET_TO_PTR(base, off); + next_off = LOAD(&blk->next_offset); + + + + STORE(&head->head_offset, next_off); + STORE(&head->count, LOAD(&head->count) - 1); + + return blk; +} +static __inline__ void list_add_head(struct _ssm_list_head * head, + struct ssm_pk_buff * blk, + void * base) +{ + uint32_t off; + uint32_t old; + + assert(head != NULL); + assert(blk != NULL); + assert(base != NULL); + + off = (uint32_t) PTR_TO_OFFSET(base, blk); + old = LOAD(&head->head_offset); + + STORE(&blk->next_offset, old); + STORE(&head->head_offset, off); + STORE(&head->count, LOAD(&head->count) + 1); +} + +static __inline__ int find_size_class_for_offset(struct ssm_pool * pool, + size_t offset) +{ + int c; + + assert(pool != NULL); + + for (c = 0; c < SSM_POOL_MAX_CLASSES; c++) { + struct _ssm_size_class * sc = &pool->hdr->size_classes[c]; + + if (sc->object_size == 0) + continue; + + if (offset >= sc->pool_start && + offset < sc->pool_start + sc->pool_size) + return c; + } + + return -1; +} + +static void init_size_classes(struct ssm_pool * pool) +{ + const struct ssm_size_class_cfg * cfg; + struct _ssm_size_class * sc; + struct _ssm_shard * shard; + pthread_mutexattr_t mattr; + pthread_condattr_t cattr; + uint8_t * region; + size_t offset; + int c; /* class iterator */ + int s; /* shard iterator */ + size_t i; + + assert(pool != NULL); + + /* Check if already initialized */ + if (LOAD(&pool->hdr->initialized) != 0) + return; + + cfg = GET_POOL_CFG(pool->uid); + + pthread_mutexattr_init(&mattr); + pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); +#ifdef HAVE_ROBUST_MUTEX + pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); +#endif + pthread_mutexattr_setprotocol(&mattr, PTHREAD_PRIO_INHERIT); + + pthread_condattr_init(&cattr); + pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); +#ifndef __APPLE__ + pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); +#endif + offset = 0; + + for (c = 0; c < SSM_POOL_MAX_CLASSES; c++) { + if (cfg[c].blocks == 0) + continue; + + sc = &pool->hdr->size_classes[c]; + + sc->object_size = cfg[c].size; + sc->pool_start = offset; + sc->pool_size = cfg[c].size * cfg[c].blocks; + sc->object_count = cfg[c].blocks; + + /* Initialize all shards */ + for (s = 0; s < SSM_POOL_SHARDS; s++) { + shard = &sc->shards[s]; + + STORE(&shard->free_list.head_offset, 0); + STORE(&shard->free_list.count, 0); + STORE(&shard->free_count, 0); + + pthread_mutex_init(&shard->mtx, &mattr); + pthread_cond_init(&shard->cond, &cattr); + } + + /* Lazy distribution: put all blocks in shard 0 initially */ + region = pool->shm_base + offset; + + for (i = 0; i < sc->object_count; ++i) { + struct ssm_pk_buff * blk; + + blk = (struct ssm_pk_buff *) + (region + i * sc->object_size); + + STORE(&blk->refcount, 0); + blk->allocator_pid = 0; + STORE(&blk->next_offset, 0); + + list_add_head(&sc->shards[0].free_list, blk, + pool->pool_base); + FETCH_ADD(&sc->shards[0].free_count, 1); + } + + offset += sc->pool_size; + } + + /* Mark as initialized - acts as memory barrier */ + STORE(&pool->hdr->initialized, 1); + + pthread_mutexattr_destroy(&mattr); + pthread_condattr_destroy(&cattr); +} + +/* + * Reclaim all blocks allocated by a specific pid in a size class. + * Called with shard mutex held. + */ +static size_t reclaim_pid_from_sc(struct _ssm_size_class * sc, + struct _ssm_shard * shard, + void * pool_base, + pid_t pid) +{ + uint8_t * region; + size_t i; + size_t recovered = 0; + struct ssm_pk_buff * blk; + + region = (uint8_t *) pool_base + sc->pool_start; + + for (i = 0; i < sc->object_count; ++i) { + blk = (struct ssm_pk_buff *)(region + i * sc->object_size); + + if (blk->allocator_pid == pid && LOAD(&blk->refcount) > 0) { + STORE(&blk->refcount, 0); + blk->allocator_pid = 0; + list_add_head(&shard->free_list, blk, pool_base); + FETCH_ADD(&shard->free_count, 1); + recovered++; + } + } + + return recovered; +} + +void ssm_pool_reclaim_orphans(struct ssm_pool * pool, + pid_t pid) +{ + size_t sc_idx; + + if (pool == NULL || pid <= 0) + return; + + for (sc_idx = 0; sc_idx < SSM_POOL_MAX_CLASSES; sc_idx++) { + struct _ssm_size_class * sc; + struct _ssm_shard * shard; + + sc = &pool->hdr->size_classes[sc_idx]; + if (sc->object_count == 0) + continue; + + /* Reclaim to shard 0 for simplicity */ + shard = &sc->shards[0]; + robust_mutex_lock(&shard->mtx); + reclaim_pid_from_sc(sc, shard, pool->pool_base, pid); + pthread_mutex_unlock(&shard->mtx); + } +} + +static __inline__ +struct ssm_pk_buff * try_alloc_from_shard(struct _ssm_shard * shard, + void * base) +{ + struct ssm_pk_buff * blk; + + robust_mutex_lock(&shard->mtx); + + if (LOAD(&shard->free_count) > 0) { + blk = list_remove_head(&shard->free_list, base); + if (blk != NULL) { + FETCH_SUB(&shard->free_count, 1); + return blk; /* Caller must unlock */ + } + FETCH_SUB(&shard->free_count, 1); + } + + pthread_mutex_unlock(&shard->mtx); + return NULL; +} + +static __inline__ ssize_t init_block(struct ssm_pool * pool, + struct _ssm_size_class * sc, + struct _ssm_shard * shard, + struct ssm_pk_buff * blk, + size_t len, + uint8_t ** ptr, + struct ssm_pk_buff ** spb) +{ + STORE(&blk->refcount, 1); + blk->allocator_pid = getpid(); + blk->size = (uint32_t) (sc->object_size - + sizeof(struct ssm_pk_buff)); + blk->pk_head = SSM_PK_BUFF_HEADSPACE; + blk->pk_tail = blk->pk_head + (uint32_t) len; + blk->off = (uint32_t) PTR_TO_OFFSET(pool->pool_base, blk); + + pthread_mutex_unlock(&shard->mtx); + + *spb = blk; + if (ptr != NULL) + *ptr = blk->data + blk->pk_head; + + return blk->off; +} + +/* Non-blocking allocation from size class */ +static ssize_t alloc_from_sc(struct ssm_pool * pool, + int idx, + size_t len, + uint8_t ** ptr, + struct ssm_pk_buff ** spb) +{ + struct _ssm_size_class * sc; + struct ssm_pk_buff * blk; + int local; + int s; + + assert(pool != NULL); + assert(idx >= 0 && idx < SSM_POOL_MAX_CLASSES); + assert(spb != NULL); + + sc = &pool->hdr->size_classes[idx]; + local = GET_SHARD_FOR_PID(getpid()); + + for (s = 0; s < SSM_POOL_SHARDS; s++) { + struct _ssm_shard * shard; + int idx; + + idx = (local + s) % SSM_POOL_SHARDS; + shard = &sc->shards[idx]; + + blk = try_alloc_from_shard(shard, pool->pool_base); + if (blk != NULL) + return init_block(pool, sc, shard, blk, len, ptr, spb); + } + + return -EAGAIN; +} + +/* Blocking allocation from size class */ +static ssize_t alloc_from_sc_b(struct ssm_pool * pool, + int idx, + size_t len, + uint8_t ** ptr, + struct ssm_pk_buff ** spb, + const struct timespec * abstime) +{ + struct _ssm_size_class * sc; + struct _ssm_shard * shard; + struct ssm_pk_buff * blk = NULL; + int local; + int s; + int ret = 0; + + assert(pool != NULL); + assert(idx >= 0 && idx < SSM_POOL_MAX_CLASSES); + assert(spb != NULL); + + sc = &pool->hdr->size_classes[idx]; + local = GET_SHARD_FOR_PID(getpid()); + + while (blk == NULL && ret != ETIMEDOUT) { + /* Try non-blocking allocation from any shard */ + for (s = 0; s < SSM_POOL_SHARDS && blk == NULL; s++) { + shard = &sc->shards[(local + s) % SSM_POOL_SHARDS]; + blk = try_alloc_from_shard(shard, pool->pool_base); + } + + if (blk != NULL) + break; + + /* Nothing available, wait for signal */ + shard = &sc->shards[local]; + robust_mutex_lock(&shard->mtx); + ret = robust_wait(&shard->cond, &shard->mtx, abstime); + pthread_mutex_unlock(&shard->mtx); + } + + if (ret == ETIMEDOUT) + return -ETIMEDOUT; + + return init_block(pool, sc, shard, blk, len, ptr, spb); +} + +/* Generate pool filename: uid=0 for GSPP, uid>0 for PUP */ +static char * pool_filename(uid_t uid) +{ + char base[64]; + + if (IS_GSPP(uid)) + snprintf(base, sizeof(base), "%s", SSM_GSPP_NAME); + else + snprintf(base, sizeof(base), SSM_PUP_NAME_FMT, (int) uid); + + return strdup(base); +} + +void ssm_pool_close(struct ssm_pool * pool) +{ + size_t file_size; + + assert(pool != NULL); + + file_size = GET_POOL_FILE_SIZE(pool->uid); + + munmap(pool->shm_base, file_size); + free(pool); +} + +void ssm_pool_destroy(struct ssm_pool * pool) +{ + char * fn; + + assert(pool != NULL); + + if (getpid() != pool->hdr->pid && kill(pool->hdr->pid, 0) == 0) { + ssm_pool_close(pool); + return; + } + + fn = pool_filename(pool->uid); + if (fn == NULL) { + ssm_pool_close(pool); + return; + } + + ssm_pool_close(pool); + + shm_unlink(fn); + free(fn); +} + +#define MM_FLAGS (PROT_READ | PROT_WRITE) +static struct ssm_pool * __pool_create(const char * name, + int flags, + uid_t uid, + gid_t gid, + mode_t mode) +{ + struct ssm_pool * pool; + int fd; + uint8_t * shm_base; + size_t file_size; + size_t total_size; + + file_size = GET_POOL_FILE_SIZE(uid); + total_size = GET_POOL_TOTAL_SIZE(uid); + + pool = malloc(sizeof(*pool)); + if (pool == NULL) + goto fail_pool; + + fd = shm_open(name, flags, mode); + if (fd == -1) + goto fail_open; + + if (flags & O_CREAT) { + if (ftruncate(fd, (off_t) file_size) < 0) + goto fail_truncate; + if (NEEDS_CHOWN(uid, gid) && fchown(fd, uid, gid) < 0) + goto fail_truncate; + } + + shm_base = mmap(NULL, file_size, MM_FLAGS, MAP_SHARED, fd, 0); + if (shm_base == MAP_FAILED) + goto fail_truncate; + + pool->shm_base = shm_base; + pool->pool_base = shm_base; + pool->hdr = (struct _ssm_pool_hdr *) (shm_base + total_size); + pool->uid = uid; + pool->total_size = total_size; + + if (flags & O_CREAT) + pool->hdr->mapped_addr = shm_base; + + close(fd); + + return pool; + + fail_truncate: + close(fd); + if (flags & O_CREAT) + shm_unlink(name); + fail_open: + free(pool); + fail_pool: + return NULL; +} + +struct ssm_pool * ssm_pool_create(uid_t uid, + gid_t gid) +{ + struct ssm_pool * pool; + char * fn; + mode_t mask; + mode_t mode; + pthread_mutexattr_t mattr; + pthread_condattr_t cattr; + + fn = pool_filename(uid); + if (fn == NULL) + goto fail_fn; + + mode = IS_GSPP(uid) ? 0660 : 0600; + mask = umask(0); + + pool = __pool_create(fn, O_CREAT | O_EXCL | O_RDWR, uid, gid, mode); + + umask(mask); + + if (pool == NULL) + goto fail_pool; + + if (pthread_mutexattr_init(&mattr)) + goto fail_mattr; + + pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); +#ifdef HAVE_ROBUST_MUTEX + pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); +#endif + if (pthread_mutex_init(&pool->hdr->mtx, &mattr)) + goto fail_mutex; + + if (pthread_condattr_init(&cattr)) + goto fail_cattr; + + pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); +#ifndef __APPLE__ + pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); +#endif + if (pthread_cond_init(&pool->hdr->healthy, &cattr)) + goto fail_healthy; + + pool->hdr->pid = getpid(); + STORE(&pool->hdr->initialized, 0); + + init_size_classes(pool); + + pthread_mutexattr_destroy(&mattr); + pthread_condattr_destroy(&cattr); + free(fn); + + return pool; + + fail_healthy: + pthread_condattr_destroy(&cattr); + fail_cattr: + pthread_mutex_destroy(&pool->hdr->mtx); + fail_mutex: + pthread_mutexattr_destroy(&mattr); + fail_mattr: + ssm_pool_close(pool); + shm_unlink(fn); + fail_pool: + free(fn); + fail_fn: + return NULL; +} + +struct ssm_pool * ssm_pool_open(uid_t uid) +{ + struct ssm_pool * pool; + char * fn; + + fn = pool_filename(uid); + if (fn == NULL) + return NULL; + + pool = __pool_create(fn, O_RDWR, uid, 0, 0); + if (pool != NULL) + init_size_classes(pool); + + free(fn); + + return pool; +} + +void ssm_pool_gspp_purge(void) +{ + char * fn; + + fn = pool_filename(SSM_GSPP_UID); + if (fn == NULL) + return; + + shm_unlink(fn); + free(fn); +} + +int ssm_pool_mlock(struct ssm_pool * pool) +{ + size_t file_size; + + assert(pool != NULL); + + file_size = GET_POOL_FILE_SIZE(pool->uid); + + return mlock(pool->shm_base, file_size); +} + +ssize_t ssm_pool_alloc(struct ssm_pool * pool, + size_t count, + uint8_t ** ptr, + struct ssm_pk_buff ** spb) +{ + int idx; + + assert(pool != NULL); + assert(spb != NULL); + + idx = select_size_class(pool->hdr, count); + if (idx >= 0) + return alloc_from_sc(pool, idx, count, ptr, spb); + + return -EMSGSIZE; +} + +ssize_t ssm_pool_alloc_b(struct ssm_pool * pool, + size_t count, + uint8_t ** ptr, + struct ssm_pk_buff ** spb, + const struct timespec * abstime) +{ + int idx; + + assert(pool != NULL); + assert(spb != NULL); + + idx = select_size_class(pool->hdr, count); + if (idx >= 0) + return alloc_from_sc_b(pool, idx, count, ptr, spb, abstime); + + return -EMSGSIZE; +} + +ssize_t ssm_pool_read(uint8_t ** dst, + struct ssm_pool * pool, + size_t off) +{ + struct ssm_pk_buff * blk; + + assert(dst != NULL); + assert(pool != NULL); + + blk = OFFSET_TO_PTR(pool->pool_base, off); + if (blk == NULL) + return -EINVAL; + + *dst = blk->data + blk->pk_head; + + return (ssize_t) (blk->pk_tail - blk->pk_head); +} + +struct ssm_pk_buff * ssm_pool_get(struct ssm_pool * pool, + size_t off) +{ + struct ssm_pk_buff * blk; + + assert(pool != NULL); + + if (off == 0 || off >= pool->total_size) + return NULL; + + blk = OFFSET_TO_PTR(pool->pool_base, off); + if (blk == NULL) + return NULL; + + if (LOAD(&blk->refcount) == 0) + return NULL; + + return blk; +} + +int ssm_pool_remove(struct ssm_pool * pool, + size_t off) +{ + struct ssm_pk_buff * blk; + struct _ssm_size_class * sc; + struct _ssm_shard * shard; + int sc_idx; + int shard_idx; + uint16_t old_ref; + + assert(pool != NULL); + + if (off == 0 || off >= pool->total_size) + return -EINVAL; + + blk = OFFSET_TO_PTR(pool->pool_base, off); + if (blk == NULL) + return -EINVAL; + + sc_idx = find_size_class_for_offset(pool, off); + if (sc_idx < 0) + return -EINVAL; + + sc = &pool->hdr->size_classes[sc_idx]; + + /* Free to allocator's shard (lazy distribution in action) */ + shard_idx = GET_SHARD_FOR_PID(blk->allocator_pid); + shard = &sc->shards[shard_idx]; + + robust_mutex_lock(&shard->mtx); + + old_ref = FETCH_SUB(&blk->refcount, 1); + if (old_ref > 1) { + /* Still referenced */ + pthread_mutex_unlock(&shard->mtx); + return 0; + } + + blk->allocator_pid = 0; +#ifdef CONFIG_OUROBOROS_DEBUG + if (old_ref == 0) { + /* Underflow - double free attempt */ + pthread_mutex_unlock(&shard->mtx); + abort(); + } + + /* Poison fields to detect use-after-free */ + blk->pk_head = 0xDEAD; + blk->pk_tail = 0xBEEF; +#endif + list_add_head(&shard->free_list, blk, pool->pool_base); + FETCH_ADD(&shard->free_count, 1); + + pthread_cond_signal(&shard->cond); + + pthread_mutex_unlock(&shard->mtx); + + return 0; +} + +size_t ssm_pk_buff_get_off(const struct ssm_pk_buff * spb) +{ + assert(spb != NULL); + + return spb->off; +} + +uint8_t * ssm_pk_buff_head(const struct ssm_pk_buff * spb) +{ + assert(spb != NULL); + + return (uint8_t *) spb->data + spb->pk_head; +} + +uint8_t * ssm_pk_buff_tail(const struct ssm_pk_buff * spb) +{ + assert(spb != NULL); + + return (uint8_t *) spb->data + spb->pk_tail; +} + +size_t ssm_pk_buff_len(const struct ssm_pk_buff * spb) +{ + assert(spb != NULL); + + return spb->pk_tail - spb->pk_head; +} + +uint8_t * ssm_pk_buff_push(struct ssm_pk_buff * spb, + size_t size) +{ + assert(spb != NULL); + + if (spb->pk_head < size) + return NULL; + + spb->pk_head -= size; + + return spb->data + spb->pk_head; +} + +uint8_t * ssm_pk_buff_push_tail(struct ssm_pk_buff * spb, + size_t size) +{ + uint8_t * buf; + + assert(spb != NULL); + + if (spb->pk_tail + size >= spb->size) + return NULL; + + buf = spb->data + spb->pk_tail; + + spb->pk_tail += size; + + return buf; +} + +uint8_t * ssm_pk_buff_pop(struct ssm_pk_buff * spb, + size_t size) +{ + uint8_t * buf; + + assert(spb != NULL); + assert(!(size > spb->pk_tail - spb->pk_head)); + + buf = spb->data + spb->pk_head; + + spb->pk_head += size; + + return buf; +} + +uint8_t * ssm_pk_buff_pop_tail(struct ssm_pk_buff * spb, + size_t size) +{ + assert(spb != NULL); + assert(!(size > spb->pk_tail - spb->pk_head)); + + spb->pk_tail -= size; + + return spb->data + spb->pk_tail; +} + +void ssm_pk_buff_truncate(struct ssm_pk_buff * spb, + size_t len) +{ + assert(spb != NULL); + assert(len <= spb->size); + + spb->pk_tail = spb->pk_head + len; +} + +int ssm_pk_buff_wait_ack(struct ssm_pk_buff * spb) +{ + assert(spb != NULL); + + FETCH_ADD(&spb->refcount, 1); + + return 0; +} + +int ssm_pk_buff_ack(struct ssm_pk_buff * spb) +{ + assert(spb != NULL); + + FETCH_SUB(&spb->refcount, 1); + + return 0; +} diff --git a/src/lib/ssm/rbuff.c b/src/lib/ssm/rbuff.c new file mode 100644 index 00000000..c149c306 --- /dev/null +++ b/src/lib/ssm/rbuff.c @@ -0,0 +1,493 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Ring buffer implementations for incoming packets + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include "config.h" +#include "ssm.h" + +#include <ouroboros/ssm_rbuff.h> +#include <ouroboros/lockfile.h> +#include <ouroboros/errno.h> +#include <ouroboros/fccntl.h> +#include <ouroboros/pthread.h> +#include <ouroboros/time.h> + +#include <assert.h> +#include <fcntl.h> +#include <signal.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <sys/mman.h> +#include <sys/stat.h> + +#define FN_MAX_CHARS 255 + +#define SSM_RBUFF_FILESIZE ((SSM_RBUFF_SIZE) * sizeof(ssize_t) \ + + 3 * sizeof(size_t) \ + + sizeof(pthread_mutex_t) \ + + 2 * sizeof(pthread_cond_t)) + +#define MODB(x) ((x) & (SSM_RBUFF_SIZE - 1)) + +#define LOAD_RELAXED(ptr) (__atomic_load_n(ptr, __ATOMIC_RELAXED)) +#define LOAD_ACQUIRE(ptr) (__atomic_load_n(ptr, __ATOMIC_ACQUIRE)) +#define STORE_RELEASE(ptr, val) \ + (__atomic_store_n(ptr, val, __ATOMIC_RELEASE)) + +#define HEAD(rb) (rb->shm_base[LOAD_RELAXED(rb->head)]) +#define TAIL(rb) (rb->shm_base[LOAD_RELAXED(rb->tail)]) +#define HEAD_IDX(rb) (LOAD_ACQUIRE(rb->head)) +#define TAIL_IDX(rb) (LOAD_ACQUIRE(rb->tail)) +#define ADVANCE_HEAD(rb) \ + (STORE_RELEASE(rb->head, MODB(LOAD_RELAXED(rb->head) + 1))) +#define ADVANCE_TAIL(rb) \ + (STORE_RELEASE(rb->tail, MODB(LOAD_RELAXED(rb->tail) + 1))) +#define QUEUED(rb) (MODB(HEAD_IDX(rb) - TAIL_IDX(rb))) +#define IS_FULL(rb) (QUEUED(rb) == (SSM_RBUFF_SIZE - 1)) +#define IS_EMPTY(rb) (HEAD_IDX(rb) == TAIL_IDX(rb)) + +struct ssm_rbuff { + ssize_t * shm_base; /* start of shared memory */ + size_t * head; /* start of ringbuffer */ + size_t * tail; + size_t * acl; /* access control */ + pthread_mutex_t * mtx; /* lock for cond vars only */ + pthread_cond_t * add; /* signal when new data */ + pthread_cond_t * del; /* signal when data removed */ + pid_t pid; /* pid of the owner */ + int flow_id; /* flow_id of the flow */ + size_t n_users; /* in-flight users */ +}; + +#define MM_FLAGS (PROT_READ | PROT_WRITE) + +static struct ssm_rbuff * rbuff_create(pid_t pid, + int flow_id, + int flags) +{ + struct ssm_rbuff * rb; + int fd; + ssize_t * shm_base; + char fn[FN_MAX_CHARS]; + + sprintf(fn, SSM_RBUFF_PREFIX "%d.%d", pid, flow_id); + + rb = malloc(sizeof(*rb)); + if (rb == NULL) + goto fail_malloc; + + fd = shm_open(fn, flags, 0666); + if (fd == -1) + goto fail_open; + + if ((flags & O_CREAT) && ftruncate(fd, SSM_RBUFF_FILESIZE) < 0) + goto fail_truncate; + + shm_base = mmap(NULL, SSM_RBUFF_FILESIZE, MM_FLAGS, MAP_SHARED, fd, 0); + + close(fd); + + rb->shm_base = shm_base; + rb->head = (size_t *) (rb->shm_base + (SSM_RBUFF_SIZE)); + rb->tail = (size_t *) (rb->head + 1); + rb->acl = (size_t *) (rb->tail + 1); + rb->mtx = (pthread_mutex_t *) (rb->acl + 1); + rb->add = (pthread_cond_t *) (rb->mtx + 1); + rb->del = rb->add + 1; + rb->pid = pid; + rb->flow_id = flow_id; + rb->n_users = 0; + + return rb; + + fail_truncate: + close(fd); + if (flags & O_CREAT) + shm_unlink(fn); + fail_open: + free(rb); + fail_malloc: + return NULL; +} + +static void rbuff_destroy(struct ssm_rbuff * rb) +{ + munmap(rb->shm_base, SSM_RBUFF_FILESIZE); + + free(rb); +} + +struct ssm_rbuff * ssm_rbuff_create(pid_t pid, + int flow_id) +{ + struct ssm_rbuff * rb; + pthread_mutexattr_t mattr; + pthread_condattr_t cattr; + mode_t mask; + + mask = umask(0); + + rb = rbuff_create(pid, flow_id, O_CREAT | O_EXCL | O_RDWR); + + umask(mask); + + if (rb == NULL) + goto fail_rb; + + if (pthread_mutexattr_init(&mattr)) + goto fail_mattr; + + pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); +#ifdef HAVE_ROBUST_MUTEX + pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); +#endif + if (pthread_mutex_init(rb->mtx, &mattr)) + goto fail_mutex; + + if (pthread_condattr_init(&cattr)) + goto fail_cattr; + + pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); +#ifndef __APPLE__ + pthread_condattr_setclock(&cattr, PTHREAD_COND_CLOCK); +#endif + if (pthread_cond_init(rb->add, &cattr)) + goto fail_add; + + if (pthread_cond_init(rb->del, &cattr)) + goto fail_del; + + *rb->acl = ACL_RDWR; + *rb->head = 0; + *rb->tail = 0; + + rb->pid = pid; + rb->flow_id = flow_id; + + pthread_mutexattr_destroy(&mattr); + pthread_condattr_destroy(&cattr); + + return rb; + + fail_del: + pthread_cond_destroy(rb->add); + fail_add: + pthread_condattr_destroy(&cattr); + fail_cattr: + pthread_mutex_destroy(rb->mtx); + fail_mutex: + pthread_mutexattr_destroy(&mattr); + fail_mattr: + ssm_rbuff_destroy(rb); + fail_rb: + return NULL; +} + +void ssm_rbuff_destroy(struct ssm_rbuff * rb) +{ + char fn[FN_MAX_CHARS]; + + assert(rb != NULL); + + sprintf(fn, SSM_RBUFF_PREFIX "%d.%d", rb->pid, rb->flow_id); + + ssm_rbuff_close(rb); + + shm_unlink(fn); +} + +struct ssm_rbuff * ssm_rbuff_open(pid_t pid, + int flow_id) +{ + return rbuff_create(pid, flow_id, O_RDWR); +} + +void ssm_rbuff_close(struct ssm_rbuff * rb) +{ + assert(rb); + + /* + * Caller must set ACL_FLOWDOWN first; if a user becomes + * cancellable, push a cleanup that decrements n_users. + */ + while (__atomic_load_n(&rb->n_users, __ATOMIC_SEQ_CST) > 0) { + struct timespec tic = { 0, 100000 }; + nanosleep(&tic, NULL); + } + + rbuff_destroy(rb); +} + +int ssm_rbuff_write(struct ssm_rbuff * rb, + size_t off) +{ + size_t acl; + bool was_empty; + int ret = 0; + + assert(rb != NULL); + + __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + + acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); + if (acl != ACL_RDWR) { + if (acl & ACL_FLOWDOWN) { + ret = -EFLOWDOWN; + goto fail_acl; + } + if (acl & ACL_RDONLY) { + ret = -ENOTALLOC; + goto fail_acl; + } + } + + robust_mutex_lock(rb->mtx); + + if (IS_FULL(rb)) { + ret = -EAGAIN; + goto fail_mutex; + } + + was_empty = IS_EMPTY(rb); + + HEAD(rb) = (ssize_t) off; + ADVANCE_HEAD(rb); + + if (was_empty) + pthread_cond_broadcast(rb->add); + + pthread_mutex_unlock(rb->mtx); + + __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + return 0; + + fail_mutex: + pthread_mutex_unlock(rb->mtx); + fail_acl: + __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + return ret; +} + +int ssm_rbuff_write_b(struct ssm_rbuff * rb, + size_t off, + const struct timespec * abstime) +{ + size_t acl; + int ret = 0; + bool was_empty; + + assert(rb != NULL); + + __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + + acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); + if (acl != ACL_RDWR) { + if (acl & ACL_FLOWDOWN) { + ret = -EFLOWDOWN; + goto fail_acl; + } + if (acl & ACL_RDONLY) { + ret = -ENOTALLOC; + goto fail_acl; + } + } + + robust_mutex_lock(rb->mtx); + + pthread_cleanup_push(__cleanup_mutex_unlock, rb->mtx); + + while (IS_FULL(rb) && ret != -ETIMEDOUT) { + acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); + if (acl & ACL_FLOWDOWN) { + ret = -EFLOWDOWN; + break; + } + ret = -robust_wait(rb->del, rb->mtx, abstime); + } + + pthread_cleanup_pop(false); + + if (ret != -ETIMEDOUT && ret != -EFLOWDOWN) { + was_empty = IS_EMPTY(rb); + HEAD(rb) = (ssize_t) off; + ADVANCE_HEAD(rb); + if (was_empty) + pthread_cond_broadcast(rb->add); + } + + pthread_mutex_unlock(rb->mtx); + + fail_acl: + __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + return ret; +} + +static int check_rb_acl(struct ssm_rbuff * rb) +{ + size_t acl; + + assert(rb != NULL); + + acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); + + if (acl & ACL_FLOWDOWN) + return -EFLOWDOWN; + + if (acl & ACL_FLOWPEER) + return -EFLOWPEER; + + return -EAGAIN; +} + +ssize_t ssm_rbuff_read(struct ssm_rbuff * rb) +{ + ssize_t ret; + + assert(rb != NULL); + + __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + + if (IS_EMPTY(rb)) { + ret = check_rb_acl(rb); + goto out; + } + + robust_mutex_lock(rb->mtx); + + if (IS_EMPTY(rb)) { + pthread_mutex_unlock(rb->mtx); + ret = check_rb_acl(rb); + goto out; + } + + ret = TAIL(rb); + ADVANCE_TAIL(rb); + + pthread_cond_broadcast(rb->del); + + pthread_mutex_unlock(rb->mtx); + + out: + __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + return ret; +} + +ssize_t ssm_rbuff_read_b(struct ssm_rbuff * rb, + const struct timespec * abstime) +{ + ssize_t idx = -1; + size_t acl; + + assert(rb != NULL); + + __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + + acl = __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); + if (IS_EMPTY(rb) && (acl & ACL_FLOWDOWN)) { + idx = -EFLOWDOWN; + goto out; + } + + robust_mutex_lock(rb->mtx); + + pthread_cleanup_push(__cleanup_mutex_unlock, rb->mtx); + + while (IS_EMPTY(rb) && + idx != -ETIMEDOUT && + check_rb_acl(rb) == -EAGAIN) { + idx = -robust_wait(rb->add, rb->mtx, abstime); + } + + pthread_cleanup_pop(false); + + if (!IS_EMPTY(rb)) { + idx = TAIL(rb); + ADVANCE_TAIL(rb); + pthread_cond_broadcast(rb->del); + } else if (idx != -ETIMEDOUT) { + idx = check_rb_acl(rb); + } + + pthread_mutex_unlock(rb->mtx); + + assert(idx != -EAGAIN); + + out: + __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); + return idx; +} + +void ssm_rbuff_set_acl(struct ssm_rbuff * rb, + uint32_t flags) +{ + assert(rb != NULL); + + robust_mutex_lock(rb->mtx); + __atomic_store_n(rb->acl, (size_t) flags, __ATOMIC_SEQ_CST); + pthread_cond_broadcast(rb->add); + pthread_cond_broadcast(rb->del); + pthread_mutex_unlock(rb->mtx); +} + +uint32_t ssm_rbuff_get_acl(struct ssm_rbuff * rb) +{ + assert(rb != NULL); + + return (uint32_t) __atomic_load_n(rb->acl, __ATOMIC_SEQ_CST); +} + +void ssm_rbuff_fini(struct ssm_rbuff * rb) +{ + assert(rb != NULL); + + __atomic_fetch_add(&rb->n_users, 1, __ATOMIC_SEQ_CST); + + robust_mutex_lock(rb->mtx); + + pthread_cleanup_push(__cleanup_mutex_unlock, rb->mtx); + + while (!IS_EMPTY(rb)) + robust_wait(rb->del, rb->mtx, NULL); + + pthread_cleanup_pop(true); + + __atomic_fetch_sub(&rb->n_users, 1, __ATOMIC_SEQ_CST); +} + +size_t ssm_rbuff_queued(struct ssm_rbuff * rb) +{ + assert(rb != NULL); + + return QUEUED(rb); +} + +int ssm_rbuff_mlock(struct ssm_rbuff * rb) +{ + assert(rb != NULL); + + return mlock(rb->shm_base, SSM_RBUFF_FILESIZE); +} diff --git a/src/lib/ssm/ssm.h.in b/src/lib/ssm/ssm.h.in new file mode 100644 index 00000000..b86327a1 --- /dev/null +++ b/src/lib/ssm/ssm.h.in @@ -0,0 +1,188 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Secure Shared Memory configuration + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#ifndef OUROBOROS_LIB_SSM_H +#define OUROBOROS_LIB_SSM_H + +#include <stddef.h> +#include <stdint.h> +#include <stdatomic.h> +#include <sys/types.h> + +/* Pool naming configuration */ +#define SSM_PREFIX "@SSM_PREFIX@" +#define SSM_GSPP_NAME "@SSM_GSPP_NAME@" +#define SSM_PUP_NAME_FMT "@SSM_PUP_NAME_FMT@" +#define SSM_GSPP_UID 0 + +/* Legacy SSM constants */ +#define SSM_RBUFF_PREFIX "@SSM_RBUFF_PREFIX@" +#define SSM_FLOW_SET_PREFIX "@SSM_FLOW_SET_PREFIX@" +#define SSM_POOL_NAME "@SSM_POOL_NAME@" +#define SSM_RBUFF_SIZE @SSM_RBUFF_SIZE@ + +/* Packet buffer space reservation */ +#define SSM_PK_BUFF_HEADSPACE @SSM_PK_BUFF_HEADSPACE@ +#define SSM_PK_BUFF_TAILSPACE @SSM_PK_BUFF_TAILSPACE@ + +/* Global Shared Packet Pool (GSPP) - for privileged processes */ +#define SSM_GSPP_256_BLOCKS @SSM_GSPP_256_BLOCKS@ +#define SSM_GSPP_512_BLOCKS @SSM_GSPP_512_BLOCKS@ +#define SSM_GSPP_1K_BLOCKS @SSM_GSPP_1K_BLOCKS@ +#define SSM_GSPP_2K_BLOCKS @SSM_GSPP_2K_BLOCKS@ +#define SSM_GSPP_4K_BLOCKS @SSM_GSPP_4K_BLOCKS@ +#define SSM_GSPP_16K_BLOCKS @SSM_GSPP_16K_BLOCKS@ +#define SSM_GSPP_64K_BLOCKS @SSM_GSPP_64K_BLOCKS@ +#define SSM_GSPP_256K_BLOCKS @SSM_GSPP_256K_BLOCKS@ +#define SSM_GSPP_1M_BLOCKS @SSM_GSPP_1M_BLOCKS@ +#define SSM_GSPP_TOTAL_SIZE @SSM_GSPP_TOTAL_SIZE@ + +/* Per-User Pool (PUP) - for unprivileged applications */ +#define SSM_PUP_256_BLOCKS @SSM_PUP_256_BLOCKS@ +#define SSM_PUP_512_BLOCKS @SSM_PUP_512_BLOCKS@ +#define SSM_PUP_1K_BLOCKS @SSM_PUP_1K_BLOCKS@ +#define SSM_PUP_2K_BLOCKS @SSM_PUP_2K_BLOCKS@ +#define SSM_PUP_4K_BLOCKS @SSM_PUP_4K_BLOCKS@ +#define SSM_PUP_16K_BLOCKS @SSM_PUP_16K_BLOCKS@ +#define SSM_PUP_64K_BLOCKS @SSM_PUP_64K_BLOCKS@ +#define SSM_PUP_256K_BLOCKS @SSM_PUP_256K_BLOCKS@ +#define SSM_PUP_1M_BLOCKS @SSM_PUP_1M_BLOCKS@ +#define SSM_PUP_TOTAL_SIZE @SSM_PUP_TOTAL_SIZE@ + +/* Legacy pool blocks (same as GSPP for compatibility) */ +#define SSM_POOL_256_BLOCKS @SSM_POOL_256_BLOCKS@ +#define SSM_POOL_512_BLOCKS @SSM_POOL_512_BLOCKS@ +#define SSM_POOL_1K_BLOCKS @SSM_POOL_1K_BLOCKS@ +#define SSM_POOL_2K_BLOCKS @SSM_POOL_2K_BLOCKS@ +#define SSM_POOL_4K_BLOCKS @SSM_POOL_4K_BLOCKS@ +#define SSM_POOL_16K_BLOCKS @SSM_POOL_16K_BLOCKS@ +#define SSM_POOL_64K_BLOCKS @SSM_POOL_64K_BLOCKS@ +#define SSM_POOL_256K_BLOCKS @SSM_POOL_256K_BLOCKS@ +#define SSM_POOL_1M_BLOCKS @SSM_POOL_1M_BLOCKS@ +#define SSM_POOL_TOTAL_SIZE @SSM_POOL_TOTAL_SIZE@ + +/* Size class configuration */ +#define SSM_POOL_MAX_CLASSES 9 +#define SSM_POOL_SHARDS @SSM_POOL_SHARDS@ + +/* Internal structures - exposed for testing */ +#ifdef __cplusplus +extern "C" { +#endif + +#include <errno.h> +#include <pthread.h> + +#include <ouroboros/pthread.h> + +static __inline__ void robust_mutex_lock(pthread_mutex_t * mtx) +{ +#ifndef HAVE_ROBUST_MUTEX + pthread_mutex_lock(mtx); +#else + if (pthread_mutex_lock(mtx) == EOWNERDEAD) + pthread_mutex_consistent(mtx); +#endif +} + +static __inline__ int robust_wait(pthread_cond_t * cond, + pthread_mutex_t * mtx, + const struct timespec * abstime) +{ + int ret = __timedwait(cond, mtx, abstime); +#ifdef HAVE_ROBUST_MUTEX + if (ret == EOWNERDEAD) + pthread_mutex_consistent(mtx); +#endif + return ret; +} + +/* Packet buffer structure used by pool, rbuff, and tests */ +struct ssm_pk_buff { + uint32_t next_offset; /* List linkage (pool < 4GB) */ + uint16_t refcount; /* Reference count (app + rtx) */ + pid_t allocator_pid; /* For orphan detection */ + uint32_t size; /* Block size (max 1MB) */ + uint32_t pk_head; /* Head offset into data */ + uint32_t pk_tail; /* Tail offset into data */ + uint32_t off; /* Block offset in pool */ + uint8_t data[]; /* Packet data */ +}; + +/* Size class configuration table */ +struct ssm_size_class_cfg { + size_t size; + size_t blocks; +}; + +struct _ssm_list_head { + uint32_t head_offset; + uint32_t count; +}; + +struct _ssm_shard { + pthread_mutex_t mtx; + pthread_cond_t cond; + struct _ssm_list_head free_list; + size_t free_count; +}; + +struct _ssm_size_class { + struct _ssm_shard shards[SSM_POOL_SHARDS]; + size_t object_size; + size_t pool_start; + size_t pool_size; + size_t object_count; +}; + +struct _ssm_pool_hdr { + pthread_mutex_t mtx; + pthread_cond_t healthy; + pid_t pid; + uint32_t initialized; + void * mapped_addr; + struct _ssm_size_class size_classes[SSM_POOL_MAX_CLASSES]; +}; + +#define SSM_PK_BUFF_TOTALSPACE (SSM_PK_BUFF_HEADSPACE + SSM_PK_BUFF_TAILSPACE) +static __inline__ int select_size_class(struct _ssm_pool_hdr * hdr, + size_t len) +{ + size_t sz; + int i; + + sz = sizeof(struct ssm_pk_buff) + SSM_PK_BUFF_TOTALSPACE + len; + + for (i = 0; i < SSM_POOL_MAX_CLASSES; i++) { + struct _ssm_size_class * sc = &hdr->size_classes[i]; + if (sc->object_size > 0 && sz <= sc->object_size) + return i; + } + + return -1; +} + +#ifdef __cplusplus +} +#endif + +#endif /* OUROBOROS_LIB_SSM_H */ diff --git a/src/lib/ssm/tests/CMakeLists.txt b/src/lib/ssm/tests/CMakeLists.txt new file mode 100644 index 00000000..d622d41c --- /dev/null +++ b/src/lib/ssm/tests/CMakeLists.txt @@ -0,0 +1,21 @@ +get_filename_component(PARENT_PATH ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) +get_filename_component(PARENT_DIR ${PARENT_PATH} NAME) + +compute_test_prefix() + +create_test_sourcelist(${PARENT_DIR}_tests test_suite.c + # Add new tests here + pool_test.c + pool_sharding_test.c + rbuff_test.c + flow_set_test.c + ) + +add_executable(${PARENT_DIR}_test ${${PARENT_DIR}_tests}) + +disable_test_logging_for_target(${PARENT_DIR}_test) +target_link_libraries(${PARENT_DIR}_test ouroboros-common) + +add_dependencies(build_tests ${PARENT_DIR}_test) + +ouroboros_register_tests(TARGET ${PARENT_DIR}_test TESTS ${${PARENT_DIR}_tests}) diff --git a/src/lib/ssm/tests/flow_set_test.c b/src/lib/ssm/tests/flow_set_test.c new file mode 100644 index 00000000..1bb97b2f --- /dev/null +++ b/src/lib/ssm/tests/flow_set_test.c @@ -0,0 +1,255 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the SSM flow set + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#define _DEFAULT_SOURCE +#else +#define _POSIX_C_SOURCE 200112L +#endif + +#include "config.h" +#include "ssm.h" + +#include <test/test.h> +#include <ouroboros/ssm_flow_set.h> +#include <ouroboros/errno.h> +#include <ouroboros/time.h> + +#include <stdio.h> +#include <unistd.h> +#include <pthread.h> + +static int test_ssm_flow_set_create_destroy(void) +{ + struct ssm_flow_set * set; + pid_t pid; + + TEST_START(); + + pid = getpid(); + + set = ssm_flow_set_create(pid); + if (set == NULL) { + printf("Failed to create flow set.\n"); + goto fail; + } + + ssm_flow_set_destroy(set); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; +fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_flow_set_add_del_has(void) +{ + struct ssm_flow_set * set; + pid_t pid; + size_t idx = 0; + int flow_id = 42; + + TEST_START(); + + pid = getpid(); + + set = ssm_flow_set_create(pid); + if (set == NULL) { + printf("Failed to create flow set.\n"); + goto fail; + } + + if (ssm_flow_set_has(set, idx, flow_id)) { + printf("Flow should not be in set initially.\n"); + goto fail_destroy; + } + + if (ssm_flow_set_add(set, idx, flow_id) < 0) { + printf("Failed to add flow to set.\n"); + goto fail_destroy; + } + + if (!ssm_flow_set_has(set, idx, flow_id)) { + printf("Flow should be in set after add.\n"); + goto fail_destroy; + } + + /* Adding same flow again should fail */ + if (ssm_flow_set_add(set, idx, flow_id) != -EPERM) { + printf("Should not be able to add flow twice.\n"); + goto fail_destroy; + } + + ssm_flow_set_del(set, idx, flow_id); + + if (ssm_flow_set_has(set, idx, flow_id)) { + printf("Flow should not be in set after delete.\n"); + goto fail_destroy; + } + + ssm_flow_set_destroy(set); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; +fail_destroy: + ssm_flow_set_destroy(set); +fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_flow_set_zero(void) +{ + struct ssm_flow_set * set; + pid_t pid; + size_t idx = 0; + int flow_id1 = 10; + int flow_id2 = 20; + + TEST_START(); + + pid = getpid(); + + set = ssm_flow_set_create(pid); + if (set == NULL) { + printf("Failed to create flow set.\n"); + goto fail; + } + + if (ssm_flow_set_add(set, idx, flow_id1) < 0) { + printf("Failed to add flow1 to set.\n"); + goto fail_destroy; + } + + if (ssm_flow_set_add(set, idx, flow_id2) < 0) { + printf("Failed to add flow2 to set.\n"); + goto fail_destroy; + } + + ssm_flow_set_zero(set, idx); + + if (ssm_flow_set_has(set, idx, flow_id1)) { + printf("Flow1 should not be in set after zero.\n"); + goto fail_destroy; + } + + if (ssm_flow_set_has(set, idx, flow_id2)) { + printf("Flow2 should not be in set after zero.\n"); + goto fail_destroy; + } + + ssm_flow_set_destroy(set); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; +fail_destroy: + ssm_flow_set_destroy(set); +fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_flow_set_notify_wait(void) +{ + struct ssm_flow_set * set; + pid_t pid; + size_t idx = 0; + int flow_id = 100; + struct flowevent events[SSM_RBUFF_SIZE]; + struct timespec timeout; + ssize_t ret; + + TEST_START(); + + pid = getpid(); + + set = ssm_flow_set_create(pid); + if (set == NULL) { + printf("Failed to create flow set.\n"); + goto fail; + } + + if (ssm_flow_set_add(set, idx, flow_id) < 0) { + printf("Failed to add flow to set.\n"); + goto fail_destroy; + } + + /* Test immediate timeout when no events */ + clock_gettime(PTHREAD_COND_CLOCK, &timeout); + ret = ssm_flow_set_wait(set, idx, events, &timeout); + if (ret != -ETIMEDOUT) { + printf("Wait should timeout immediately when no events.\n"); + goto fail_destroy; + } + + /* Notify an event */ + ssm_flow_set_notify(set, flow_id, FLOW_PKT); + + /* Should be able to read the event immediately */ + clock_gettime(PTHREAD_COND_CLOCK, &timeout); + ts_add(&timeout, &timeout, &((struct timespec) {1, 0})); + + ret = ssm_flow_set_wait(set, idx, events, &timeout); + if (ret != 1) { + printf("Wait should return 1 event, got %zd.\n", ret); + goto fail_destroy; + } + + if (events[0].flow_id != flow_id) { + printf("Event flow_id mismatch: expected %d, got %d.\n", + flow_id, events[0].flow_id); + goto fail_destroy; + } + + if (events[0].event != FLOW_PKT) { + printf("Event type mismatch: expected %d, got %d.\n", + FLOW_PKT, events[0].event); + goto fail_destroy; + } + + ssm_flow_set_destroy(set); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; +fail_destroy: + ssm_flow_set_destroy(set); +fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int flow_set_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_ssm_flow_set_create_destroy(); + ret |= test_ssm_flow_set_add_del_has(); + ret |= test_ssm_flow_set_zero(); + ret |= test_ssm_flow_set_notify_wait(); + + return ret; +} diff --git a/src/lib/ssm/tests/pool_sharding_test.c b/src/lib/ssm/tests/pool_sharding_test.c new file mode 100644 index 00000000..ec464a92 --- /dev/null +++ b/src/lib/ssm/tests/pool_sharding_test.c @@ -0,0 +1,474 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the SSM pool sharding with fallback + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#define _DEFAULT_SOURCE +#else +#define _POSIX_C_SOURCE 200112L +#endif + +#include "config.h" +#include "ssm.h" + +#include <test/test.h> +#include <ouroboros/ssm_pool.h> + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <stdbool.h> +#include <sys/wait.h> +#include <sys/types.h> +#include <signal.h> + +#define TEST_SIZE 256 + +/* Helper to get pool header for inspection */ +static struct _ssm_pool_hdr * get_pool_hdr(struct ssm_pool * pool) +{ + /* ssm_pool is opaque, but we know its layout: + * uint8_t * shm_base + * struct _ssm_pool_hdr * hdr + * void * pool_base + */ + struct _ssm_pool_hdr ** hdr_ptr = + (struct _ssm_pool_hdr **)((uint8_t *)pool + sizeof(void *)); + + return *hdr_ptr; +} + +static int test_lazy_distribution(void) +{ + struct ssm_pool * pool; + struct _ssm_pool_hdr * hdr; + struct _ssm_size_class * sc; + int i; + int sc_idx; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail; + } + + hdr = get_pool_hdr(pool); + if (hdr == NULL) { + printf("Failed to get pool header.\n"); + goto fail_pool; + } + + /* Inspect the class that TEST_SIZE allocations will use */ + sc_idx = select_size_class(hdr, TEST_SIZE); + if (sc_idx < 0) { + printf("No size class fits TEST_SIZE=%d.\n", TEST_SIZE); + for (i = 0; i < SSM_POOL_MAX_CLASSES; i++) { + printf(" Class %d: object_size=%zu count=%zu\n", i, + hdr->size_classes[i].object_size, + hdr->size_classes[i].object_count); + } + goto fail_pool; + } + + sc = &hdr->size_classes[sc_idx]; + + /* Verify all blocks start in shard 0 */ + if (sc->shards[0].free_count == 0) { + printf("Shard 0 should have all blocks initially.\n"); + goto fail_pool; + } + + /* Verify other shards are empty */ + for (i = 1; i < SSM_POOL_SHARDS; i++) { + if (sc->shards[i].free_count != 0) { + printf("Shard %d should be empty, has %zu.\n", + i, sc->shards[i].free_count); + goto fail_pool; + } + } + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_pool: + ssm_pool_destroy(pool); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_shard_migration(void) +{ + struct ssm_pool * pool; + struct _ssm_pool_hdr * hdr; + struct _ssm_size_class * sc; + struct ssm_pk_buff * spb; + uint8_t * ptr; + ssize_t off; + int shard_idx; + int sc_idx; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail; + } + + hdr = get_pool_hdr(pool); + + /* Inspect the class that TEST_SIZE allocations will use */ + sc_idx = select_size_class(hdr, TEST_SIZE); + if (sc_idx < 0) { + printf("No size class fits TEST_SIZE=%d.\n", TEST_SIZE); + goto fail_pool; + } + + sc = &hdr->size_classes[sc_idx]; + + /* Allocate from this process */ + off = ssm_pool_alloc(pool, TEST_SIZE, &ptr, &spb); + if (off < 0) { + printf("Allocation failed: %zd.\n", off); + goto fail_pool; + } + + /* Free it - should go to this process's shard */ + shard_idx = getpid() % SSM_POOL_SHARDS; + if (ssm_pool_remove(pool, off) != 0) { + printf("Remove failed.\n"); + goto fail_pool; + } + + /* Verify block migrated away from shard 0 or in allocator's shard */ + if (sc->shards[shard_idx].free_count == 0 && + sc->shards[0].free_count == 0) { + printf("Block should have been freed to a shard.\n"); + goto fail_pool; + } + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_pool: + ssm_pool_destroy(pool); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_fallback_stealing(void) +{ + struct ssm_pool * pool; + struct _ssm_pool_hdr * hdr; + struct _ssm_size_class * sc; + struct ssm_pk_buff ** spbs; + uint8_t ** ptrs; + size_t total_blocks; + size_t total_free; + size_t i; + int sc_idx; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail; + } + + hdr = get_pool_hdr(pool); + + /* Inspect the class that TEST_SIZE allocations will use */ + sc_idx = select_size_class(hdr, TEST_SIZE); + if (sc_idx < 0) { + printf("No size class fits TEST_SIZE=%d.\n", TEST_SIZE); + goto fail_pool; + } + + sc = &hdr->size_classes[sc_idx]; + total_blocks = sc->object_count; + + spbs = malloc(total_blocks * sizeof(struct ssm_pk_buff *)); + ptrs = malloc(total_blocks * sizeof(uint8_t *)); + if (spbs == NULL || ptrs == NULL) { + printf("Failed to allocate test arrays.\n"); + free(spbs); + free(ptrs); + goto fail_pool; + } + + /* Allocate half the blocks from single process */ + for (i = 0; i < total_blocks / 2; i++) { + ssize_t off = ssm_pool_alloc(pool, TEST_SIZE, + &ptrs[i], &spbs[i]); + if (off < 0) { + printf("Allocation %zu failed: %zd.\n", i, off); + free(spbs); + free(ptrs); + goto fail_pool; + } + } + + /* Free them all - they go to local_shard */ + for (i = 0; i < total_blocks / 2; i++) { + size_t off = ssm_pk_buff_get_off(spbs[i]); + if (ssm_pool_remove(pool, off) != 0) { + printf("Remove %zu failed.\n", i); + free(spbs); + free(ptrs); + goto fail_pool; + } + } + + /* Freed blocks should be in shards (all blocks free again) */ + total_free = 0; + for (i = 0; i < SSM_POOL_SHARDS; i++) { + total_free += sc->shards[i].free_count; + } + + if (total_free != total_blocks) { + printf("Expected %zu free blocks total, got %zu.\n", + total_blocks, total_free); + free(spbs); + free(ptrs); + goto fail_pool; + } + + /* Allocate again - should succeed by taking from shards */ + for (i = 0; i < total_blocks / 2; i++) { + ssize_t off = ssm_pool_alloc(pool, TEST_SIZE, + &ptrs[i], &spbs[i]); + if (off < 0) { + printf("Fallback alloc %zu failed: %zd.\n", i, off); + free(spbs); + free(ptrs); + goto fail_pool; + } + } + + /* Now all allocated blocks are in use again */ + /* Cleanup - free all allocated blocks */ + for (i = 0; i < total_blocks / 2; i++) { + size_t off = ssm_pk_buff_get_off(spbs[i]); + ssm_pool_remove(pool, off); + } + + free(spbs); + free(ptrs); + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_pool: + ssm_pool_destroy(pool); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_multiprocess_sharding(void) +{ + struct ssm_pool * pool; + struct _ssm_pool_hdr * hdr; + struct _ssm_size_class * sc; + pid_t children[SSM_POOL_SHARDS]; + int i; + int status; + + TEST_START(); + + for (i = 0; i < SSM_POOL_SHARDS; i++) + children[i] = -1; + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail; + } + + /* Fork processes to test different shards */ + for (i = 0; i < SSM_POOL_SHARDS; i++) { + children[i] = fork(); + if (children[i] == -1) { + printf("Fork %d failed.\n", i); + goto fail_children; + } + + if (children[i] == 0) { + /* Child process */ + struct ssm_pool * child_pool; + struct ssm_pk_buff * spb; + uint8_t * ptr; + ssize_t off; + int my_shard; + + child_pool = ssm_pool_open(getuid()); + if (child_pool == NULL) + exit(EXIT_FAILURE); + + my_shard = getpid() % SSM_POOL_SHARDS; + (void) my_shard; /* Reserved for future use */ + + /* Each child allocates and frees a block */ + off = ssm_pool_alloc(child_pool, TEST_SIZE, + &ptr, &spb); + if (off < 0) { + ssm_pool_close(child_pool); + exit(EXIT_FAILURE); + } + + /* Small delay to ensure allocation visible */ + usleep(10000); + + if (ssm_pool_remove(child_pool, off) != 0) { + ssm_pool_close(child_pool); + exit(EXIT_FAILURE); + } + + ssm_pool_close(child_pool); + exit(EXIT_SUCCESS); + } + } + + /* Wait for all children */ + for (i = 0; i < SSM_POOL_SHARDS; i++) { + if (waitpid(children[i], &status, 0) == -1) { + printf("Waitpid %d failed.\n", i); + goto fail_children; + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + printf("Child %d failed.\n", i); + goto fail_pool; + } + } + + /* Verify blocks distributed across shards */ + hdr = get_pool_hdr(pool); + + /* Inspect the class that TEST_SIZE allocations used */ + i = select_size_class(hdr, TEST_SIZE); + if (i < 0) { + printf("No size class fits TEST_SIZE=%d.\n", TEST_SIZE); + goto fail_pool; + } + + sc = &hdr->size_classes[i]; + + /* After children allocate and free, blocks should be in shards + * (though exact distribution depends on PID values) + */ + for (i = 0; i < SSM_POOL_SHARDS; i++) { + /* At least some shards should have blocks */ + if (sc->shards[i].free_count > 0) { + break; + } + } + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_children: + /* Kill any remaining children */ + for (i = 0; i < SSM_POOL_SHARDS; i++) { + if (children[i] > 0) + kill(children[i], SIGKILL); + } + fail_pool: + ssm_pool_destroy(pool); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_exhaustion_with_fallback(void) +{ + struct ssm_pool * pool; + struct ssm_pk_buff * spb; + uint8_t * ptr; + ssize_t off; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail; + } + + /* Allocate until exhausted across all shards */ + while (true) { + off = ssm_pool_alloc(pool, TEST_SIZE, &ptr, &spb); + if (off < 0) { + if (off == -EAGAIN) + break; + printf("Unexpected error: %zd.\n", off); + goto fail_pool; + } + } + + /* Should fail with -EAGAIN when truly exhausted */ + off = ssm_pool_alloc(pool, TEST_SIZE, &ptr, &spb); + if (off != -EAGAIN) { + printf("Expected -EAGAIN, got %zd.\n", off); + goto fail_pool; + } + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_pool: + ssm_pool_destroy(pool); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int pool_sharding_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_lazy_distribution(); + ret |= test_shard_migration(); + ret |= test_fallback_stealing(); + ret |= test_multiprocess_sharding(); + ret |= test_exhaustion_with_fallback(); + + return ret; +} diff --git a/src/lib/ssm/tests/pool_test.c b/src/lib/ssm/tests/pool_test.c new file mode 100644 index 00000000..0f9db24d --- /dev/null +++ b/src/lib/ssm/tests/pool_test.c @@ -0,0 +1,1060 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the Secure Shared Memory (SSM) system + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include "config.h" +#include "ssm.h" + +#include <test/test.h> +#include <ouroboros/ssm_pool.h> +#include <ouroboros/ssm_rbuff.h> + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <stdbool.h> +#include <stdatomic.h> +#include <sys/wait.h> +#include <sys/types.h> +#include <signal.h> +#include <time.h> + +#define POOL_256 256 +#define POOL_512 512 +#define POOL_1K 1024 +#define POOL_2K 2048 +#define POOL_4K 4096 +#define POOL_16K 16384 +#define POOL_64K 65536 +#define POOL_256K 262144 +#define POOL_1M 1048576 +#define POOL_2M (2 * 1024 * 1024) + +static int test_ssm_pool_basic_allocation(void) +{ + struct ssm_pool * pool; + uint8_t * ptr; + struct ssm_pk_buff * spb; + ssize_t ret; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + ret = ssm_pool_alloc(pool, POOL_256, &ptr, &spb); + if (ret < 0) { + printf("Alloc failed: %zd.\n", ret); + goto fail_alloc; + } + + if (spb == NULL) { + printf("Spb is NULL.\n"); + goto fail_alloc; + } + + if (ptr == NULL) { + printf("Ptr is NULL.\n"); + goto fail_alloc; + } + + if (ssm_pk_buff_len(spb) != POOL_256) { + printf("Bad length: %zu.\n", ssm_pk_buff_len(spb)); + goto fail_alloc; + } + + ret = ssm_pool_remove(pool, ret); + if (ret != 0) { + printf("Remove failed: %zd.\n", ret); + goto fail_alloc; + } + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_multiple_allocations(void) +{ + struct ssm_pool * pool; + uint8_t * ptr1; + uint8_t * ptr2; + uint8_t * ptr3; + struct ssm_pk_buff * spb1; + struct ssm_pk_buff * spb2; + struct ssm_pk_buff * spb3; + ssize_t ret1; + ssize_t ret2; + ssize_t ret3; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + ret1 = ssm_pool_alloc(pool, POOL_256, &ptr1, &spb1); + ret2 = ssm_pool_alloc(pool, POOL_256, &ptr2, &spb2); + ret3 = ssm_pool_alloc(pool, POOL_256, &ptr3, &spb3); + if (ret1 < 0 || ret2 < 0 || ret3 < 0) { + printf("Allocs failed: %zd, %zd, %zd.\n", ret1, ret2, ret3); + goto fail_alloc; + } + + if (spb1 == NULL) { + printf("Spb1 is NULL.\n"); + goto fail_alloc; + } + + if (ptr1 == NULL) { + printf("Ptr1 is NULL.\n"); + goto fail_alloc; + } + + if (spb2 == NULL) { + printf("Spb2 is NULL.\n"); + goto fail_alloc; + } + + if (ptr2 == NULL) { + printf("Ptr2 is NULL.\n"); + goto fail_alloc; + } + + if (spb3 == NULL) { + printf("Spb3 is NULL.\n"); + goto fail_alloc; + } + + if (ptr3 == NULL) { + printf("Ptr3 is NULL.\n"); + goto fail_alloc; + } + + if (ssm_pk_buff_len(spb1) != POOL_256) { + printf("Bad length spb1: %zu.\n", ssm_pk_buff_len(spb1)); + goto fail_alloc; + } + + if (ssm_pk_buff_len(spb2) != POOL_256) { + printf("Bad length spb2: %zu.\n", ssm_pk_buff_len(spb2)); + goto fail_alloc; + } + + if (ssm_pk_buff_len(spb3) != POOL_256) { + printf("Bad length spb3: %zu.\n", ssm_pk_buff_len(spb3)); + goto fail_alloc; + } + + if (ssm_pool_remove(pool, ret2) != 0) { + printf("Remove ret2 failed.\n"); + goto fail_alloc; + } + + if (ssm_pool_remove(pool, ret1) != 0) { + printf("Remove ret1 failed.\n"); + goto fail_alloc; + } + + if (ssm_pool_remove(pool, ret3) != 0) { + printf("Remove ret3 failed.\n"); + goto fail_alloc; + } + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_no_fallback_for_large(void) +{ + struct ssm_pool * pool; + uint8_t * ptr; + struct ssm_pk_buff * spb; + ssize_t ret; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + ret = ssm_pool_alloc(pool, POOL_2M, &ptr, &spb); + if (ret >= 0) { + printf("Oversized alloc succeeded: %zd.\n", ret); + goto fail_alloc; + } + + if (ret != -EMSGSIZE) { + printf("Wrong error: %zd.\n", ret); + goto fail_alloc; + } + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_blocking_vs_nonblocking(void) +{ + struct ssm_pool * pool; + uint8_t * ptr; + struct ssm_pk_buff * spb; + ssize_t ret; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + ret = ssm_pool_alloc(pool, POOL_2M, &ptr, &spb); + if (ret != -EMSGSIZE) { + printf("Nonblocking oversized: %zd.\n", ret); + goto fail_alloc; + } + + ret = ssm_pool_alloc_b(pool, POOL_2M, &ptr, &spb, NULL); + if (ret != -EMSGSIZE) { + printf("Blocking oversized: %zd.\n", ret); + goto fail_alloc; + } + + ret = ssm_pool_alloc(pool, POOL_256, &ptr, &spb); + if (ret < 0) { + printf("Valid alloc failed: %zd.\n", ret); + goto fail_alloc; + } + + ssm_pool_remove(pool, ret); + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_stress_test(void) +{ + struct ssm_pool * pool; + uint8_t * ptr; + struct ssm_pk_buff * spb; + ssize_t * indices = NULL; + ssize_t ret; + size_t count = 0; + size_t i; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + indices = malloc(100 * sizeof(*indices)); + if (indices == NULL) { + printf("Malloc failed.\n"); + goto fail_alloc; + } + + for (i = 0; i < 50; i++) { + size_t j; + size_t num; + size_t size; + + num = (i % 50) + 1; + + for (j = 0; j < num && count < 50; j++) { + switch (i % 4) { + case 0: + /* FALLTHRU */ + case 1: + size = POOL_256; + break; + case 2: + /* FALLTHRU */ + case 3: + size = POOL_1K; + break; + default: + size = POOL_256; + break; + } + + ret = ssm_pool_alloc(pool, size, &ptr, &spb); + if (ret < 0) { + printf("Alloc at iter %zu: %zd.\n", i, ret); + goto fail_test; + } + indices[count++] = ret; + } + + for (j = 0; j < count / 2; j++) { + size_t idx = j * 2; + if (idx < count) { + ret = ssm_pool_remove(pool, indices[idx]); + if (ret != 0) { + printf("Remove at iter %zu: %zd.\n", + i, ret); + goto fail_test; + } + memmove(&indices[idx], &indices[idx + 1], + (count - idx - 1) * sizeof(*indices)); + count--; + } + } + + if (i % 10 == 0) { + ret = ssm_pool_alloc(pool, POOL_256, &ptr, &spb); + if (ret < 0) { + printf("Periodic alloc at %zu: %zd.\n", i, ret); + goto fail_test; + } + ssm_pool_remove(pool, ret); + } + } + + for (i = 0; i < count; i++) + ssm_pool_remove(pool, indices[i]); + + free(indices); + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_test: + for (i = 0; i < count; i++) + ssm_pool_remove(pool, indices[i]); + free(indices); + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_open_initializes_ssm(void) +{ + struct ssm_pool * creator; + struct ssm_pool * opener; + uint8_t * ptr; + struct ssm_pk_buff * spb; + ssize_t ret; + + TEST_START(); + + creator = ssm_pool_create(getuid(), getgid()); + if (creator == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + ret = ssm_pool_alloc(creator, POOL_256, &ptr, &spb); + if (ret < 0) { + printf("Creator alloc failed: %zd.\n", ret); + goto fail_creator; + } + ssm_pool_remove(creator, ret); + + opener = ssm_pool_open(getuid()); + if (opener == NULL) { + printf("Open failed.\n"); + goto fail_creator; + } + + ret = ssm_pool_alloc(opener, POOL_256, &ptr, &spb); + if (ret < 0) { + printf("Opener alloc failed: %zd.\n", ret); + goto fail_opener; + } + + ssm_pool_remove(opener, ret); + ssm_pool_close(opener); + ssm_pool_destroy(creator); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_opener: + ssm_pool_close(opener); + fail_creator: + ssm_pool_destroy(creator); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_bounds_checking(void) +{ + struct ssm_pool * pool; + struct ssm_pk_buff * spb; + ssize_t ret; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + ret = ssm_pool_alloc(pool, POOL_256, NULL, &spb); + if (ret < 0) { + printf("alloc failed: %zd.\n", ret); + goto fail_alloc; + } + + spb = ssm_pool_get(pool, 0); + if (spb != NULL) { + printf("Get at offset 0.\n"); + goto fail_alloc; + } + + spb = ssm_pool_get(pool, 100000000UL); + if (spb != NULL) { + printf("Get beyond pool.\n"); + goto fail_alloc; + } + + ret = ssm_pool_remove(pool, 0); + if (ret != -EINVAL) { + printf("Remove at offset 0: %zd.\n", ret); + goto fail_alloc; + } + + ret = ssm_pool_remove(pool, 100000000UL); + if (ret != -EINVAL) { + printf("Remove beyond pool: %zd.\n", ret); + goto fail_alloc; + } + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_inter_process_communication(void) +{ + struct ssm_pool * pool; + struct ssm_rbuff * rb; + struct ssm_pk_buff * spb; + uint8_t * ptr; + uint8_t * data; + const char * msg = "inter-process test"; + size_t len; + ssize_t idx; + pid_t pid; + int status; + + TEST_START(); + + len = strlen(msg) + 1; + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + rb = ssm_rbuff_create(getpid(), 1); + if (rb == NULL) { + printf("Rbuff create failed.\n"); + goto fail_pool; + } + + pid = fork(); + if (pid < 0) { + printf("Fork failed.\n"); + goto fail_rbuff; + } + + if (pid == 0) { + idx = ssm_rbuff_read_b(rb, NULL); + if (idx < 0) { + printf("Child: rbuff read: %zd.\n", idx); + exit(1); + } + + spb = ssm_pool_get(pool, idx); + if (spb == NULL) { + printf("Child: pool get failed.\n"); + exit(1); + } + + data = ssm_pk_buff_head(spb); + if (data == NULL) { + printf("Child: data is NULL.\n"); + ssm_pool_remove(pool, idx); + exit(1); + } + + if (strcmp((char *)data, msg) != 0) { + printf("Child: data mismatch.\n"); + ssm_pool_remove(pool, idx); + exit(1); + } + + ssm_pool_remove(pool, idx); + exit(0); + } + + idx = ssm_pool_alloc(pool, len, &ptr, &spb); + if (idx < 0) { + printf("Parent: pool alloc: %zd.\n", idx); + goto fail_child; + } + + memcpy(ptr, msg, len); + + if (ssm_rbuff_write(rb, idx) < 0) { + printf("Parent: rbuff write failed.\n"); + ssm_pool_remove(pool, idx); + goto fail_child; + } + + if (waitpid(pid, &status, 0) < 0) { + printf("Parent: waitpid failed.\n"); + ssm_pool_remove(pool, idx); + goto fail_rbuff; + } + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + printf("Child failed.\n"); + ssm_pool_remove(pool, idx); + goto fail_rbuff; + } + + ssm_rbuff_destroy(rb); + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_child: + waitpid(pid, &status, 0); + fail_rbuff: + ssm_rbuff_destroy(rb); + fail_pool: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_read_operation(void) +{ + struct ssm_pool * pool; + struct ssm_pk_buff * spb; + uint8_t * wptr; + uint8_t * rptr; + const char * data = "ssm_pool_read test"; + size_t len; + ssize_t idx; + ssize_t ret; + + TEST_START(); + + len = strlen(data) + 1; + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + idx = ssm_pool_alloc(pool, len, &wptr, &spb); + if (idx < 0) { + printf("alloc failed: %zd.\n", idx); + goto fail_alloc; + } + + memcpy(wptr, data, len); + + ret = ssm_pool_read(&rptr, pool, idx); + if (ret < 0) { + printf("Read failed: %zd.\n", ret); + goto fail_read; + } + + if (rptr == NULL) { + printf("NULL pointer.\n"); + goto fail_read; + } + + if (strcmp((char *)rptr, data) != 0) { + printf("Data mismatch.\n"); + goto fail_read; + } + + ssm_pool_remove(pool, idx); + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_read: + ssm_pool_remove(pool, idx); + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_mlock_operation(void) +{ + struct ssm_pool * pool; + int ret; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + ret = ssm_pool_mlock(pool); + if (ret < 0) + printf("Mlock failed: %d (may need privileges).\n", ret); + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pk_buff_operations(void) +{ + struct ssm_pool * pool; + struct ssm_pk_buff * spb; + uint8_t * ptr; + uint8_t * head; + uint8_t * tail; + const char * data = "packet buffer test"; + size_t dlen; + size_t len; + ssize_t idx; + + TEST_START(); + + dlen = strlen(data); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + idx = ssm_pool_alloc(pool, POOL_256, &ptr, &spb); + if (idx < 0) { + printf("alloc failed: %zd.\n", idx); + goto fail_alloc; + } + + head = ssm_pk_buff_head(spb); + if (head != ptr) { + printf("Head mismatch.\n"); + goto fail_ops; + } + + len = ssm_pk_buff_len(spb); + if (len != POOL_256) { + printf("Bad length: %zu.\n", len); + goto fail_ops; + } + + tail = ssm_pk_buff_tail(spb); + if (tail != ptr + len) { + printf("Tail mismatch.\n"); + goto fail_ops; + } + + memcpy(head, data, dlen); + + tail = ssm_pk_buff_push_tail(spb, 32); + if (tail == NULL) { + printf("push_tail failed.\n"); + goto fail_ops; + } + + if (ssm_pk_buff_len(spb) != POOL_256 + 32) { + printf("Length after push_tail: %zu.\n", + ssm_pk_buff_len(spb)); + goto fail_ops; + } + + if (memcmp(head, data, dlen) != 0) { + printf("Data corrupted.\n"); + goto fail_ops; + } + + tail = ssm_pk_buff_pop_tail(spb, 32); + if (tail == NULL) { + printf("pop_tail failed.\n"); + goto fail_ops; + } + + if (ssm_pk_buff_len(spb) != POOL_256) { + printf("Length after pop_tail: %zu.\n", + ssm_pk_buff_len(spb)); + goto fail_ops; + } + + ssm_pool_remove(pool, idx); + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_ops: + ssm_pool_remove(pool, idx); + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +#define OVERHEAD (offsetof(struct ssm_pk_buff, data) + \ + SSM_PK_BUFF_HEADSPACE + SSM_PK_BUFF_TAILSPACE) +static int test_ssm_pool_size_class_boundaries(void) +{ + struct ssm_pool * pool; + struct ssm_pk_buff * spb; + uint8_t * ptr; + size_t sizes[] = { + POOL_512 - OVERHEAD, + POOL_512 - OVERHEAD + 1, + POOL_1K - OVERHEAD, + POOL_1K - OVERHEAD + 1, + POOL_2K - OVERHEAD, + POOL_2K - OVERHEAD + 1, + POOL_4K - OVERHEAD, + POOL_4K - OVERHEAD + 1, + POOL_16K - OVERHEAD, + POOL_16K - OVERHEAD + 1, + POOL_64K - OVERHEAD, + POOL_64K - OVERHEAD + 1, + POOL_256K - OVERHEAD, + }; + size_t expected_classes[] = { + 512, 1024, 1024, 2048, 2048, 4096, 4096, 16384, + 16384, 65536, 65536, 262144, 262144 + }; + size_t i; + ssize_t idx; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + for (i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + struct ssm_pk_buff * hdr; + size_t actual_class; + + idx = ssm_pool_alloc(pool, sizes[i], &ptr, &spb); + if (idx < 0) { + printf("Alloc at %zu failed: %zd.\n", sizes[i], idx); + goto fail_alloc; + } + + if (ssm_pk_buff_len(spb) != sizes[i]) { + printf("Length mismatch at %zu: %zu.\n", + sizes[i], ssm_pk_buff_len(spb)); + ssm_pool_remove(pool, idx); + goto fail_alloc; + } + + /* Verify correct size class was used + * hdr->size is the data array size (object_size - header) */ + hdr = spb; + actual_class = hdr->size + offsetof(struct ssm_pk_buff, data); + if (actual_class != expected_classes[i]) { + printf("Wrong class for len=%zu: want %zu, got %zu.\n", + sizes[i], expected_classes[i], actual_class); + ssm_pool_remove(pool, idx); + goto fail_alloc; + } + + memset(ptr, i & 0xFF, sizes[i]); + + ssm_pool_remove(pool, idx); + } + + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_exhaustion(void) +{ + struct ssm_pool * pool; + struct ssm_pk_buff * spb; + uint8_t * ptr; + ssize_t * indices; + size_t count = 0; + size_t i; + ssize_t ret; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + indices = malloc(2048 * sizeof(*indices)); + if (indices == NULL) { + printf("Malloc failed.\n"); + goto fail_alloc; + } + + for (i = 0; i < 2048; i++) { + ret = ssm_pool_alloc(pool, POOL_256, &ptr, &spb); + if (ret < 0) { + if (ret == -EAGAIN) + break; + printf("Alloc error: %zd.\n", ret); + goto fail_test; + } + indices[count++] = ret; + } + + if (count == 0) { + printf("No allocs succeeded.\n"); + goto fail_test; + } + + ret = ssm_pool_alloc(pool, POOL_256, &ptr, &spb); + if (ret >= 0) { + ssm_pool_remove(pool, ret); + } else if (ret != -EAGAIN) { + printf("Unexpected error: %zd.\n", ret); + goto fail_test; + } + + for (i = 0; i < count; i++) + ssm_pool_remove(pool, indices[i]); + + ret = ssm_pool_alloc(pool, POOL_256, &ptr, &spb); + if (ret < 0) { + printf("Alloc after free failed: %zd.\n", ret); + goto fail_test; + } + ssm_pool_remove(pool, ret); + + free(indices); + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_test: + for (i = 0; i < count; i++) + ssm_pool_remove(pool, indices[i]); + free(indices); + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_pool_reclaim_orphans(void) +{ + struct ssm_pool * pool; + uint8_t * ptr1; + uint8_t * ptr2; + uint8_t * ptr3; + struct ssm_pk_buff * spb1; + struct ssm_pk_buff * spb2; + struct ssm_pk_buff * spb3; + ssize_t ret1; + ssize_t ret2; + ssize_t ret3; + pid_t my_pid; + pid_t fake_pid = 99999; + + TEST_START(); + + pool = ssm_pool_create(getuid(), getgid()); + if (pool == NULL) { + printf("Failed to create pool.\n"); + goto fail_create; + } + + my_pid = getpid(); + + /* Allocate some blocks */ + ret1 = ssm_pool_alloc(pool, POOL_256, &ptr1, &spb1); + ret2 = ssm_pool_alloc(pool, POOL_512, &ptr2, &spb2); + ret3 = ssm_pool_alloc(pool, POOL_1K, &ptr3, &spb3); + if (ret1 < 0 || ret2 < 0 || ret3 < 0) { + printf("Allocs failed: %zd, %zd, %zd.\n", ret1, ret2, ret3); + goto fail_alloc; + } + + /* Simulate blocks from another process by changing allocator_pid */ + spb1->allocator_pid = fake_pid; + spb2->allocator_pid = fake_pid; + /* Keep spb3 with our pid */ + + /* Reclaim orphans from fake_pid */ + ssm_pool_reclaim_orphans(pool, fake_pid); + + /* Verify spb1 and spb2 have refcount 0 (reclaimed) */ + if (spb1->refcount != 0) { + printf("spb1 refcount should be 0, got %u.\n", spb1->refcount); + goto fail_test; + } + + if (spb2->refcount != 0) { + printf("spb2 refcount should be 0, got %u.\n", spb2->refcount); + goto fail_test; + } + + /* Verify spb3 still has refcount 1 (not reclaimed) */ + if (spb3->refcount != 1) { + printf("spb3 refcount should be 1, got %u.\n", spb3->refcount); + goto fail_test; + } + + /* Clean up */ + ssm_pool_remove(pool, ret3); + + /* Try allocating again - should get blocks from reclaimed pool */ + ret1 = ssm_pool_alloc(pool, POOL_256, &ptr1, &spb1); + if (ret1 < 0) { + printf("Alloc after reclaim failed: %zd.\n", ret1); + goto fail_test; + } + + /* Verify new allocation has our pid */ + if (spb1->allocator_pid != my_pid) { + printf("New block has wrong pid: %d vs %d.\n", + spb1->allocator_pid, my_pid); + goto fail_test; + } + + ssm_pool_remove(pool, ret1); + ssm_pool_destroy(pool); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_test: + ssm_pool_remove(pool, ret3); + fail_alloc: + ssm_pool_destroy(pool); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int pool_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_ssm_pool_basic_allocation(); + ret |= test_ssm_pool_multiple_allocations(); + ret |= test_ssm_pool_no_fallback_for_large(); + ret |= test_ssm_pool_blocking_vs_nonblocking(); + ret |= test_ssm_pool_stress_test(); + ret |= test_ssm_pool_open_initializes_ssm(); + ret |= test_ssm_pool_bounds_checking(); + ret |= test_ssm_pool_inter_process_communication(); + ret |= test_ssm_pool_read_operation(); + ret |= test_ssm_pool_mlock_operation(); + ret |= test_ssm_pk_buff_operations(); + ret |= test_ssm_pool_size_class_boundaries(); + ret |= test_ssm_pool_exhaustion(); + ret |= test_ssm_pool_reclaim_orphans(); + + return ret; +} diff --git a/src/lib/ssm/tests/rbuff_test.c b/src/lib/ssm/tests/rbuff_test.c new file mode 100644 index 00000000..58cb39c3 --- /dev/null +++ b/src/lib/ssm/tests/rbuff_test.c @@ -0,0 +1,675 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the SSM notification ring buffer + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#define _DEFAULT_SOURCE +#else +#define _POSIX_C_SOURCE 200112L +#endif + +#include "config.h" +#include "ssm.h" + +#include <test/test.h> +#include <ouroboros/ssm_rbuff.h> +#include <ouroboros/errno.h> +#include <ouroboros/time.h> + +#include <errno.h> +#include <stdio.h> +#include <unistd.h> +#include <pthread.h> + +static int test_ssm_rbuff_create_destroy(void) +{ + struct ssm_rbuff * rb; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 1); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_write_read(void) +{ + struct ssm_rbuff * rb; + ssize_t idx; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 2); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + if (ssm_rbuff_write(rb, 42) < 0) { + printf("Failed to write value.\n"); + goto fail_rb; + } + + if (ssm_rbuff_queued(rb) != 1) { + printf("Queue length should be 1, got %zu.\n", + ssm_rbuff_queued(rb)); + goto fail_rb; + } + + idx = ssm_rbuff_read(rb); + if (idx != 42) { + printf("Expected 42, got %zd.\n", idx); + goto fail_rb; + } + + if (ssm_rbuff_queued(rb) != 0) { + printf("Queue should be empty, got %zu.\n", + ssm_rbuff_queued(rb)); + goto fail_rb; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_rb: + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_read_empty(void) +{ + struct ssm_rbuff * rb; + ssize_t ret; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 3); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + ret = ssm_rbuff_read(rb); + if (ret != -EAGAIN) { + printf("Expected -EAGAIN, got %zd.\n", ret); + goto fail_rb; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_rb: + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_fill_drain(void) +{ + struct ssm_rbuff * rb; + size_t i; + ssize_t ret; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 4); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + for (i = 0; i < SSM_RBUFF_SIZE - 1; ++i) { + if (ssm_rbuff_queued(rb) != i) { + printf("Expected %zu queued, got %zu.\n", + i, ssm_rbuff_queued(rb)); + goto fail_rb; + } + if (ssm_rbuff_write(rb, i) < 0) { + printf("Failed to write at index %zu.\n", i); + goto fail_rb; + } + } + + if (ssm_rbuff_queued(rb) != SSM_RBUFF_SIZE - 1) { + printf("Expected %d queued, got %zu.\n", + SSM_RBUFF_SIZE - 1, ssm_rbuff_queued(rb)); + goto fail_rb; + } + + ret = ssm_rbuff_write(rb, 999); + if (ret != -EAGAIN) { + printf("Expected -EAGAIN on full buffer, got %zd.\n", ret); + goto fail_rb; + } + + for (i = 0; i < SSM_RBUFF_SIZE - 1; ++i) { + ret = ssm_rbuff_read(rb); + if (ret != (ssize_t) i) { + printf("Expected %zu, got %zd.\n", i, ret); + goto fail_rb; + } + } + + if (ssm_rbuff_queued(rb) != 0) { + printf("Expected empty queue, got %zu.\n", + ssm_rbuff_queued(rb)); + goto fail_rb; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_rb: + while (ssm_rbuff_read(rb) >= 0) + ; + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_acl(void) +{ + struct ssm_rbuff * rb; + uint32_t acl; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 5); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + acl = ssm_rbuff_get_acl(rb); + if (acl != ACL_RDWR) { + printf("Expected ACL_RDWR, got %u.\n", acl); + goto fail_rb; + } + + ssm_rbuff_set_acl(rb, ACL_RDONLY); + acl = ssm_rbuff_get_acl(rb); + if (acl != ACL_RDONLY) { + printf("Expected ACL_RDONLY, got %u.\n", acl); + goto fail_rb; + } + + if (ssm_rbuff_write(rb, 1) != -ENOTALLOC) { + printf("Expected -ENOTALLOC on RDONLY.\n"); + goto fail_rb; + } + + ssm_rbuff_set_acl(rb, ACL_FLOWDOWN); + if (ssm_rbuff_write(rb, 1) != -EFLOWDOWN) { + printf("Expected -EFLOWDOWN on FLOWDOWN.\n"); + goto fail_rb; + } + + if (ssm_rbuff_read(rb) != -EFLOWDOWN) { + printf("Expected -EFLOWDOWN on read with FLOWDOWN.\n"); + goto fail_rb; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_rb: + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_open_close(void) +{ + struct ssm_rbuff * rb1; + struct ssm_rbuff * rb2; + pid_t pid; + + TEST_START(); + + pid = getpid(); + + rb1 = ssm_rbuff_create(pid, 6); + if (rb1 == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + if (ssm_rbuff_write(rb1, 123) < 0) { + printf("Failed to write value.\n"); + goto fail_rb1; + } + + rb2 = ssm_rbuff_open(pid, 6); + if (rb2 == NULL) { + printf("Failed to open existing rbuff.\n"); + goto fail_rb1; + } + + if (ssm_rbuff_queued(rb2) != 1) { + printf("Expected 1 queued in opened rbuff, got %zu.\n", + ssm_rbuff_queued(rb2)); + goto fail_rb2; + } + + if (ssm_rbuff_read(rb2) != 123) { + printf("Failed to read from opened rbuff.\n"); + goto fail_rb2; + } + + ssm_rbuff_close(rb2); + ssm_rbuff_destroy(rb1); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_rb2: + ssm_rbuff_close(rb2); + fail_rb1: + ssm_rbuff_destroy(rb1); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +struct thread_args { + struct ssm_rbuff * rb; + int iterations; + int delay_us; +}; + +static void * writer_thread(void * arg) +{ + struct thread_args * args = (struct thread_args *) arg; + struct timespec delay = {0, 0}; + int i; + + delay.tv_nsec = args->delay_us * 1000L; + + for (i = 0; i < args->iterations; ++i) { + while (ssm_rbuff_write(args->rb, i) < 0) + nanosleep(&delay, NULL); + } + + return NULL; +} + +static void * reader_thread(void * arg) +{ + struct thread_args * args = (struct thread_args *) arg; + struct timespec delay = {0, 0}; + int i; + ssize_t val; + + delay.tv_nsec = args->delay_us * 1000L; + + for (i = 0; i < args->iterations; ++i) { + val = ssm_rbuff_read(args->rb); + while (val < 0) { + nanosleep(&delay, NULL); + val = ssm_rbuff_read(args->rb); + } + if (val != i) { + printf("Expected %d, got %zd.\n", i, val); + return (void *) -1; + } + } + + return NULL; +} + +static void * blocking_writer_thread(void * arg) +{ + struct thread_args * args = (struct thread_args *) arg; + int i; + + for (i = 0; i < args->iterations; ++i) { + if (ssm_rbuff_write_b(args->rb, i, NULL) < 0) + return (void *) -1; + } + + return NULL; +} + +static void * blocking_reader_thread(void * arg) +{ + struct thread_args * args = (struct thread_args *) arg; + int i; + ssize_t val; + + for (i = 0; i < args->iterations; ++i) { + val = ssm_rbuff_read_b(args->rb, NULL); + if (val < 0 || val != i) { + printf("Expected %d, got %zd.\n", i, val); + return (void *) -1; + } + } + + return NULL; +} + +static int test_ssm_rbuff_blocking(void) +{ + struct ssm_rbuff * rb; + pthread_t wthread; + pthread_t rthread; + struct thread_args args; + struct timespec delay = {0, 10 * MILLION}; + void * ret_w; + void * ret_r; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 8); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + args.rb = rb; + args.iterations = 50; + args.delay_us = 0; + + if (pthread_create(&rthread, NULL, blocking_reader_thread, &args)) { + printf("Failed to create reader thread.\n"); + goto fail_rthread; + } + + nanosleep(&delay, NULL); + + if (pthread_create(&wthread, NULL, blocking_writer_thread, &args)) { + printf("Failed to create writer thread.\n"); + pthread_cancel(rthread); + goto fail_wthread; + } + + pthread_join(wthread, &ret_w); + pthread_join(rthread, &ret_r); + + if (ret_w != NULL || ret_r != NULL) { + printf("Thread returned error.\n"); + goto fail_ret; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_ret: + fail_wthread: + pthread_join(rthread, NULL); + fail_rthread: + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_blocking_timeout(void) +{ + struct ssm_rbuff * rb; + struct timespec abs_timeout; + struct timespec interval = {0, 100 * MILLION}; + struct timespec start; + struct timespec end; + ssize_t ret; + long elapsed_ms; + size_t i; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 9); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + clock_gettime(PTHREAD_COND_CLOCK, &start); + ts_add(&start, &interval, &abs_timeout); + + ret = ssm_rbuff_read_b(rb, &abs_timeout); + + clock_gettime(PTHREAD_COND_CLOCK, &end); + + if (ret != -ETIMEDOUT) { + printf("Expected -ETIMEDOUT, got %zd.\n", ret); + goto fail_rb; + } + + elapsed_ms = (end.tv_sec - start.tv_sec) * 1000L + + (end.tv_nsec - start.tv_nsec) / 1000000L; + + if (elapsed_ms < 90 || elapsed_ms > 200) { + printf("Timeout took %ld ms, expected ~100 ms.\n", + elapsed_ms); + goto fail_rb; + } + + for (i = 0; i < SSM_RBUFF_SIZE - 1; ++i) { + if (ssm_rbuff_write(rb, i) < 0) { + printf("Failed to fill buffer.\n"); + goto fail_rb; + } + } + + clock_gettime(PTHREAD_COND_CLOCK, &start); + ts_add(&start, &interval, &abs_timeout); + + ret = ssm_rbuff_write_b(rb, 999, &abs_timeout); + + clock_gettime(PTHREAD_COND_CLOCK, &end); + + if (ret != -ETIMEDOUT) { + printf("Expected -ETIMEDOUT on full buffer, got %zd.\n", + ret); + goto fail_rb; + } + + elapsed_ms = (end.tv_sec - start.tv_sec) * 1000L + + (end.tv_nsec - start.tv_nsec) / 1000000L; + + if (elapsed_ms < 90 || elapsed_ms > 200) { + printf("Write timeout took %ld ms, expected ~100 ms.\n", + elapsed_ms); + goto fail_rb; + } + + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_rb: + while (ssm_rbuff_read(rb) >= 0) + ; + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_blocking_flowdown(void) +{ + struct ssm_rbuff * rb; + struct timespec abs_timeout; + struct timespec now; + struct timespec interval = {5, 0}; + ssize_t ret; + size_t i; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 10); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, &interval, &abs_timeout); + + ssm_rbuff_set_acl(rb, ACL_FLOWDOWN); + + ret = ssm_rbuff_read_b(rb, &abs_timeout); + if (ret != -EFLOWDOWN) { + printf("Expected -EFLOWDOWN, got %zd.\n", ret); + goto fail_rb; + } + + ssm_rbuff_set_acl(rb, ACL_RDWR); + + for (i = 0; i < SSM_RBUFF_SIZE - 1; ++i) { + if (ssm_rbuff_write(rb, i) < 0) { + printf("Failed to fill buffer.\n"); + goto fail_rb; + } + } + + clock_gettime(PTHREAD_COND_CLOCK, &now); + ts_add(&now, &interval, &abs_timeout); + + ssm_rbuff_set_acl(rb, ACL_FLOWDOWN); + + ret = ssm_rbuff_write_b(rb, 999, &abs_timeout); + if (ret != -EFLOWDOWN) { + printf("Expected -EFLOWDOWN on write, got %zd.\n", ret); + goto fail_rb; + } + + ssm_rbuff_set_acl(rb, ACL_RDWR); + while (ssm_rbuff_read(rb) >= 0) + ; + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_rb: + while (ssm_rbuff_read(rb) >= 0) + ; + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ssm_rbuff_threaded(void) +{ + struct ssm_rbuff * rb; + pthread_t wthread; + pthread_t rthread; + struct thread_args args; + void * ret_w; + void * ret_r; + + TEST_START(); + + rb = ssm_rbuff_create(getpid(), 7); + if (rb == NULL) { + printf("Failed to create rbuff.\n"); + goto fail; + } + + args.rb = rb; + args.iterations = 100; + args.delay_us = 100; + + if (pthread_create(&wthread, NULL, writer_thread, &args)) { + printf("Failed to create writer thread.\n"); + goto fail_rb; + } + + if (pthread_create(&rthread, NULL, reader_thread, &args)) { + printf("Failed to create reader thread.\n"); + pthread_cancel(wthread); + pthread_join(wthread, NULL); + goto fail_rb; + } + + pthread_join(wthread, &ret_w); + pthread_join(rthread, &ret_r); + + if (ret_w != NULL || ret_r != NULL) { + printf("Thread returned error.\n"); + goto fail_rb; + } + + ssm_rbuff_destroy(rb); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + + fail_rb: + ssm_rbuff_destroy(rb); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int rbuff_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_ssm_rbuff_create_destroy(); + ret |= test_ssm_rbuff_write_read(); + ret |= test_ssm_rbuff_read_empty(); + ret |= test_ssm_rbuff_fill_drain(); + ret |= test_ssm_rbuff_acl(); + ret |= test_ssm_rbuff_open_close(); + ret |= test_ssm_rbuff_threaded(); + ret |= test_ssm_rbuff_blocking(); + ret |= test_ssm_rbuff_blocking_timeout(); + ret |= test_ssm_rbuff_blocking_flowdown(); + + return ret; +} diff --git a/src/lib/tests/CMakeLists.txt b/src/lib/tests/CMakeLists.txt index 9e23b0ee..32836589 100644 --- a/src/lib/tests/CMakeLists.txt +++ b/src/lib/tests/CMakeLists.txt @@ -1,27 +1,32 @@ get_filename_component(PARENT_PATH ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) get_filename_component(PARENT_DIR ${PARENT_PATH} NAME) +compute_test_prefix() + create_test_sourcelist(${PARENT_DIR}_tests test_suite.c # Add new tests here + auth_test.c + auth_test_ml_dsa.c + auth_test_slh_dsa.c bitmap_test.c btree_test.c - crc32_test.c + crypt_test.c + hash_test.c + kex_test.c + kex_test_ml_kem.c md5_test.c sha3_test.c - shm_rbuff_test.c - time_utils_test.c + sockets_test.c + time_test.c + tpm_test.c + tw_test.c ) -add_executable(${PARENT_DIR}_test EXCLUDE_FROM_ALL ${${PARENT_DIR}_tests}) +add_executable(${PARENT_DIR}_test ${${PARENT_DIR}_tests}) +disable_test_logging_for_target(${PARENT_DIR}_test) target_link_libraries(${PARENT_DIR}_test ouroboros-common) -add_dependencies(check ${PARENT_DIR}_test) - -set(tests_to_run ${${PARENT_DIR}_tests}) -remove(tests_to_run test_suite.c) +add_dependencies(build_tests ${PARENT_DIR}_test) -foreach (test ${tests_to_run}) - get_filename_component(test_name ${test} NAME_WE) - add_test(${test_name} ${C_TEST_PATH}/${PARENT_DIR}_test ${test_name}) -endforeach (test) +ouroboros_register_tests(TARGET ${PARENT_DIR}_test TESTS ${${PARENT_DIR}_tests}) diff --git a/src/lib/tests/auth_test.c b/src/lib/tests/auth_test.c new file mode 100644 index 00000000..0f3ef715 --- /dev/null +++ b/src/lib/tests/auth_test.c @@ -0,0 +1,603 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the authentication functions + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "config.h" + +#include <test/test.h> +#include <ouroboros/crypt.h> +#include <ouroboros/random.h> +#include <ouroboros/utils.h> + +#include <test/certs/ecdsa.h> + +#define TEST_MSG_SIZE 1500 + +static int test_auth_create_destroy_ctx(void) +{ + struct auth_ctx * ctx; + + TEST_START(); + + ctx = auth_create_ctx(); + if (ctx == NULL) { + printf("Failed to create auth context.\n"); + goto fail_create; + } + + auth_destroy_ctx(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_load_free_crt(void) +{ + void * crt; + + TEST_START(); + + if (crypt_load_crt_str(root_ca_crt_ec, &crt) < 0) { + printf("Failed to load certificate string.\n"); + goto fail_load; + } + + crypt_free_crt(crt); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_crypt_get_pubkey_crt(void) +{ + void * pk; + void * crt; + + TEST_START(); + + if (crypt_load_crt_str(signed_server_crt_ec, &crt) < 0) { + printf("Failed to load server certificate from string.\n"); + goto fail_load; + } + + if (crypt_get_pubkey_crt(crt, &pk) < 0) { + printf("Failed to get public key from certificate.\n"); + goto fail_get_pubkey; + } + + crypt_free_key(pk); + crypt_free_crt(crt); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + + fail_get_pubkey: + crypt_free_crt(crt); + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_check_crt_name(void) +{ + void * crt; + + TEST_START(); + + if (crypt_load_crt_str(signed_server_crt_ec, &crt) < 0) { + printf("Failed to load certificate from string.\n"); + goto fail_load; + } + + if (crypt_check_crt_name(crt, "test-1.unittest.o7s") < 0) { + printf("Failed to verify correct name.\n"); + goto fail_check; + } + + if (crypt_check_crt_name(crt, "bogus.name") == 0) { + printf("Failed to detect incorrect name.\n"); + goto fail_check; + } + + crypt_free_crt(crt); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_check: + crypt_free_crt(crt); + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_load_free_privkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_privkey_str(server_pkp_ec, &key) < 0) { + printf("Failed to load server key pair from string.\n"); + goto fail_load; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_load_free_pubkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_pubkey_str(server_pk_ec, &key) < 0) { + printf("Failed to load server public key from string.\n"); + goto fail_load; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_crypt_check_pubkey_crt(void) +{ + void * pk; + void * crt_pk; + void * crt; + + TEST_START(); + + if (crypt_load_crt_str(signed_server_crt_ec, &crt) < 0) { + printf("Failed to load public certificate from string.\n"); + goto fail_crt; + } + + if (crypt_load_pubkey_str(server_pk_ec, &pk) < 0) { + printf("Failed to load public key from string.\n"); + goto fail_pubkey; + } + + if (crypt_get_pubkey_crt(crt, &crt_pk) < 0) { + printf("Failed to get public key from certificate.\n"); + goto fail_get_pubkey; + } + + if (crypt_cmp_key(pk, crt_pk) != 0) { + printf("Public keys do not match .\n"); + goto fail_check; + } + + + crypt_free_key(crt_pk); + crypt_free_key(pk); + crypt_free_crt(crt); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_check: + crypt_free_key(crt_pk); + fail_get_pubkey: + crypt_free_key(pk); + fail_pubkey: + crypt_free_crt(crt); + fail_crt: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_store_add(void) +{ + struct auth_ctx * ctx; + void * _root_ca_crt; + + TEST_START(); + + ctx = auth_create_ctx(); + if (ctx == NULL) { + printf("Failed to create auth context.\n"); + goto fail_create; + } + + if (crypt_load_crt_str(root_ca_crt_ec, &_root_ca_crt) < 0) { + printf("Failed to load root crt from string.\n"); + goto fail_load; + } + + if (auth_add_crt_to_store(ctx, _root_ca_crt) < 0) { + printf("Failed to add root crt to auth store.\n"); + goto fail_add; + } + + crypt_free_crt(_root_ca_crt); + auth_destroy_ctx(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + + fail_add: + crypt_free_crt(_root_ca_crt); + fail_load: + crypt_free_crt(_root_ca_crt); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_verify_crt(void) +{ + struct auth_ctx * auth; + void * _server_crt; + void * _signed_server_crt; + void * _root_ca_crt; + void * _im_ca_crt; + + TEST_START(); + + auth = auth_create_ctx(); + if (auth == NULL) { + printf("Failed to create auth context.\n"); + goto fail_create_ctx; + } + + if (crypt_load_crt_str(server_crt_ec, &_server_crt) < 0) { + printf("Failed to load self-signed crt from string.\n"); + goto fail_load_server_crt; + } + + if (crypt_load_crt_str(signed_server_crt_ec, &_signed_server_crt) < 0) { + printf("Failed to load signed crt from string.\n"); + goto fail_load_signed_server_crt; + } + + if (crypt_load_crt_str(root_ca_crt_ec, &_root_ca_crt) < 0) { + printf("Failed to load root crt from string.\n"); + goto fail_load_root_ca_crt; + } + + if (crypt_load_crt_str(im_ca_crt_ec, &_im_ca_crt) < 0) { + printf("Failed to load intermediate crt from string.\n"); + goto fail_load_im_ca_crt; + } + + if (auth_add_crt_to_store(auth, _root_ca_crt) < 0) { + printf("Failed to add root ca crt to auth store.\n"); + goto fail_verify; + } + + if (auth_add_crt_to_store(auth, _im_ca_crt) < 0) { + printf("Failed to add intermediate ca crt to auth store.\n"); + goto fail_verify; + } + + if (auth_verify_crt(auth, _signed_server_crt) < 0) { + printf("Failed to verify signed crt with ca crt.\n"); + goto fail_verify; + } + + if (auth_verify_crt(auth, _server_crt) == 0) { + printf("Failed to detect untrusted crt.\n"); + goto fail_verify; + } + + crypt_free_crt(_im_ca_crt); + crypt_free_crt(_root_ca_crt); + crypt_free_crt(_signed_server_crt); + crypt_free_crt(_server_crt); + + auth_destroy_ctx(auth); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + crypt_free_crt(_im_ca_crt); + fail_load_im_ca_crt: + crypt_free_crt(_root_ca_crt); + fail_load_root_ca_crt: + crypt_free_crt(_signed_server_crt); + fail_load_signed_server_crt: + crypt_free_crt(_server_crt); + fail_load_server_crt: + auth_destroy_ctx(auth); + fail_create_ctx: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_verify_crt_missing_root_ca(void) +{ + struct auth_ctx * auth; + void * _signed_server_crt; + void * _im_ca_crt; + + TEST_START(); + + auth = auth_create_ctx(); + if (auth == NULL) { + printf("Failed to create auth context.\n"); + goto fail_create_ctx; + } + + if (crypt_load_crt_str(signed_server_crt_ec, &_signed_server_crt) < 0) { + printf("Failed to load signed crt from string.\n"); + goto fail_load_signed; + } + + if (crypt_load_crt_str(im_ca_crt_ec, &_im_ca_crt) < 0) { + printf("Failed to load intermediate crt from string.\n"); + goto fail_load_im_ca; + } + + /* Add only the intermediate CA - root CA is missing */ + if (auth_add_crt_to_store(auth, _im_ca_crt) < 0) { + printf("Failed to add intermediate ca crt to auth store.\n"); + goto fail_add; + } + + if (auth_verify_crt(auth, _signed_server_crt) == 0) { + printf("Verification should fail without root CA.\n"); + goto fail_add; + } + + crypt_free_crt(_im_ca_crt); + crypt_free_crt(_signed_server_crt); + auth_destroy_ctx(auth); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_add: + crypt_free_crt(_im_ca_crt); + fail_load_im_ca: + crypt_free_crt(_signed_server_crt); + fail_load_signed: + auth_destroy_ctx(auth); + fail_create_ctx: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int test_auth_sign(void) +{ + uint8_t buf[TEST_MSG_SIZE]; + void * pkp; + void * pk; + buffer_t msg; + buffer_t sig; + + TEST_START(); + + msg.data = buf; + msg.len = sizeof(buf); + + if (random_buffer(msg.data, msg.len) < 0) { + printf("Failed to generate random message.\n"); + goto fail_init; + } + + if (crypt_load_privkey_str(server_pkp_ec, &pkp) < 0) { + printf("Failed to load server key pair from string.\n"); + goto fail_init; + } + + if (crypt_load_pubkey_str(server_pk_ec, &pk) < 0) { + printf("Failed to load public key.\n"); + goto fail_pubkey; + } + + if (auth_sign(pkp, 0, msg, &sig) < 0) { + printf("Failed to sign message.\n"); + goto fail_sign; + } + + if (auth_verify_sig(pk, 0, msg, sig) < 0) { + printf("Failed to verify signature.\n"); + goto fail_verify; + } + + freebuf(sig); + + crypt_free_key(pk); + crypt_free_key(pkp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + freebuf(sig); + fail_sign: + crypt_free_key(pk); + fail_pubkey: + crypt_free_key(pkp); + fail_init: + return TEST_RC_FAIL; +} + +int test_auth_bad_signature(void) +{ + uint8_t buf[TEST_MSG_SIZE]; + void * pkp; + void * pk; + buffer_t msg; + buffer_t sig; + buffer_t fake_sig; + + TEST_START(); + + msg.data = buf; + msg.len = sizeof(buf); + + if (random_buffer(msg.data, msg.len) < 0) { + printf("Failed to generate random message.\n"); + goto fail_init; + } + + if (crypt_load_privkey_str(server_pkp_ec, &pkp) < 0) { + printf("Failed to load server key pair from string.\n"); + goto fail_init; + } + + if (crypt_load_pubkey_str(server_pk_ec, &pk) < 0) { + printf("Failed to load public key.\n"); + goto fail_pubkey; + } + + if (auth_sign(pkp, 0, msg, &sig) < 0) { + printf("Failed to sign message.\n"); + goto fail_sign; + } + + fake_sig.data = malloc(sig.len); + if (fake_sig.data == NULL) { + printf("Failed to allocate memory for fake signature.\n"); + goto fail_malloc; + } + + fake_sig.len = sig.len; + if (random_buffer(fake_sig.data, fake_sig.len) < 0) { + printf("Failed to generate random fake signature.\n"); + goto fail_malloc; + } + + if (auth_verify_sig(pk, 0, msg, fake_sig) == 0) { + printf("Failed to detect bad signature.\n"); + goto fail_verify; + } + + freebuf(fake_sig); + freebuf(sig); + + crypt_free_key(pk); + crypt_free_key(pkp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + freebuf(fake_sig); + fail_malloc: + freebuf(sig); + fail_sign: + crypt_free_key(pk); + fail_pubkey: + crypt_free_key(pkp); + fail_init: + return TEST_RC_FAIL; +} + +#define SSC_BUF_SIZE 4096 /* OpenSSL version my return different lengths */ +int test_crt_str(void) +{ + char str[SSC_BUF_SIZE]; + void * crt; + + TEST_START(); + + if (crypt_load_crt_str(signed_server_crt_ec, &crt) < 0) { + printf("Failed to load certificate from string.\n"); + goto fail_load; + } + + if (crypt_crt_str(crt, str) < 0) { + printf("Failed to convert certificate to string.\n"); + goto fail_to_str; + } + + printf("Certificate string:\n%s\n", str); + + crypt_free_crt(crt); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + + fail_to_str: + crypt_free_crt(crt); + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int auth_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_auth_create_destroy_ctx(); +#ifdef HAVE_OPENSSL + ret |= test_load_free_crt(); + ret |= test_check_crt_name(); + ret |= test_crypt_get_pubkey_crt(); + ret |= test_load_free_privkey(); + ret |= test_load_free_pubkey(); + ret |= test_crypt_check_pubkey_crt(); + ret |= test_store_add(); + ret |= test_verify_crt(); + ret |= test_verify_crt_missing_root_ca(); + ret |= test_auth_sign(); + ret |= test_auth_bad_signature(); + ret |= test_crt_str(); +#else + (void) test_load_free_crt; + (void) test_check_crt_name; + (void) test_crypt_get_pubkey_crt; + (void) test_load_free_privkey; + (void) test_load_free_pubkey; + (void) test_crypt_check_pubkey_crt; + (void) test_store_add; + (void) test_verify_crt; + (void) test_verify_crt_missing_root_ca; + (void) test_auth_sign; + (void) test_auth_bad_signature; + (void) test_crt_str; + + ret = TEST_RC_SKIP; +#endif + return ret; +} diff --git a/src/lib/tests/auth_test_ml_dsa.c b/src/lib/tests/auth_test_ml_dsa.c new file mode 100644 index 00000000..cc72e61b --- /dev/null +++ b/src/lib/tests/auth_test_ml_dsa.c @@ -0,0 +1,356 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the ML-DSA-65 authentication functions + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "config.h" + +#include <test/test.h> +#include <ouroboros/crypt.h> +#include <ouroboros/random.h> +#include <ouroboros/utils.h> + +#include <test/certs/ml_dsa.h> + +#define TEST_MSG_SIZE 1500 + +static int test_auth_create_destroy_ctx(void) +{ + struct auth_ctx * ctx; + + TEST_START(); + + ctx = auth_create_ctx(); + if (ctx == NULL) { + printf("Failed to create auth context.\n"); + goto fail_create; + } + + auth_destroy_ctx(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_load_free_crt(void) +{ + void * crt; + + TEST_START(); + + if (crypt_load_crt_str(root_ca_crt_ml, &crt) < 0) { + printf("Failed to load root crt from string.\n"); + goto fail_load; + } + + crypt_free_crt(crt); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_load_free_privkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_privkey_str(server_pkp_ml, &key) < 0) { + printf("Failed to load server key pair from string.\n"); + goto fail_load; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_load_free_pubkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_pubkey_str(server_pk_ml, &key) < 0) { + printf("Failed to load server public key from string.\n"); + goto fail_load; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_verify_crt(void) +{ + struct auth_ctx * auth; + void * _server_crt; + void * _signed_server_crt; + void * _root_ca_crt; + void * _im_ca_crt; + + TEST_START(); + + auth = auth_create_ctx(); + if (auth == NULL) { + printf("Failed to create auth context.\n"); + goto fail_create_ctx; + } + + if (crypt_load_crt_str(server_crt_ml, &_server_crt) < 0) { + printf("Failed to load self-signed crt from string.\n"); + goto fail_load_server_crt; + } + + if (crypt_load_crt_str(signed_server_crt_ml, &_signed_server_crt) < 0) { + printf("Failed to load signed crt from string.\n"); + goto fail_load_signed_server_crt; + } + + if (crypt_load_crt_str(root_ca_crt_ml, &_root_ca_crt) < 0) { + printf("Failed to load root crt from string.\n"); + goto fail_load_root_ca_crt; + } + + if (crypt_load_crt_str(im_ca_crt_ml, &_im_ca_crt) < 0) { + printf("Failed to load intermediate crt from string.\n"); + goto fail_load_im_ca_crt; + } + + if (auth_add_crt_to_store(auth, _root_ca_crt) < 0) { + printf("Failed to add root ca crt to auth store.\n"); + goto fail_verify; + } + + if (auth_add_crt_to_store(auth, _im_ca_crt) < 0) { + printf("Failed to add intermediate ca crt to auth store.\n"); + goto fail_verify; + } + + if (auth_verify_crt(auth, _signed_server_crt) < 0) { + printf("Failed to verify signed crt with ca crt.\n"); + goto fail_verify; + } + + if (auth_verify_crt(auth, _server_crt) == 0) { + printf("Failed to detect untrusted crt.\n"); + goto fail_verify; + } + + crypt_free_crt(_im_ca_crt); + crypt_free_crt(_root_ca_crt); + crypt_free_crt(_signed_server_crt); + crypt_free_crt(_server_crt); + + auth_destroy_ctx(auth); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + crypt_free_crt(_im_ca_crt); + fail_load_im_ca_crt: + crypt_free_crt(_root_ca_crt); + fail_load_root_ca_crt: + crypt_free_crt(_signed_server_crt); + fail_load_signed_server_crt: + crypt_free_crt(_server_crt); + fail_load_server_crt: + auth_destroy_ctx(auth); + fail_create_ctx: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_auth_sign(void) +{ + uint8_t buf[TEST_MSG_SIZE]; + void * pkp; + void * pk; + buffer_t msg; + buffer_t sig; + + TEST_START(); + + msg.data = buf; + msg.len = sizeof(buf); + + if (random_buffer(msg.data, msg.len) < 0) { + printf("Failed to generate random message.\n"); + goto fail_init; + } + + if (crypt_load_privkey_str(server_pkp_ml, &pkp) < 0) { + printf("Failed to load server key pair from string.\n"); + goto fail_init; + } + + if (crypt_load_pubkey_str(server_pk_ml, &pk) < 0) { + printf("Failed to load public key from string.\n"); + goto fail_pubkey; + } + + if (auth_sign(pkp, 0, msg, &sig) < 0) { + printf("Failed to sign message.\n"); + goto fail_sign; + } + + if (auth_verify_sig(pk, 0, msg, sig) < 0) { + printf("Failed to verify signature.\n"); + goto fail_verify; + } + + freebuf(sig); + + crypt_free_key(pk); + crypt_free_key(pkp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + freebuf(sig); + fail_sign: + crypt_free_key(pk); + fail_pubkey: + crypt_free_key(pkp); + fail_init: + return TEST_RC_FAIL; +} + +static int test_auth_bad_signature(void) +{ + uint8_t buf[TEST_MSG_SIZE]; + void * pkp; + void * pk; + buffer_t msg; + buffer_t sig; + buffer_t fake_sig; + + TEST_START(); + + msg.data = buf; + msg.len = sizeof(buf); + + if (random_buffer(msg.data, msg.len) < 0) { + printf("Failed to generate random message.\n"); + goto fail_init; + } + + if (crypt_load_privkey_str(server_pkp_ml, &pkp) < 0) { + printf("Failed to load server key pair from string.\n"); + goto fail_init; + } + + if (crypt_load_pubkey_str(server_pk_ml, &pk) < 0) { + printf("Failed to load public key from string.\n"); + goto fail_pubkey; + } + + if (auth_sign(pkp, 0, msg, &sig) < 0) { + printf("Failed to sign message.\n"); + goto fail_sign; + } + + fake_sig.data = malloc(sig.len); + if (fake_sig.data == NULL) { + printf("Failed to allocate memory for fake signature.\n"); + goto fail_malloc; + } + + fake_sig.len = sig.len; + if (random_buffer(fake_sig.data, fake_sig.len) < 0) { + printf("Failed to generate random fake signature.\n"); + goto fail_malloc; + } + + if (auth_verify_sig(pk, 0, msg, fake_sig) == 0) { + printf("Failed to detect bad ML-DSA-65 signature.\n"); + goto fail_verify; + } + + freebuf(fake_sig); + freebuf(sig); + + crypt_free_key(pk); + crypt_free_key(pkp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + freebuf(fake_sig); + fail_malloc: + freebuf(sig); + fail_sign: + crypt_free_key(pk); + fail_pubkey: + crypt_free_key(pkp); + fail_init: + return TEST_RC_FAIL; +} + +int auth_test_ml_dsa(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + +#ifdef HAVE_OPENSSL_ML_DSA + ret |= test_auth_create_destroy_ctx(); + ret |= test_load_free_crt(); + ret |= test_load_free_privkey(); + ret |= test_load_free_pubkey(); + ret |= test_verify_crt(); + ret |= test_auth_sign(); + ret |= test_auth_bad_signature(); +#else + (void) test_auth_create_destroy_ctx; + (void) test_load_free_crt; + (void) test_load_free_privkey; + (void) test_load_free_pubkey; + (void) test_verify_crt; + (void) test_auth_sign; + (void) test_auth_bad_signature; + + ret = TEST_RC_SKIP; +#endif + return ret; +} diff --git a/src/lib/tests/auth_test_slh_dsa.c b/src/lib/tests/auth_test_slh_dsa.c new file mode 100644 index 00000000..511d20fe --- /dev/null +++ b/src/lib/tests/auth_test_slh_dsa.c @@ -0,0 +1,367 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the SLH-DSA-SHA2-128s authentication functions + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "config.h" + +#include <test/test.h> +#include <ouroboros/crypt.h> +#include <ouroboros/random.h> +#include <ouroboros/utils.h> + +#include <test/certs/slh_dsa.h> + +#define TEST_MSG_SIZE 1500 + +static int test_auth_create_destroy_ctx(void) +{ + struct auth_ctx * ctx; + + TEST_START(); + + ctx = auth_create_ctx(); + if (ctx == NULL) { + printf("Failed to create auth context.\n"); + goto fail_create; + } + + auth_destroy_ctx(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_load_free_crt(void) +{ + void * crt; + + TEST_START(); + + if (crypt_load_crt_str(root_ca_crt_slh, &crt) < 0) { + printf("Failed to load root crt.\n"); + goto fail_load; + } + + crypt_free_crt(crt); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_load_free_privkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_privkey_str(server_pkp_slh, &key) < 0) { + printf("Failed to load server key pair.\n"); + goto fail_load; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_load_free_pubkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_pubkey_str(server_pk_slh, &key) < 0) { + printf("Failed to load server public key.\n"); + goto fail_load; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_load: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_verify_crt(void) +{ + struct auth_ctx * auth; + void * _server_crt; + void * _signed_server_crt; + void * _root_ca_crt; + void * _im_ca_crt; + + TEST_START(); + + auth = auth_create_ctx(); + if (auth == NULL) { + printf("Failed to create auth context.\n"); + goto fail_create_ctx; + } + + if (crypt_load_crt_str(server_crt_slh, + &_server_crt) < 0) { + printf("Failed to load self-signed crt.\n"); + goto fail_load_server_crt; + } + + if (crypt_load_crt_str(signed_server_crt_slh, + &_signed_server_crt) < 0) { + printf("Failed to load signed crt.\n"); + goto fail_load_signed_server_crt; + } + + if (crypt_load_crt_str(root_ca_crt_slh, + &_root_ca_crt) < 0) { + printf("Failed to load root crt.\n"); + goto fail_load_root_ca_crt; + } + + if (crypt_load_crt_str(im_ca_crt_slh, + &_im_ca_crt) < 0) { + printf("Failed to load im crt.\n"); + goto fail_load_im_ca_crt; + } + + if (auth_add_crt_to_store(auth, _root_ca_crt) < 0) { + printf("Failed to add root ca crt.\n"); + goto fail_verify; + } + + if (auth_add_crt_to_store(auth, _im_ca_crt) < 0) { + printf("Failed to add im ca crt.\n"); + goto fail_verify; + } + + if (auth_verify_crt(auth, _signed_server_crt) < 0) { + printf("Failed to verify signed crt.\n"); + goto fail_verify; + } + + if (auth_verify_crt(auth, _server_crt) == 0) { + printf("Failed to detect untrusted crt.\n"); + goto fail_verify; + } + + crypt_free_crt(_im_ca_crt); + crypt_free_crt(_root_ca_crt); + crypt_free_crt(_signed_server_crt); + crypt_free_crt(_server_crt); + + auth_destroy_ctx(auth); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + crypt_free_crt(_im_ca_crt); + fail_load_im_ca_crt: + crypt_free_crt(_root_ca_crt); + fail_load_root_ca_crt: + crypt_free_crt(_signed_server_crt); + fail_load_signed_server_crt: + crypt_free_crt(_server_crt); + fail_load_server_crt: + auth_destroy_ctx(auth); + fail_create_ctx: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_auth_sign(void) +{ + uint8_t buf[TEST_MSG_SIZE]; + void * pkp; + void * pk; + buffer_t msg; + buffer_t sig; + + TEST_START(); + + msg.data = buf; + msg.len = sizeof(buf); + + if (random_buffer(msg.data, msg.len) < 0) { + printf("Failed to gen random message.\n"); + goto fail_init; + } + + if (crypt_load_privkey_str(server_pkp_slh, + &pkp) < 0) { + printf("Failed to load server key pair.\n"); + goto fail_init; + } + + if (crypt_load_pubkey_str(server_pk_slh, + &pk) < 0) { + printf("Failed to load public key.\n"); + goto fail_pubkey; + } + + if (auth_sign(pkp, 0, msg, &sig) < 0) { + printf("Failed to sign message.\n"); + goto fail_sign; + } + + if (auth_verify_sig(pk, 0, msg, sig) < 0) { + printf("Failed to verify signature.\n"); + goto fail_verify; + } + + freebuf(sig); + + crypt_free_key(pk); + crypt_free_key(pkp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + freebuf(sig); + fail_sign: + crypt_free_key(pk); + fail_pubkey: + crypt_free_key(pkp); + fail_init: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_auth_bad_signature(void) +{ + uint8_t buf[TEST_MSG_SIZE]; + void * pkp; + void * pk; + buffer_t msg; + buffer_t sig; + buffer_t fake_sig; + + TEST_START(); + + msg.data = buf; + msg.len = sizeof(buf); + + if (random_buffer(msg.data, msg.len) < 0) { + printf("Failed to gen random message.\n"); + goto fail_init; + } + + if (crypt_load_privkey_str(server_pkp_slh, + &pkp) < 0) { + printf("Failed to load server key pair.\n"); + goto fail_init; + } + + if (crypt_load_pubkey_str(server_pk_slh, + &pk) < 0) { + printf("Failed to load public key.\n"); + goto fail_pubkey; + } + + if (auth_sign(pkp, 0, msg, &sig) < 0) { + printf("Failed to sign message.\n"); + goto fail_sign; + } + + fake_sig.data = malloc(sig.len); + if (fake_sig.data == NULL) { + printf("Failed to alloc fake sig buf.\n"); + goto fail_malloc; + } + + fake_sig.len = sig.len; + if (random_buffer(fake_sig.data, + fake_sig.len) < 0) { + printf("Failed to gen random fake sig.\n"); + goto fail_malloc; + } + + if (auth_verify_sig(pk, 0, msg, fake_sig) == 0) { + printf("Failed to detect bad sig.\n"); + goto fail_verify; + } + + freebuf(fake_sig); + freebuf(sig); + + crypt_free_key(pk); + crypt_free_key(pkp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_verify: + freebuf(fake_sig); + fail_malloc: + freebuf(sig); + fail_sign: + crypt_free_key(pk); + fail_pubkey: + crypt_free_key(pkp); + fail_init: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int auth_test_slh_dsa(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + +#ifdef HAVE_OPENSSL_SLH_DSA + ret |= test_auth_create_destroy_ctx(); + ret |= test_load_free_crt(); + ret |= test_load_free_privkey(); + ret |= test_load_free_pubkey(); + ret |= test_verify_crt(); + ret |= test_auth_sign(); + ret |= test_auth_bad_signature(); +#else + (void) test_auth_create_destroy_ctx; + (void) test_load_free_crt; + (void) test_load_free_privkey; + (void) test_load_free_pubkey; + (void) test_verify_crt; + (void) test_auth_sign; + (void) test_auth_bad_signature; + + ret = TEST_RC_SKIP; +#endif + return ret; +} diff --git a/src/lib/tests/bitmap_test.c b/src/lib/tests/bitmap_test.c index fdacdce8..81735a41 100644 --- a/src/lib/tests/bitmap_test.c +++ b/src/lib/tests/bitmap_test.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Test of the bitmap * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -27,7 +27,8 @@ #define BITMAP_SIZE 200 -int bitmap_test(int argc, char ** argv) +int bitmap_test(int argc, + char ** argv) { struct bmp * bmp; ssize_t bits = BITMAP_SIZE; @@ -60,27 +61,23 @@ int bitmap_test(int argc, char ** argv) if (!bmp_is_id_valid(bmp, id)) { if (i < BITMAP_SIZE + offset) { printf("Failed valid ID %d (%zd).\n", i, id); - bmp_destroy(bmp); - return -1; + goto fail; } if (id >= offset && id < bits + offset) { printf("Valid ID %zd returned invalid.\n", id); - bmp_destroy(bmp); - return -1; + goto fail; } continue; } if (!bmp_is_id_used(bmp, id)) { printf("ID not marked in use.\n"); - bmp_destroy(bmp); - return -1; + goto fail; } if (id != i) { printf("Wrong ID returned.\n"); - bmp_destroy(bmp); - return -1; + goto fail; } } @@ -89,20 +86,24 @@ int bitmap_test(int argc, char ** argv) if (bmp_release(bmp, r)) { printf("Failed to release ID.\n"); - return -1; + goto fail; } id = bmp_allocate(bmp); if (!bmp_is_id_valid(bmp, id)) continue; + if (id != r) { printf("Wrong prev ID returned.\n"); - bmp_destroy(bmp); - return -1; + goto fail; } } bmp_destroy(bmp); return 0; + + fail: + bmp_destroy(bmp); + return -1; } diff --git a/src/lib/tests/btree_test.c b/src/lib/tests/btree_test.c index 00b3cde1..d1a72af8 100644 --- a/src/lib/tests/btree_test.c +++ b/src/lib/tests/btree_test.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Test of the B-tree implementation * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/src/lib/tests/crypt_test.c b/src/lib/tests/crypt_test.c new file mode 100644 index 00000000..028c4eb5 --- /dev/null +++ b/src/lib/tests/crypt_test.c @@ -0,0 +1,459 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the cryptography functions + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include "config.h" + +#include <test/test.h> +#include <ouroboros/random.h> +#include <ouroboros/crypt.h> +#include <ouroboros/utils.h> + +#include <stdio.h> + +#define TEST_PACKET_SIZE 1500 + +extern const uint16_t crypt_supported_nids[]; +extern const uint16_t md_supported_nids[]; + +static int test_crypt_create_destroy(void) +{ + struct crypt_ctx * ctx; + uint8_t key[SYMMKEYSZ]; + struct crypt_sk sk = { + .nid = NID_aes_256_gcm, + .key = key, + .rot_bit = KEY_ROTATION_BIT + }; + + TEST_START(); + + memset(key, 0, sizeof(key)); + + ctx = crypt_create_ctx(&sk); + if (ctx == NULL) { + printf("Failed to initialize cryptography.\n"); + goto fail; + } + + crypt_destroy_ctx(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_crypt_encrypt_decrypt(int nid) +{ + uint8_t pkt[TEST_PACKET_SIZE]; + struct crypt_ctx * ctx; + uint8_t key[SYMMKEYSZ]; + struct crypt_sk sk = { + .nid = NID_aes_256_gcm, + .key = key, + .rot_bit = KEY_ROTATION_BIT + }; + buffer_t in; + buffer_t out; + buffer_t out2; + const char * cipher; + + cipher = crypt_nid_to_str(nid); + TEST_START("(%s)", cipher); + + if (random_buffer(key, sizeof(key)) < 0) { + printf("Failed to generate random key.\n"); + goto fail_init; + } + + if (random_buffer(pkt, sizeof(pkt)) < 0) { + printf("Failed to generate random data.\n"); + goto fail_init; + } + + ctx = crypt_create_ctx(&sk); + if (ctx == NULL) { + printf("Failed to initialize cryptography.\n"); + goto fail_init; + } + + in.len = sizeof(pkt); + in.data = pkt; + + if (crypt_encrypt(ctx, in, &out) < 0) { + printf("Encryption failed.\n"); + goto fail_encrypt; + } + + if (out.len < in.len) { + printf("Encryption returned too little data.\n"); + goto fail_encrypt; + } + + if (crypt_decrypt(ctx, out, &out2) < 0) { + printf("Decryption failed.\n"); + goto fail_decrypt; + } + + if (out2.len != in.len) { + printf("Decrypted data length does not match original.\n"); + goto fail_chk; + } + + if (memcmp(in.data, out2.data, in.len) != 0) { + printf("Decrypted data does not match original.\n"); + goto fail_chk; + } + + crypt_destroy_ctx(ctx); + freebuf(out2); + freebuf(out); + + TEST_SUCCESS("(%s)", cipher); + + return TEST_RC_SUCCESS; + fail_chk: + freebuf(out2); + fail_decrypt: + freebuf(out); + fail_encrypt: + crypt_destroy_ctx(ctx); + fail_init: + TEST_FAIL("(%s)", cipher); + return TEST_RC_FAIL; +} + +static int test_encrypt_decrypt_all(void) +{ + int ret = 0; + int i; + + for (i = 0; crypt_supported_nids[i] != NID_undef; i++) + ret |= test_crypt_encrypt_decrypt(crypt_supported_nids[i]); + + return ret; +} + +#ifdef HAVE_OPENSSL +#include <openssl/evp.h> +#include <openssl/obj_mac.h> + +static int test_cipher_nid_values(void) +{ + int i; + + TEST_START(); + + /* Loop over all supported ciphers and verify NIDs match OpenSSL's */ + for (i = 0; crypt_supported_nids[i] != NID_undef; i++) { + uint16_t our_nid = crypt_supported_nids[i]; + const char * str = crypt_nid_to_str(our_nid); + const EVP_CIPHER * cipher; + int openssl_nid; + + if (str == NULL) { + printf("crypt_nid_to_str failed for NID %u\n", our_nid); + goto fail; + } + + cipher = EVP_get_cipherbyname(str); + if (cipher == NULL) { + printf("OpenSSL doesn't recognize cipher '%s'\n", str); + goto fail; + } + + openssl_nid = EVP_CIPHER_nid(cipher); + + if (our_nid != openssl_nid) { + printf("NID mismatch for '%s': ours=%u, OpenSSL=%d\n", + str, our_nid, openssl_nid); + goto fail; + } + + /* Test reverse conversion */ + if (crypt_str_to_nid(str) != our_nid) { + printf("crypt_str_to_nid failed for '%s'\n", str); + goto fail; + } + } + + /* Test error cases */ + if (crypt_str_to_nid("invalid") != NID_undef) { + printf("crypt_str_to_nid: no NID_undef for invalid.\n"); + goto fail; + } + + if (crypt_nid_to_str(9999) != NULL) { + printf("crypt_nid_to_str should return NULL for invalid NID\n"); + goto fail; + } + + if (crypt_str_to_nid(NULL) != NID_undef) { + printf("crypt_str_to_nid should return NID_undef for NULL\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_md_nid_values(void) +{ + int i; + + TEST_START(); + + for (i = 0; md_supported_nids[i] != NID_undef; i++) { + uint16_t our_nid = md_supported_nids[i]; + const EVP_MD * md; + int openssl_nid; + + md = EVP_get_digestbynid(our_nid); + if (md == NULL) { + printf("OpenSSL doesn't recognize NID %u\n", our_nid); + goto fail; + } + + openssl_nid = EVP_MD_nid(md); + if (our_nid != openssl_nid) { + printf("NID mismatch: ours=%u, OpenSSL=%d\n", + our_nid, openssl_nid); + goto fail; + } + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} +#endif + +static int test_key_rotation(void) +{ + uint8_t pkt[TEST_PACKET_SIZE]; + struct crypt_ctx * tx_ctx; + struct crypt_ctx * rx_ctx; + uint8_t key[SYMMKEYSZ]; + struct crypt_sk sk = { + .nid = NID_aes_256_gcm, + .key = key, + .rot_bit = 7 + }; + buffer_t in; + buffer_t enc; + buffer_t dec; + uint32_t i; + uint32_t threshold; + + TEST_START(); + + if (random_buffer(key, sizeof(key)) < 0) { + printf("Failed to generate random key.\n"); + goto fail; + } + + if (random_buffer(pkt, sizeof(pkt)) < 0) { + printf("Failed to generate random data.\n"); + goto fail; + } + + tx_ctx = crypt_create_ctx(&sk); + if (tx_ctx == NULL) { + printf("Failed to create TX context.\n"); + goto fail; + } + + rx_ctx = crypt_create_ctx(&sk); + if (rx_ctx == NULL) { + printf("Failed to create RX context.\n"); + goto fail_tx; + } + + in.len = sizeof(pkt); + in.data = pkt; + + threshold = (1U << sk.rot_bit); + + /* Encrypt and decrypt across multiple rotations */ + for (i = 0; i < threshold * 3; i++) { + if (crypt_encrypt(tx_ctx, in, &enc) < 0) { + printf("Encryption failed at packet %u.\n", i); + goto fail_rx; + } + + if (crypt_decrypt(rx_ctx, enc, &dec) < 0) { + printf("Decryption failed at packet %u.\n", i); + freebuf(enc); + goto fail_rx; + } + + if (dec.len != in.len || + memcmp(in.data, dec.data, in.len) != 0) { + printf("Data mismatch at packet %u.\n", i); + freebuf(dec); + freebuf(enc); + goto fail_rx; + } + + freebuf(dec); + freebuf(enc); + } + + crypt_destroy_ctx(rx_ctx); + crypt_destroy_ctx(tx_ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_rx: + crypt_destroy_ctx(rx_ctx); + fail_tx: + crypt_destroy_ctx(tx_ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_key_phase_bit(void) +{ + uint8_t pkt[TEST_PACKET_SIZE]; + struct crypt_ctx * ctx; + uint8_t key[SYMMKEYSZ]; + struct crypt_sk sk = { + .nid = NID_aes_256_gcm, + .key = key, + .rot_bit = 7 + }; + buffer_t in; + buffer_t out; + uint32_t count; + uint32_t threshold; + uint8_t phase_before; + uint8_t phase_after; + int ivsz; + + TEST_START(); + + if (random_buffer(key, sizeof(key)) < 0) { + printf("Failed to generate random key.\n"); + goto fail; + } + + if (random_buffer(pkt, sizeof(pkt)) < 0) { + printf("Failed to generate random data.\n"); + goto fail; + } + + ctx = crypt_create_ctx(&sk); + if (ctx == NULL) { + printf("Failed to initialize cryptography.\n"); + goto fail; + } + + ivsz = crypt_get_ivsz(ctx); + if (ivsz <= 0) { + printf("Invalid IV size.\n"); + goto fail_ctx; + } + + in.len = sizeof(pkt); + in.data = pkt; + + /* Encrypt packets up to just before rotation threshold */ + threshold = (1U << sk.rot_bit); + + /* Encrypt threshold - 1 packets (indices 0 to threshold-2) */ + for (count = 0; count < threshold - 1; count++) { + if (crypt_encrypt(ctx, in, &out) < 0) { + printf("Encryption failed at count %u.\n", count); + goto fail_ctx; + } + freebuf(out); + } + + /* Packet at index threshold-1: phase should still be initial */ + if (crypt_encrypt(ctx, in, &out) < 0) { + printf("Encryption failed before rotation.\n"); + goto fail_ctx; + } + phase_before = (out.data[0] & 0x80) ? 1 : 0; + freebuf(out); + + /* Packet at index threshold: phase should have toggled */ + if (crypt_encrypt(ctx, in, &out) < 0) { + printf("Encryption failed at rotation threshold.\n"); + goto fail_ctx; + } + phase_after = (out.data[0] & 0x80) ? 1 : 0; + freebuf(out); + + /* Phase bit should have toggled */ + if (phase_before == phase_after) { + printf("Phase bit did not toggle: before=%u, after=%u.\n", + phase_before, phase_after); + goto fail_ctx; + } + + crypt_destroy_ctx(ctx); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_ctx: + crypt_destroy_ctx(ctx); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int crypt_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_crypt_create_destroy(); + ret |= test_encrypt_decrypt_all(); +#ifdef HAVE_OPENSSL + ret |= test_cipher_nid_values(); + ret |= test_md_nid_values(); + ret |= test_key_rotation(); + ret |= test_key_phase_bit(); +#else + (void) test_key_rotation; + (void) test_key_phase_bit; + + return TEST_RC_SKIP; +#endif + return ret; +} diff --git a/src/lib/tests/hash_test.c b/src/lib/tests/hash_test.c new file mode 100644 index 00000000..451d3c25 --- /dev/null +++ b/src/lib/tests/hash_test.c @@ -0,0 +1,312 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the hashing functions + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#include <ouroboros/hash.h> +#include <test/test.h> + +#include <stdlib.h> +#include <stdint.h> +#include <assert.h> +#include <string.h> +#include <stdio.h> + +/* + * Test vectors calculated at + * https://www.lammertbies.nl/comm/info/crc-calculation.html + */ + +struct vec_entry { + char * in; + char * out; +}; + +static int test_crc8(void) +{ + int ret = 0; + + struct vec_entry vec [] = { + { "", "00" }, + { "123456789", "df" }, + { NULL, NULL } + }; + + struct vec_entry * cur = vec; + + TEST_START(); + + while (cur->in != NULL) { + uint8_t crc; + char res[3]; + + str_hash(HASH_CRC8, &crc, cur->in); + + sprintf(res, "%02x", crc); + if (strcmp(res, cur->out) != 0) { + printf("Hash failed %s != %s.\n", res, cur->out); + ret |= -1; + } + + ++cur; + } + + TEST_END(ret); + + return ret; +} + +static int test_crc16(void) +{ + int ret = 0; + + struct vec_entry vec [] = { + { "", "ffff" }, + { "123456789", "29b1" }, + { NULL, NULL } + }; + + struct vec_entry * cur = vec; + + TEST_START(); + + while (cur->in != NULL) { + uint8_t crc[2]; + char res[5]; + + str_hash(HASH_CRC16, crc, cur->in); + + sprintf(res, "%02x%02x", crc[0], crc[1]); + if (strcmp(res, cur->out) != 0) { + printf("Hash failed %s != %s.\n", res, cur->out); + ret |= -1; + } + + ++cur; + } + + TEST_END(ret); + + return ret; +} + +static int test_crc32(void) +{ + int ret = 0; + + struct vec_entry vec [] = { + { "0", "f4dbdf21" }, + { "123456789", "cbf43926" }, + { "987654321", "015f0201" }, + { NULL, NULL } + }; + + struct vec_entry * cur = vec; + + TEST_START(); + + while (cur->in != NULL) { + uint8_t crc[4]; + char res[9]; + + str_hash(HASH_CRC32, crc, cur->in); + + sprintf(res, HASH_FMT32, HASH_VAL32(crc)); + if (strcmp(res, cur->out) != 0) { + printf("Hash failed %s != %s.\n", res, cur->out); + ret |= -1; + } + + ++cur; + } + + TEST_END(ret); + + return ret; +} + +static int test_crc64(void) +{ + int ret = 0; + + struct vec_entry vec [] = { + { "", "0000000000000000" }, + { "123456789", "ae8b14860a799888" }, + { "0123456789abcdef", + "091485ca7018730e" }, + { NULL, NULL } + }; + + struct vec_entry * cur = vec; + + TEST_START(); + + while (cur->in != NULL) { + uint8_t crc[8]; + char res[17]; + + str_hash(HASH_CRC64, crc, cur->in); + + sprintf(res, HASH_FMT64, HASH_VAL64(crc)); + if (strcmp(res, cur->out) != 0) { + printf("Hash failed %s != %s.\n", res, cur->out); + ret |= -1; + } + + ++cur; + } + + TEST_END(ret); + + return ret; +} + +static int test_md5(void) +{ + int ret = 0; + + struct vec_entry vec [] = {{ + "abc", + "900150983cd24fb0d6963f7d28e17f72" + }, { + "The quick brown fox jumps over the lazy dog", + "9e107d9d372bb6826bd81d3542a419d6" + }, { + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "8215ef0796a20bcaaae116d3876c664a" + }, { + NULL, + NULL + }}; + + struct vec_entry * cur = vec; + + TEST_START(); + + + while (cur->in != NULL) { + uint8_t md5[16]; + char res[33]; + + str_hash(HASH_MD5, md5, cur->in); + + sprintf(res, HASH_FMT128, HASH_VAL128(md5)); + if (strcmp(res, cur->out) != 0) { + printf("Hash failed %s != %s.\n", res, cur->out); + ret |= -1; + } + + ++cur; + } + + TEST_END(ret); + + return ret; +} + +static int test_sha3(void) +{ + int ret = 0; + + uint8_t sha3[64]; + char res[129]; + + char * in = "abc"; + + char * out = + "e642824c3f8cf24ad09234ee7d3c766f" + "c9a3a5168d0c94ad73b46fdf"; + + TEST_START(); + + str_hash(HASH_SHA3_224, sha3, in); + + sprintf(res, HASH_FMT224, HASH_VAL224(sha3)); + if (strcmp(res, out) != 0) { + printf("SHA3-224 failed %s != %s", res, out); + ret |= -1; + } + + out = + "3a985da74fe225b2045c172d6bd390bd" + "855f086e3e9d525b46bfe24511431532"; + + str_hash(HASH_SHA3_256, sha3, in); + + sprintf(res, HASH_FMT256, HASH_VAL256(sha3)); + if (strcmp(res, out) != 0) { + printf("SHA3-256 failed %s != %s.\n", res, out); + ret |= -1; + } + + out = + "ec01498288516fc926459f58e2c6ad8d" + "f9b473cb0fc08c2596da7cf0e49be4b2" + "98d88cea927ac7f539f1edf228376d25"; + + str_hash(HASH_SHA3_384, sha3, in); + + sprintf(res, HASH_FMT384, HASH_VAL384(sha3)); + if (strcmp(res, out) != 0) { + printf("SHA3-384failed %s != %s.'n", res, out); + ret |= -1; + } + + out = + "b751850b1a57168a5693cd924b6b096e" + "08f621827444f70d884f5d0240d2712e" + "10e116e9192af3c91a7ec57647e39340" + "57340b4cf408d5a56592f8274eec53f0"; + + str_hash(HASH_SHA3_512, sha3, in); + + sprintf(res, HASH_FMT512, HASH_VAL512(sha3)); + if (strcmp(res, out) != 0) { + printf("SHA3-512 failed %s != %s.\n", res, out); + ret |= -1; + } + + TEST_END(ret); + + return ret; +} + +int hash_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_crc8(); + + ret |= test_crc16(); + + ret |= test_crc32(); + + ret |= test_crc64(); + + ret |= test_md5(); + + ret |= test_sha3(); + + return ret; +} diff --git a/src/lib/tests/kex_test.c b/src/lib/tests/kex_test.c new file mode 100644 index 00000000..6a4f802e --- /dev/null +++ b/src/lib/tests/kex_test.c @@ -0,0 +1,844 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the key exchange functions + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include "config.h" + +#include <test/test.h> +#include <ouroboros/utils.h> +#include <ouroboros/crypt.h> + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#ifdef HAVE_OPENSSL +#include <openssl/evp.h> +#include <openssl/x509.h> +#endif + +/* Test configuration strings */ +#define KEX_CONFIG_CUSTOM \ + "kex=X25519\n" + +#define KEX_CONFIG_NONE \ + "none\n" + +#define KEX_CONFIG_WHITESPACE \ + "# Comment line\n" \ + "kex = X448" \ + "\n" \ + "# Another comment\n" + +#define KEX_CONFIG_CIPHER \ + "kex=X25519\n" \ + "cipher=chacha20-poly1305\n" + +#define KEX_CONFIG_DIGEST \ + "kex=X25519\n" \ + "digest=sha384\n" + +/* Test key material for key loading tests */ +#define X25519_PRIVKEY_PEM \ + "-----BEGIN PRIVATE KEY-----\n" \ + "MC4CAQAwBQYDK2VuBCIEIJDd3+/0k2IZlaH5sZ9Z2e5J8dV2U0nsXaSUm70ZaMhL\n" \ + "-----END PRIVATE KEY-----\n" + +#define X25519_PUBKEY_PEM \ + "-----BEGIN PUBLIC KEY-----\n" \ + "MCowBQYDK2VuAyEAKYLIycSZtLFlwAX07YWWgBAYhEnRxHfgK1TVw9+mtBs=\n" \ + "-----END PUBLIC KEY-----\n" + +/* Helper macro to open string constant as FILE stream */ +#define FMEMOPEN_STR(str) fmemopen((void *) (str), strlen(str), "r") + +extern const uint16_t kex_supported_nids[]; + +int parse_sec_config(struct sec_config * cfg, + FILE * fp); + +static int test_kex_create_destroy(void) +{ + struct sec_config cfg; + + TEST_START(); + + memset(&cfg, 0, sizeof(cfg)); + cfg.x.nid = NID_X9_62_prime256v1; + cfg.x.str = kex_nid_to_str(cfg.x.nid); + cfg.c.nid = NID_aes_256_gcm; + cfg.c.str = crypt_nid_to_str(cfg.c.nid); + + if (cfg.x.nid == NID_undef || cfg.c.nid == NID_undef) { + printf("Failed to initialize kex config.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_dh_pkp_create_destroy(void) +{ + struct sec_config kex; + void * pkp; + uint8_t buf[CRYPT_KEY_BUFSZ]; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + SET_KEX_ALGO(&kex, "prime256v1"); + + if (kex_pkp_create(&kex, &pkp, buf) < 0) { + printf("Failed to create DH PKP.\n"); + goto fail; + } + + kex_pkp_destroy(pkp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_get_algo_from_pk(const char * algo) +{ + struct sec_config kex; + void * pkp; + buffer_t pk; + ssize_t len; + uint8_t buf[CRYPT_KEY_BUFSZ]; + char extracted_algo[256]; + + TEST_START("(%s)", algo); + + memset(&kex, 0, sizeof(kex)); + SET_KEX_ALGO(&kex, algo); + + len = kex_pkp_create(&kex, &pkp, buf); + if (len < 0) { + printf("Failed to create key pair.\n"); + goto fail; + } + + pk.len = (size_t) len; + pk.data = buf; + + /* Use raw decode for hybrid KEMs, DER for others */ + if (IS_HYBRID_KEM(algo)) { + if (kex_get_algo_from_pk_raw(pk, extracted_algo) < 0) { + printf("Failed to extract algo from pk.\n"); + goto fail_pkp; + } + } else { + if (kex_get_algo_from_pk_der(pk, extracted_algo) < 0) { + printf("Failed to extract algo from pk.\n"); + goto fail_pkp; + } + } + + /* All algorithms should now return the specific group name */ + if (strcmp(extracted_algo, algo) != 0) { + printf("Algo mismatch: expected %s, got %s.\n", + algo, extracted_algo); + goto fail_pkp; + } + + kex_pkp_destroy(pkp); + + TEST_SUCCESS("(%s)", algo); + + return TEST_RC_SUCCESS; + fail_pkp: + kex_pkp_destroy(pkp); + fail: + TEST_FAIL("(%s)", algo); + return TEST_RC_FAIL; +} + +static int test_kex_get_algo_from_pk_all(void) +{ + int ret = 0; + int i; + + for (i = 0; kex_supported_nids[i] != NID_undef; i++) { + const char * algo = kex_nid_to_str(kex_supported_nids[i]); + ret |= test_kex_get_algo_from_pk(algo); + } + + return ret; +} + +static int test_kex_dhe_derive(const char * algo) +{ + struct sec_config kex; + void * pkp1; + void * pkp2; + buffer_t pk1; + buffer_t pk2; + ssize_t len; + uint8_t buf1[CRYPT_KEY_BUFSZ]; + uint8_t buf2[CRYPT_KEY_BUFSZ]; + uint8_t s1[SYMMKEYSZ]; + uint8_t s2[SYMMKEYSZ]; + + TEST_START("(%s)", algo); + + memset(&kex, 0, sizeof(kex)); + SET_KEX_ALGO(&kex, algo); + + len = kex_pkp_create(&kex, &pkp1, buf1); + if (len < 0) { + printf("Failed to create first key pair for %s.\n", algo); + goto fail; + } + + pk1.len = (size_t) len; + pk1.data = buf1; + + len = kex_pkp_create(&kex, &pkp2, buf2); + if (len < 0) { + printf("Failed to create second key pair for %s.\n", algo); + goto fail_pkp1; + } + + pk2.len = (size_t) len; + pk2.data = buf2; + + if (kex_dhe_derive(&kex, pkp1, pk2, s1) < 0) { + printf("Failed to derive first key for %s.\n", algo); + goto fail_pkp2; + } + + if (kex_dhe_derive(&kex, pkp2, pk1, s2) < 0) { + printf("Failed to derive second key for %s.\n", algo); + goto fail_pkp2; + } + + if (memcmp(s1, s2, SYMMKEYSZ) != 0) { + printf("Derived keys do not match for %s.\n", algo); + goto fail_pkp2; + } + + kex_pkp_destroy(pkp2); + kex_pkp_destroy(pkp1); + + TEST_SUCCESS("(%s)", algo); + + return TEST_RC_SUCCESS; + fail_pkp2: + kex_pkp_destroy(pkp2); + fail_pkp1: + kex_pkp_destroy(pkp1); + fail: + TEST_FAIL("(%s)", algo); + return TEST_RC_FAIL; +} + +static int test_kex_validate_algo(void) +{ + TEST_START(); + + if (kex_validate_algo("prime256v1") != 0) { + printf("prime256v1 should be valid.\n"); + goto fail; + } + + if (kex_validate_algo("X25519") != 0) { + printf("X25519 should be valid.\n"); + goto fail; + } + +#ifdef HAVE_OPENSSL_ML_KEM + if (kex_validate_algo("ML-KEM-768") != 0) { + printf("ML-KEM-768 should be valid.\n"); + goto fail; + } +#endif + + if (kex_validate_algo("ffdhe2048") != 0) { + printf("ffdhe2048 should be valid.\n"); + goto fail; + } + + if (kex_validate_algo("invalid_algo") == 0) { + printf("invalid_algo should be rejected.\n"); + goto fail; + } + + if (kex_validate_algo("rsa2048") == 0) { + printf("rsa2048 should be rejected.\n"); + goto fail; + } + + if (kex_validate_algo(NULL) == 0) { + printf("NULL should be rejected.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_dhe_corrupted_pubkey(const char * algo) +{ + struct sec_config kex; + void * pkp; + buffer_t pk; + ssize_t len; + uint8_t buf[CRYPT_KEY_BUFSZ]; + uint8_t s[SYMMKEYSZ]; + + TEST_START("(%s)", algo); + + memset(&kex, 0, sizeof(kex)); + SET_KEX_ALGO(&kex, algo); + + len = kex_pkp_create(&kex, &pkp, buf); + if (len < 0) { + printf("Failed to create key pair.\n"); + goto fail; + } + + pk.len = (size_t) len; + pk.data = buf; + + /* Corrupt the public key */ + buf[0] ^= 0xFF; + buf[len - 1] ^= 0xFF; + + if (kex_dhe_derive(&kex, pkp, pk, s) == 0) { + printf("Should fail with corrupted public key.\n"); + goto fail_pkp; + } + + kex_pkp_destroy(pkp); + + TEST_SUCCESS("(%s)", algo); + + return TEST_RC_SUCCESS; + fail_pkp: + kex_pkp_destroy(pkp); + fail: + TEST_FAIL("(%s)", algo); + return TEST_RC_FAIL; +} + +static int test_kex_dhe_wrong_algo(void) +{ + struct sec_config kex1; + struct sec_config kex2; + void * pkp1; + void * pkp2; + buffer_t pk2; + ssize_t len; + uint8_t buf1[CRYPT_KEY_BUFSZ]; + uint8_t buf2[CRYPT_KEY_BUFSZ]; + uint8_t s[SYMMKEYSZ]; + const char * algo1 = "X25519"; + const char * algo2 = "X448"; + + TEST_START("(%s vs %s)", algo1, algo2); + + memset(&kex1, 0, sizeof(kex1)); + memset(&kex2, 0, sizeof(kex2)); + SET_KEX_ALGO(&kex1, algo1); + SET_KEX_ALGO(&kex2, algo2); + + if (kex_pkp_create(&kex1, &pkp1, buf1) < 0) { + printf("Failed to create first key pair.\n"); + goto fail; + } + + len = kex_pkp_create(&kex2, &pkp2, buf2); + if (len < 0) { + printf("Failed to create second key pair.\n"); + goto fail_pkp1; + } + + pk2.len = (size_t) len; + pk2.data = buf2; + + /* Try to derive with mismatched algorithms */ + if (kex_dhe_derive(&kex1, pkp1, pk2, s) == 0) { + printf("Should fail with mismatched algorithms.\n"); + goto fail_pkp2; + } + + kex_pkp_destroy(pkp2); + kex_pkp_destroy(pkp1); + + TEST_SUCCESS("(%s vs %s)", algo1, algo2); + + return TEST_RC_SUCCESS; + fail_pkp2: + kex_pkp_destroy(pkp2); + fail_pkp1: + kex_pkp_destroy(pkp1); + fail: + TEST_FAIL("(%s vs %s)", algo1, algo2); + return TEST_RC_FAIL; +} + +static int test_kex_load_dhe_privkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_privkey_str(X25519_PRIVKEY_PEM, &key) < 0) { + printf("Failed to load X25519 private key.\n"); + goto fail; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_load_dhe_pubkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_pubkey_str(X25519_PUBKEY_PEM, &key) < 0) { + printf("Failed to load X25519 public key.\n"); + goto fail; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +#ifdef HAVE_OPENSSL +#include <openssl/obj_mac.h> + +static int test_kex_nid_values(void) +{ + int i; + + TEST_START(); + + /* Verify all KEX algorithm NIDs match OpenSSL's */ + for (i = 0; kex_supported_nids[i] != NID_undef; i++) { + uint16_t our_nid = kex_supported_nids[i]; + const char * kex_name; + int openssl_nid; + + kex_name = kex_nid_to_str(our_nid); + if (kex_name == NULL) { + printf("kex_nid_to_str failed for NID %u\n", our_nid); + goto fail; + } + + /* Test reverse conversion */ + if (kex_str_to_nid(kex_name) != our_nid) { + printf("kex_str_to_nid failed for '%s'\n", kex_name); + goto fail; + } + + /* Get OpenSSL's NID for this name */ + openssl_nid = OBJ_txt2nid(kex_name); + if (openssl_nid != NID_undef) { + /* OpenSSL recognizes this algorithm */ + if (our_nid != openssl_nid) { + printf("NID mismatch for '%s': " + "ours=%d, OpenSSL=%d\n", + kex_name, our_nid, openssl_nid); + goto fail; + } + } else { + /* Verify no NID collision with different algorithm */ + const char * ossl_name = OBJ_nid2sn(our_nid); + if (ossl_name != NULL && + strcmp(ossl_name, kex_name) != 0) { + printf("NID collision for '%d': " + "ours=%s, OpenSSL=%s\n", + our_nid, kex_name, ossl_name); + goto fail; + } + } + } + + /* Test error cases */ + if (kex_str_to_nid("invalid") != NID_undef) { + printf("kex_str_to_nid should return NID_undef for invalid\n"); + goto fail; + } + + if (kex_nid_to_str(9999) != NULL) { + printf("kex_nid_to_str should return NULL for invalid NID\n"); + goto fail; + } + + if (kex_str_to_nid(NULL) != NID_undef) { + printf("kex_str_to_nid should return NID_undef for NULL\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} +#endif + +static int test_kex_all(void) +{ + int ret = 0; + int i; + + for (i = 0; kex_supported_nids[i] != NID_undef; i++) { + const char * algo = kex_nid_to_str(kex_supported_nids[i]); + + /* KEM tests are in kex_test_ml_kem.c */ + if (IS_KEM_ALGORITHM(algo)) + continue; + + ret |= test_kex_dhe_derive(algo); + } + + return ret; +} + +static int test_kex_dhe_corrupted_pubkey_all(void) +{ + int ret = 0; + int i; + + /* Test corruption for all DHE algorithms */ + /* KEM error injection tests are in kex_test_ml_kem.c */ + for (i = 0; kex_supported_nids[i] != NID_undef; i++) { + const char * algo = kex_nid_to_str(kex_supported_nids[i]); + + if (IS_KEM_ALGORITHM(algo)) + continue; + + ret |= test_kex_dhe_corrupted_pubkey(algo); + } + + return ret; +} + +static int test_kex_parse_config_empty(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR("\n"); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse empty config.\n"); + fclose(fp); + goto fail; + } + + if (strcmp(kex.x.str, "prime256v1") != 0) { + printf("Empty config should use prime256v1.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_parse_config_custom(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_CUSTOM); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse custom config.\n"); + fclose(fp); + goto fail; + } + + if (strcmp(kex.x.str, "X25519") != 0) { + printf("Algorithm not set correctly.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_parse_config_none(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_NONE); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse 'none' config.\n"); + fclose(fp); + goto fail; + } + + if (kex.x.nid != NID_undef) { + printf("'none' keyword should disable encryption.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_parse_config_whitespace(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_WHITESPACE); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse config with comments.\n"); + fclose(fp); + goto fail; + } + + if (strcmp(kex.x.str, "X448") != 0) { + printf("Algorithm with whitespace not parsed correctly.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_parse_config_cipher(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_CIPHER); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse cipher config.\n"); + fclose(fp); + goto fail; + } + + if (strcmp(kex.x.str, "X25519") != 0) { + printf("Algorithm not set correctly.\n"); + fclose(fp); + goto fail; + } + + if (kex.c.nid != NID_chacha20_poly1305) { + printf("Cipher not set correctly.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_parse_config_digest(void) +{ + struct sec_config kex; + FILE * fp; + + TEST_START(); + + memset(&kex, 0, sizeof(kex)); + + fp = FMEMOPEN_STR(KEX_CONFIG_DIGEST); + if (fp == NULL) { + printf("Failed to open memory stream.\n"); + goto fail; + } + + if (parse_sec_config(&kex, fp) < 0) { + printf("Failed to parse digest config.\n"); + fclose(fp); + goto fail; + } + + if (strcmp(kex.x.str, "X25519") != 0) { + printf("Algorithm not set correctly.\n"); + fclose(fp); + goto fail; + } + + if (kex.d.nid != NID_sha384) { + printf("Digest not set correctly.\n"); + fclose(fp); + goto fail; + } + + fclose(fp); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int kex_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_kex_create_destroy(); + ret |= test_kex_parse_config_empty(); + ret |= test_kex_parse_config_none(); +#ifdef HAVE_OPENSSL + ret |= test_kex_parse_config_custom(); + ret |= test_kex_parse_config_whitespace(); + ret |= test_kex_parse_config_cipher(); + ret |= test_kex_parse_config_digest(); + ret |= test_kex_nid_values(); + ret |= test_kex_dh_pkp_create_destroy(); + ret |= test_kex_all(); + ret |= test_kex_validate_algo(); + ret |= test_kex_get_algo_from_pk_all(); + ret |= test_kex_dhe_wrong_algo(); + ret |= test_kex_dhe_corrupted_pubkey_all(); + ret |= test_kex_load_dhe_privkey(); + ret |= test_kex_load_dhe_pubkey(); +#else + (void) test_kex_parse_config_custom; + (void) test_kex_parse_config_whitespace; + (void) test_kex_parse_config_cipher; + (void) test_kex_parse_config_digest; + (void) test_kex_dh_pkp_create_destroy; + (void) test_kex_all; + (void) test_kex_validate_algo; + (void) test_kex_get_algo_from_pk_all; + (void) test_kex_dhe_wrong_algo(); + (void) test_kex_dhe_corrupted_pubkey_all; + (void) test_kex_load_dhe_privkey; + (void) test_kex_load_dhe_pubkey; + + ret = TEST_RC_SKIP; +#endif + return ret; +} diff --git a/src/lib/tests/kex_test_ml_kem.c b/src/lib/tests/kex_test_ml_kem.c new file mode 100644 index 00000000..7761c3dc --- /dev/null +++ b/src/lib/tests/kex_test_ml_kem.c @@ -0,0 +1,549 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the post-quantum key exchange functions + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include "config.h" + +#include <test/test.h> +#include <ouroboros/utils.h> +#include <ouroboros/crypt.h> +#include <ouroboros/random.h> + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#ifdef HAVE_OPENSSL +#include <openssl/evp.h> +#include <openssl/x509.h> +#endif + +extern const uint16_t kex_supported_nids[]; +extern const uint16_t md_supported_nids[]; + +static int get_random_kdf(void) +{ + static int idx = 0; + int count; + + if (md_supported_nids[0] == NID_undef) + return NID_undef; + + for (count = 0; md_supported_nids[count] != NID_undef; count++) + ; + + return md_supported_nids[(idx++) % count]; +} + +/* ML-KEM-768 test key material */ + +#define MLKEM768_PRIVKEY_PEM \ + "-----BEGIN PRIVATE KEY-----\n" \ + "MIIJvgIBADALBglghkgBZQMEBAIEggmqMIIJpgRA+QIIiQLQkS5fl5RluSmgXRjZ\n" \ + "YU16W4TVt0dmnBP41rLTTRT3S8CRtkb+xmoFAcWTfEzbdr5pp3g2CBRx+APXTwSC\n" \ + "CWBll6AecTd1Kqdyix3zNQcthDBP0XnwdTHDqkKuFzMP58Y+0gc9Bo+W0xBOK2ZK\n" \ + "gcAmix3YLJuDS8Teep/Tdc7KIm5AaLNoI8BIMgKC/ASsW8kC+78BV4OIgqNWurS9\n" \ + "BrTiCmiag7c+6DsVDJHJ4kfcccwUDBKiW0v+LAkk1HXBcx6usrwuFC0H3ICli2sC\n" \ + "o5DfGL7g4kWHhobXjAZnxn298C8FGmLQK5kah4nZiJ+MuHqrirziCGTLKkY1a8vC\n" \ + "GFgzfHIcvB4dtyi9dxZmWpSXqDf2AVNgqrD2C7WQEULQOKxm/I8Mw31Yp8TC6SAP\n" \ + "RzM4cBAXF00W4Rce05O0am/ga5dStAhikMESyckCoEGlPFFXOmjy1HmOasI+AbGk\n" \ + "2BKp6cfbImbjd0ePdCSFEgIQwAQHm7+4UoZR2JmNwSI1AC2P4FMRAIaD2A69i6LC\n" \ + "kFniGcOog5m09nw5FqZmeEfNs6yyFGSX16D1YyjuooAFGlU0FFX7aKwsYM8t1gkS\n" \ + "YSUfMxIW9yzhSW4vZHuGyxlxBMr1y51RZrW8gnvW5p/Ip5yDBJRahY6KMWT15C14\n" \ + "C2rIe8U+d4Xi5IMI3D1JNpwFebYhKs3/ManxoU7Fwwa0GzQrgLYU5KhqO8/hopnl\n" \ + "8mQH+BPh+TR5lqYawS7HZXFJE8JzOnCtOSgB6Hz2U7oG9ik8h0FRqVD3ak20EmZU\n" \ + "c7gpGW8Odc51uaIBzDu4ej4dGgwo4awYaX4ugLOutHqGqRfCjIVb6XQ4m35p4KKi\n" \ + "qBVQ211aIhavUIgNECJ7WUETilXyyHLB9x3EFJdidEfSRUxLYJNAC5XM2WFCyhnE\n" \ + "pKmossSNq6ZOqBjPegE0J6zfNg65dR/OlIdGVDgrVTIpwYAUzBMW2nTnCa00EmPj\n" \ + "F7tRscHI8qb/QlnRVEUN+S+A2CtVIH1c666zOoRFRI9G4bmVoa8k2x0ANB51tCns\n" \ + "vAYqkMybIgMvWwbqoAxeW0G1O3qObGXtgs94BzhAEM3RbG/hy3GR1qUNSk/qyDKc\n" \ + "t1qpiaao0aLVsnpb28eBIk6+q0I82reGdV31OYvUpnVxRbRPFXEFs5PNS3s/7I8a\n" \ + "SlSLUGOh+mhrUzDPSJCzgEvOmrwrRxe3F52tS0nAt6Z5zKToASHphoISUi7lGX1F\n" \ + "Owx62qhSqqlI98bKqh7yQRZYrHXqE0bscAHCcIaZ8RVya42JHDCoQWyxqBuLOWEl\n" \ + "+Fz6vI5DqEnJkA7ke49EvBAOJ58lxAXQIV5remtzYGPKdyG2oamiFHiLVQDzGX/l\n" \ + "aFNMGXRWcK4/Y3mnkJvx9QGtq6KstQN/J4a51ZeX5YwNBcoY9UcFS6kHRW5rR3UM\n" \ + "tEZj5VN8BL9nyWM9h7hUSHQboaxO7M5qswfXB8f21xR16T40Ki4nawx/6zHGCQsc\n" \ + "uKr5SaCV88tghqJYHBorU5iKB5KsLDSHqYYrNo/Vy8W6kMA2jGAO24d4G32DSshR\n" \ + "sEF9W1nuAHK/5ste01G5KmX2KhdZBE37oGhM98HRQ6hU8qwuKrhdV7vZis5C8LXY\n" \ + "7MbDyDt1NnFqWFc6lYeVa6eRcmYzeAbXahrxwiiaLIdHXD95aZ/0S6+tKBGgQzwm\n" \ + "ZsbwdXhl+n+yqDNE6Sow2bwueqhDwZVWoMCv5SK+HAGPtcZ7UU9oWrqpiL085m7F\n" \ + "5G49KJUEZadVtj4Z9zrkeQkida+4I7v3Y3MzsWsGJww7YhTDJpsxxmSm85bHwx98\n" \ + "hZXSqckJTL4c2nBzgrBlukIT9Wl+qItMthVvABPzp4wGZhdgKrEIRl3yCnhhUgpL\n" \ + "lUxYegwWDMEjZxKlSbIyl5p9lCS8w2lsBzsQ2FJiAy/MWLa56aA+wFs3C8smZ6Cf\n" \ + "p5NWa8Rm+k898GWBxZivhF03CBOZ42du0YUZdCPoA5V1KC6bh4JyWFI49VFbQFMG\n" \ + "gwAqc0ErAH3iMammKC9746WWagnUIG3o8LygZrusuGeTohXJhVUTJDw2s0rzNhbw\n" \ + "5IyookkY5BWENKFKTIgdBxvYelOKwbGE8Z36FEW0ABlmx7SRCKWlNVjSEAIXmMiQ\n" \ + "VLdQF33QVYD9RR5chja254VuJH4plo+5JwiKWz8LlCIBm7CVkifZMLofmMk3s3L4\n" \ + "sXtE+Bhfm5Plk3RrgDdlHH+hK7gk61XGdynGjDY7aLtCKZ0SMsVskSLom1pbIR5M\n" \ + "KLYsQ1Pse4mhfDOFCkWFLI5TShGMuIoo1k7XeIE6g8QoUlV5EXyWHHhIVaE4yWGP\n" \ + "AVgEp0UswKFeeo3SoCAeADA3U88ymxpBJp73yDIqok5dM3SgkjfPWZDkgkAI8WHs\n" \ + "CKKeqrSOs1kkE3JXtE7kcTHT6XHo162TmgGkqMVwOQ3EmR6FRpYxJhZvuVbjJsSx\n" \ + "YjW3ScnR4Zivoi7q95ypco331pIlIZpqV0NydUpMyQaz1cnoPKYDh1xa6LhcqEKK\n" \ + "8a68iXjQgzgqQBDABonVybNDtlJ5lnTTuKhak8PBFAmmhj1JdrPqoIvQRCmLaark\n" \ + "J7/q9RLtk6kTOJ0qtLe2qqwCxJwyoMd2Q5F4+xTWZHu90ljRdcnYewarqcKzoL27\n" \ + "tcpTOmVz88I1hYVUJEV7aB36QMhTS1dquTqJZCD0hBPWAMToEoD4OFvKWmbFmzaW\n" \ + "xrMc4ECYeDAAKYs2YqoXSLfAixBmZjb6UDB61l2GA58pFJW0ZwN8S5tApA2NRi+7\n" \ + "oC/zgMgBGHft6E0+OUVb8It89pY1t7ybq5+fkBvEixDId3f1pK3gqcaYqG/YhoMJ\n" \ + "MJWkqYxCNGmdZ8gFo46V6K+4xZUblQWKypN6+RYO4kDh0koppWGEULjgBoCH+V8E\n" \ + "7GcoE8SRdQY1BIMoRVWb8Ur8ZYIVU8lqgaZPlWM3oRCiWk0kRxexFF0i5WlILIK9\n" \ + "GT8saX+bmRd9KSy3JrpPhQn59CpJBRxz8WKdJ3wwtqE/2TbxQhLooEWHYVrZEG5E\n" \ + "SkIoOkUAJUR+CzLLFDMdUE8w3CasE4ys+hco7AA5TAms24A1FXcxMgNb6VHA0bi5\n" \ + "c8rPCZvjubLXR4A0/A2Ualo4cy3UAr9k0rbZOJnjqk8eExkeaxbyh42cJpU75i4O\n" \ + "NLYsRZJkg9bkCpPgZKb707sPZO72CX3h/lQdXVgGkZ7Tqd1qzM+JOhSWvrYiBLa+\n" \ + "5IKSmFwT+5sw1InEesXwRN09000U90vAkbZG/sZqBQHFk3xM23a+aad4NggUcfgD\n" \ + "108=\n" \ + "-----END PRIVATE KEY-----\n" + +#define MLKEM768_PUBKEY_PEM \ + "-----BEGIN PUBLIC KEY-----\n" \ + "MIIEsjALBglghkgBZQMEBAIDggShAMPIO3U2cWpYVzqVh5Vrp5FyZjN4BtdqGvHC\n" \ + "KJosh0dcP3lpn/RLr60oEaBDPCZmxvB1eGX6f7KoM0TpKjDZvC56qEPBlVagwK/l\n" \ + "Ir4cAY+1xntRT2hauqmIvTzmbsXkbj0olQRlp1W2Phn3OuR5CSJ1r7gju/djczOx\n" \ + "awYnDDtiFMMmmzHGZKbzlsfDH3yFldKpyQlMvhzacHOCsGW6QhP1aX6oi0y2FW8A\n" \ + "E/OnjAZmF2AqsQhGXfIKeGFSCkuVTFh6DBYMwSNnEqVJsjKXmn2UJLzDaWwHOxDY\n" \ + "UmIDL8xYtrnpoD7AWzcLyyZnoJ+nk1ZrxGb6Tz3wZYHFmK+EXTcIE5njZ27RhRl0\n" \ + "I+gDlXUoLpuHgnJYUjj1UVtAUwaDACpzQSsAfeIxqaYoL3vjpZZqCdQgbejwvKBm\n" \ + "u6y4Z5OiFcmFVRMkPDazSvM2FvDkjKiiSRjkFYQ0oUpMiB0HG9h6U4rBsYTxnfoU\n" \ + "RbQAGWbHtJEIpaU1WNIQAheYyJBUt1AXfdBVgP1FHlyGNrbnhW4kfimWj7knCIpb\n" \ + "PwuUIgGbsJWSJ9kwuh+YyTezcvixe0T4GF+bk+WTdGuAN2Ucf6EruCTrVcZ3KcaM\n" \ + "Njtou0IpnRIyxWyRIuibWlshHkwotixDU+x7iaF8M4UKRYUsjlNKEYy4iijWTtd4\n" \ + "gTqDxChSVXkRfJYceEhVoTjJYY8BWASnRSzAoV56jdKgIB4AMDdTzzKbGkEmnvfI\n" \ + "MiqiTl0zdKCSN89ZkOSCQAjxYewIop6qtI6zWSQTcle0TuRxMdPpcejXrZOaAaSo\n" \ + "xXA5DcSZHoVGljEmFm+5VuMmxLFiNbdJydHhmK+iLur3nKlyjffWkiUhmmpXQ3J1\n" \ + "SkzJBrPVyeg8pgOHXFrouFyoQorxrryJeNCDOCpAEMAGidXJs0O2UnmWdNO4qFqT\n" \ + "w8EUCaaGPUl2s+qgi9BEKYtpquQnv+r1Eu2TqRM4nSq0t7aqrALEnDKgx3ZDkXj7\n" \ + "FNZke73SWNF1ydh7BqupwrOgvbu1ylM6ZXPzwjWFhVQkRXtoHfpAyFNLV2q5Oolk\n" \ + "IPSEE9YAxOgSgPg4W8paZsWbNpbGsxzgQJh4MAApizZiqhdIt8CLEGZmNvpQMHrW\n" \ + "XYYDnykUlbRnA3xLm0CkDY1GL7ugL/OAyAEYd+3oTT45RVvwi3z2ljW3vJurn5+Q\n" \ + "G8SLEMh3d/WkreCpxpiob9iGgwkwlaSpjEI0aZ1nyAWjjpXor7jFlRuVBYrKk3r5\n" \ + "Fg7iQOHSSimlYYRQuOAGgIf5XwTsZygTxJF1BjUEgyhFVZvxSvxlghVTyWqBpk+V\n" \ + "YzehEKJaTSRHF7EUXSLlaUgsgr0ZPyxpf5uZF30pLLcmuk+FCfn0KkkFHHPxYp0n\n" \ + "fDC2oT/ZNvFCEuigRYdhWtkQbkRKQig6RQAlRH4LMssUMx1QTzDcJqwTjKz6Fyjs\n" \ + "ADlMCazbgDUVdzEyA1vpUcDRuLlzys8Jm+O5stdHgDT8DZRqWjhzLdQCv2TSttk4\n" \ + "meOqTx4TGR5rFvKHjZwmlTvmLg40tixFkmSD1uQKk+BkpvvTuw9k7vYJfeH+VB1d\n" \ + "WAaRntOp\n" \ + "-----END PUBLIC KEY-----\n" + +/* Helper macro to open string constant as FILE stream */ +#define FMEMOPEN_STR(str) fmemopen((void *) (str), strlen(str), "r") + +static int test_kex_load_kem_privkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_privkey_str(MLKEM768_PRIVKEY_PEM, &key) < 0) { + printf("Failed to load ML-KEM-768 private key.\n"); + goto fail; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_load_kem_pubkey(void) +{ + void * key; + + TEST_START(); + + if (crypt_load_pubkey_str(MLKEM768_PUBKEY_PEM, &key) < 0) { + printf("Failed to load ML-KEM-768 public key.\n"); + goto fail; + } + + crypt_free_key(key); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_kex_kem(const char * algo) +{ + struct sec_config kex; + void * pkp; + buffer_t pk; + buffer_t ct; + ssize_t len; + ssize_t ct_len; + uint8_t buf1[CRYPT_KEY_BUFSZ]; + uint8_t buf2[CRYPT_KEY_BUFSZ]; + uint8_t s1[SYMMKEYSZ]; + uint8_t s2[SYMMKEYSZ]; + int kdf; + + TEST_START("(%s)", algo); + + kdf = get_random_kdf(); + + memset(&kex, 0, sizeof(kex)); + SET_KEX_ALGO(&kex, algo); + + len = kex_pkp_create(&kex, &pkp, buf1); + if (len < 0) { + printf("Failed to create key pair for %s.\n", algo); + goto fail; + } + + pk.len = (size_t) len; + pk.data = buf1; + + if (IS_HYBRID_KEM(algo)) + ct_len = kex_kem_encap_raw(pk, buf2, kdf, s1); + else + ct_len = kex_kem_encap(pk, buf2, kdf, s1); + + if (ct_len < 0) { + printf("Failed to encapsulate for %s.\n", algo); + goto fail_pkp; + } + + ct.len = (size_t) ct_len; + ct.data = buf2; + + if (kex_kem_decap(pkp, ct, kdf, s2) < 0) { + printf("Failed to decapsulate for %s.\n", algo); + goto fail_pkp; + } + + if (memcmp(s1, s2, SYMMKEYSZ) != 0) { + printf("Shared secrets don't match for %s.\n", algo); + goto fail_pkp; + } + + kex_pkp_destroy(pkp); + + TEST_SUCCESS("(%s)", algo); + + return TEST_RC_SUCCESS; + fail_pkp: + kex_pkp_destroy(pkp); + fail: + TEST_FAIL("(%s)", algo); + return TEST_RC_FAIL; +} + +static int test_kex_kem_corrupted_ciphertext(const char * algo) +{ + struct sec_config kex; + void * pkp; + buffer_t pk; + buffer_t ct; + ssize_t len; + ssize_t ct_len; + uint8_t buf1[CRYPT_KEY_BUFSZ]; + uint8_t buf2[CRYPT_KEY_BUFSZ]; + uint8_t s1[SYMMKEYSZ]; + uint8_t s2[SYMMKEYSZ]; + int kdf; + + TEST_START("(%s)", algo); + + kdf = get_random_kdf(); + + memset(&kex, 0, sizeof(kex)); + SET_KEX_ALGO(&kex, algo); + + len = kex_pkp_create(&kex, &pkp, buf1); + if (len < 0) { + printf("Failed to create key pair.\n"); + goto fail; + } + + pk.len = (size_t) len; + pk.data = buf1; + + if (IS_HYBRID_KEM(algo)) + ct_len = kex_kem_encap_raw(pk, buf2, kdf, s1); + else + ct_len = kex_kem_encap(pk, buf2, kdf, s1); + + if (ct_len < 0) { + printf("Failed to encapsulate.\n"); + goto fail_pkp; + } + + ct.len = (size_t) ct_len; + ct.data = buf2; + + /* Corrupt the ciphertext */ + buf2[0] ^= 0xFF; + buf2[ct_len - 1] ^= 0xFF; + + /* ML-KEM uses implicit rejection */ + if (kex_kem_decap(pkp, ct, kdf, s2) < 0) { + printf("Decapsulation failed unexpectedly.\n"); + goto fail_pkp; + } + + /* The shared secrets should NOT match with corrupted CT */ + if (memcmp(s1, s2, SYMMKEYSZ) == 0) { + printf("Corrupted ciphertext produced same secret.\n"); + goto fail_pkp; + } + + kex_pkp_destroy(pkp); + + TEST_SUCCESS("(%s)", algo); + + return TEST_RC_SUCCESS; + fail_pkp: + kex_pkp_destroy(pkp); + fail: + TEST_FAIL("(%s)", algo); + return TEST_RC_FAIL; +} + +static int test_kex_kem_wrong_keypair(const char * algo) +{ + struct sec_config kex; + void * pkp1; + void * pkp2; + buffer_t pk1; + buffer_t ct; + ssize_t len; + ssize_t ct_len; + uint8_t buf1[CRYPT_KEY_BUFSZ]; + uint8_t buf2[CRYPT_KEY_BUFSZ]; + uint8_t buf3[CRYPT_KEY_BUFSZ]; + uint8_t s1[SYMMKEYSZ]; + uint8_t s2[SYMMKEYSZ]; + + TEST_START("(%s)", algo); + + memset(&kex, 0, sizeof(kex)); + SET_KEX_ALGO(&kex, algo); + + len = kex_pkp_create(&kex, &pkp1, buf1); + if (len < 0) { + printf("Failed to create first key pair.\n"); + goto fail; + } + + pk1.len = (size_t) len; + pk1.data = buf1; + + if (kex_pkp_create(&kex, &pkp2, buf2) < 0) { + printf("Failed to create second key pair.\n"); + goto fail_pkp1; + } + + if (IS_HYBRID_KEM(algo)) + ct_len = kex_kem_encap_raw(pk1, buf3, NID_sha256, s1); + else + ct_len = kex_kem_encap(pk1, buf3, NID_sha256, s1); + + if (ct_len < 0) { + printf("Failed to encapsulate.\n"); + goto fail_pkp2; + } + + ct.len = (size_t) ct_len; + ct.data = buf3; + + if (kex_kem_decap(pkp2, ct, NID_sha256, s2) == 0) { + if (memcmp(s1, s2, SYMMKEYSZ) == 0) { + printf("Wrong keypair produced same secret.\n"); + goto fail_pkp2; + } + } + + kex_pkp_destroy(pkp2); + kex_pkp_destroy(pkp1); + + TEST_SUCCESS("(%s)", algo); + + return TEST_RC_SUCCESS; + fail_pkp2: + kex_pkp_destroy(pkp2); + fail_pkp1: + kex_pkp_destroy(pkp1); + fail: + TEST_FAIL("(%s)", algo); + return TEST_RC_FAIL; +} + +static int test_kex_kem_truncated_ciphertext(const char * algo) +{ + struct sec_config kex; + void * pkp; + buffer_t pk; + buffer_t ct; + ssize_t len; + ssize_t ct_len; + uint8_t buf1[CRYPT_KEY_BUFSZ]; + uint8_t buf2[CRYPT_KEY_BUFSZ]; + uint8_t s1[SYMMKEYSZ]; + uint8_t s2[SYMMKEYSZ]; + + TEST_START("(%s)", algo); + + memset(&kex, 0, sizeof(kex)); + SET_KEX_ALGO(&kex, algo); + + len = kex_pkp_create(&kex, &pkp, buf1); + if (len < 0) { + printf("Failed to create key pair.\n"); + goto fail; + } + + pk.len = (size_t) len; + pk.data = buf1; + + if (IS_HYBRID_KEM(algo)) + ct_len = kex_kem_encap_raw(pk, buf2, NID_sha256, s1); + else + ct_len = kex_kem_encap(pk, buf2, NID_sha256, s1); + + if (ct_len < 0) { + printf("Failed to encapsulate.\n"); + goto fail_pkp; + } + + /* Truncate the ciphertext */ + ct.len = (size_t) ct_len / 2; + ct.data = buf2; + + if (kex_kem_decap(pkp, ct, NID_sha256, s2) == 0) { + printf("Should fail with truncated ciphertext.\n"); + goto fail_pkp; + } + + kex_pkp_destroy(pkp); + + TEST_SUCCESS("(%s)", algo); + + return TEST_RC_SUCCESS; + fail_pkp: + kex_pkp_destroy(pkp); + fail: + TEST_FAIL("(%s)", algo); + return TEST_RC_FAIL; +} + +static int test_kex_kem_all(void) +{ + int ret = 0; + int i; + + for (i = 0; kex_supported_nids[i] != NID_undef; i++) { + const char * algo = kex_nid_to_str(kex_supported_nids[i]); + + if (!IS_KEM_ALGORITHM(algo)) + continue; + + ret |= test_kex_kem(algo); + } + + return ret; +} + +static int test_kex_kem_corrupted_ciphertext_all(void) +{ + int ret = 0; + int i; + + for (i = 0; kex_supported_nids[i] != NID_undef; i++) { + const char * algo = kex_nid_to_str(kex_supported_nids[i]); + + if (!IS_KEM_ALGORITHM(algo)) + continue; + + ret |= test_kex_kem_corrupted_ciphertext(algo); + } + + return ret; +} + +static int test_kex_kem_wrong_keypair_all(void) +{ + int ret = 0; + int i; + + for (i = 0; kex_supported_nids[i] != NID_undef; i++) { + const char * algo = kex_nid_to_str(kex_supported_nids[i]); + + if (!IS_KEM_ALGORITHM(algo)) + continue; + + ret |= test_kex_kem_wrong_keypair(algo); + } + + return ret; +} + +static int test_kex_kem_truncated_ciphertext_all(void) +{ + int ret = 0; + int i; + + for (i = 0; kex_supported_nids[i] != NID_undef; i++) { + const char * algo = kex_nid_to_str(kex_supported_nids[i]); + + if (!IS_KEM_ALGORITHM(algo)) + continue; + + ret |= test_kex_kem_truncated_ciphertext(algo); + } + + return ret; +} + +int kex_test_ml_kem(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + +#ifdef HAVE_OPENSSL_ML_KEM + ret |= test_kex_load_kem_privkey(); + ret |= test_kex_load_kem_pubkey(); + ret |= test_kex_kem_all(); + ret |= test_kex_kem_corrupted_ciphertext_all(); + ret |= test_kex_kem_wrong_keypair_all(); + ret |= test_kex_kem_truncated_ciphertext_all(); +#else + (void) test_kex_load_kem_privkey; + (void) test_kex_load_kem_pubkey; + (void) test_kex_kem_all; + (void) test_kex_kem_corrupted_ciphertext_all; + (void) test_kex_kem_wrong_keypair_all; + (void) test_kex_kem_truncated_ciphertext_all; + + ret = TEST_RC_SKIP; +#endif + return ret; +} diff --git a/src/lib/tests/md5_test.c b/src/lib/tests/md5_test.c index 00497ae7..ea3e12b3 100644 --- a/src/lib/tests/md5_test.c +++ b/src/lib/tests/md5_test.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Test of the MD5 function * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/src/lib/tests/sha3_test.c b/src/lib/tests/sha3_test.c index 945402ee..ccd4e12a 100644 --- a/src/lib/tests/sha3_test.c +++ b/src/lib/tests/sha3_test.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Test of the SHA3 function * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/src/lib/tests/shm_rbuff_test.c b/src/lib/tests/shm_rbuff_test.c deleted file mode 100644 index 1d16a09d..00000000 --- a/src/lib/tests/shm_rbuff_test.c +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * Test of the shm_rbuff - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -#define _POSIX_C_SOURCE 200112L - -#include "config.h" - -#include <ouroboros/shm_rbuff.h> - -#include <errno.h> -#include <stdio.h> -#include <unistd.h> - -int shm_rbuff_test(int argc, - char ** argv) -{ - struct shm_rbuff * rb; - size_t i; - - (void) argc; - (void) argv; - - printf("Test: create rbuff..."); - - rb = shm_rbuff_create(getpid(), 1); - if (rb == NULL) - goto err; - - printf("success.\n\n"); - printf("Test: write a value..."); - - if (shm_rbuff_write(rb, 1) < 0) - goto error; - - printf("success.\n\n"); - printf("Test: check queue length is 1..."); - - if (shm_rbuff_queued(rb) != 1) - goto error; - - printf("success.\n\n"); - printf("Test: read a value..."); - - if (shm_rbuff_read(rb) != 1) - goto error; - - printf("success.\n\n"); - printf("Test: check queue is empty..."); - - if (shm_rbuff_read(rb) != -EAGAIN) - goto error; - - printf("success.\n\n"); - printf("Test: fill the queue..."); - - for (i = 0; i < SHM_RBUFF_SIZE - 1; ++i) { - if (shm_rbuff_queued(rb) != i) - goto error; - if (shm_rbuff_write(rb, 1) < 0) - goto error; - } - - printf("success.\n\n"); - printf("Test: check queue is full..."); - - if (shm_rbuff_queued(rb) != SHM_RBUFF_SIZE - 1) - goto error; - - printf("success [%zd entries].\n\n", shm_rbuff_queued(rb)); - - printf("Test: check queue is full by writing value..."); - if (!(shm_rbuff_write(rb, 1) < 0)) - goto error; - - printf("success [%zd entries].\n\n", shm_rbuff_queued(rb)); - - /* empty the rbuff */ - while (shm_rbuff_read(rb) >= 0) - ; - - shm_rbuff_destroy(rb); - - return 0; - - error: - /* empty the rbuff */ - while (shm_rbuff_read(rb) >= 0) - ; - - shm_rbuff_destroy(rb); - err: - printf("failed.\n\n"); - return -1; -} diff --git a/src/lib/tests/sockets_test.c b/src/lib/tests/sockets_test.c new file mode 100644 index 00000000..c00bfdc1 --- /dev/null +++ b/src/lib/tests/sockets_test.c @@ -0,0 +1,102 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Tests for socket.c + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#define _DEFAULT_SOURCE +#else +#define _POSIX_C_SOURCE 200112L +#endif + +#include <ouroboros/sockets.h> +#include <test/test.h> + +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> + +#define TEST_PID 1234 +#define TEST_PID_STR "1234" +#define TEST_SERVER_PATH "/tmp/test.sock" +#define TEST_SERVER_PREFIX "/tmp/ouroboros/test." +#define TEST_SOCK_PATH_PREFIX "var/run/ouroboros/test." + +static int test_sock_path(void) +{ + char * path; + char * exp = TEST_SOCK_PATH_PREFIX TEST_PID_STR SOCK_PATH_SUFFIX; + + TEST_START(); + + path = sock_path(TEST_PID, TEST_SOCK_PATH_PREFIX); + if (path == NULL) { + printf("Path is NULL.\n"); + goto fail_path; + } + + if (strcmp(path, exp) != 0) { + printf("Expected path '%s', got '%s'.\n", exp, path); + goto fail_cmp; + } + + free(path); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + fail_cmp: + free(path); + fail_path: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_server_socket_open(void) +{ + int sockfd; + + TEST_START(); + + sockfd = server_socket_open(TEST_SERVER_PATH); + if (sockfd < 0) { + printf("Failed to open server socket.\n"); + goto fail_sock; + } + + close(sockfd); + + unlink(TEST_SERVER_PATH); + + TEST_SUCCESS(); + return TEST_RC_SUCCESS; + fail_sock: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int sockets_test(void) +{ + int ret = 0; + + ret |= test_sock_path(); + ret |= test_server_socket_open(); + + return ret; +} diff --git a/src/lib/tests/time_test.c b/src/lib/tests/time_test.c new file mode 100644 index 00000000..919cf075 --- /dev/null +++ b/src/lib/tests/time_test.c @@ -0,0 +1,529 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Test of the time utilities + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#define _POSIX_C_SOURCE 200809L + +#include <test/test.h> +#include <ouroboros/time.h> + +#include <stdio.h> + +static int ts_check(struct timespec * s, + time_t sec, + time_t nsec) +{ + return s->tv_sec == sec && s->tv_nsec == nsec; +} + +static int tv_check(struct timeval * v, + time_t sec, + time_t usec) +{ + return v->tv_sec == sec && v->tv_usec == usec; +} + + +static int test_time_ts_init(void) +{ + struct timespec s = TIMESPEC_INIT_S (100); + struct timespec ms = TIMESPEC_INIT_MS(100); + struct timespec us = TIMESPEC_INIT_US(100); + struct timespec ns = TIMESPEC_INIT_NS(100); + + TEST_START(); + + if (!ts_check(&s, 100, 0)) { + printf("timespec_init_s failed.\n"); + goto fail; + } + + if (!ts_check(&ms, 0, 100 * MILLION)) { + printf("timespec_init_ms failed.\n"); + goto fail; + } + + if (!ts_check(&us, 0, 100* 1000L)) { + printf("timespec_init_us failed.\n"); + goto fail; + } + + if (!ts_check(&ns, 0, 100)) { + printf("timespec_init_ns failed.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_time_tv_init(void) +{ + struct timeval s = TIMEVAL_INIT_S (100); + struct timeval ms = TIMEVAL_INIT_MS(100); + struct timeval us = TIMEVAL_INIT_US(100); + + TEST_START(); + + if (!tv_check(&s, 100, 0)) { + printf("timeval_init_s failed.\n"); + goto fail; + } + + if (!tv_check(&ms, 0, 100 * 1000L)) { + printf("timeval_init_ms failed.\n"); + goto fail; + } + + if (!tv_check(&us, 0, 100)) { + printf("timeval_init_us failed.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ts_diff(void) +{ + struct timespec s0 = TIMESPEC_INIT_S (100); + struct timespec s1 = TIMESPEC_INIT_S (200); + struct timespec ms0 = TIMESPEC_INIT_MS(100); + struct timespec ms1 = TIMESPEC_INIT_MS(200); + struct timespec us0 = TIMESPEC_INIT_US(100); + struct timespec us1 = TIMESPEC_INIT_US(200); + struct timespec ns0 = TIMESPEC_INIT_NS(100); + struct timespec ns1 = TIMESPEC_INIT_NS(200); + struct timespec res; + + TEST_START(); + + ts_diff(&s0, &s1, &res); + if (!ts_check(&res, -100, 0)) { + printf("timespec_diff failed at s0 - s1.\n"); + goto fail; + } + + ts_diff(&s1, &s0, &res); + if (!ts_check(&res, 100, 0)) { + printf("timespec_diff failed at s1 - s0.\n"); + goto fail; + } + + ts_diff(&ms0, &ms1, &res); + if (!ts_check(&res, -1, 900 * MILLION)) { + printf("timespec_diff failed at ms0 - ms1.\n"); + goto fail; + } + + ts_diff(&ms1, &ms0, &res); + if (!ts_check(&res, 0, 100 * MILLION)) { + printf("timespec_diff failed at ms1 - ms0.\n"); + goto fail; + } + + ts_diff(&us0, &us1, &res); + if (!ts_check(&res, -1, 999900 * 1000L)) { + printf("timespec_diff failed at us0 - us1.\n"); + goto fail; + } + + ts_diff(&us1, &us0, &res); + if (!ts_check(&res, 0, 100 * 1000L)) { + printf("timespec_diff failed at us1 - us0.\n"); + goto fail; + } + + ts_diff(&ns0, &ns1, &res); + if (!ts_check(&res, -1, 999999900)) { + printf("timespec_diff failed at ns0 - ns1.\n"); + goto fail; + } + + ts_diff(&ns1, &ns0, &res); + if (!ts_check(&res, 0, 100)) { + printf("timespec_diff failed at ns1 - ns0.\n"); + goto fail; + } + + ts_diff(&s0, &ms0, &res); + if (!ts_check(&res, 99, 900 * MILLION)) { + printf("timespec_diff failed at s0 - ms0.\n"); + goto fail; + } + + ts_diff(&s0, &us0, &res); + if (!ts_check(&res, 99, 999900 * 1000L)) { + printf("timespec_diff failed at s0 - us0.\n"); + goto fail; + } + + ts_diff(&s0, &ns0, &res); + if (!ts_check(&res, 99, 999999900)) { + printf("timespec_diff failed at s0 - ns0.\n"); + goto fail; + } + + ts_diff(&ms0, &us0, &res); + if (!ts_check(&res, 0, 99900 * 1000L)) { + printf("timespec_diff failed at ms0 - us0.\n"); + goto fail; + } + + ts_diff(&ms0, &ns0, &res); + if (!ts_check(&res, 0, 99999900)) { + printf("timespec_diff failed at ms0 - ns0.\n"); + goto fail; + } + + ts_diff(&us0, &ns0, &res); + if (!ts_check(&res, 0, 99900)) { + printf("timespec_diff failed at us0 - ns0.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tv_diff(void) +{ + struct timeval s0 = TIMEVAL_INIT_S (100); + struct timeval s1 = TIMEVAL_INIT_S (200); + struct timeval ms0 = TIMEVAL_INIT_MS(100); + struct timeval ms1 = TIMEVAL_INIT_MS(200); + struct timeval us0 = TIMEVAL_INIT_US(100); + struct timeval us1 = TIMEVAL_INIT_US(200); + struct timeval res; + + TEST_START(); + + tv_diff(&s0, &s1, &res); + if (!tv_check(&res, -100, 0)) { + printf("timeval_diff failed at s0 - s1.\n"); + goto fail; + } + + tv_diff(&s1, &s0, &res); + if (!tv_check(&res, 100, 0)) { + printf("timeval_diff failed at s1 - s0.\n"); + goto fail; + } + + tv_diff(&ms0, &ms1, &res); + if (!tv_check(&res, -1, 900 * 1000L)) { + printf("timeval_diff failed at ms0 - ms1.\n"); + goto fail; + } + + tv_diff(&ms1, &ms0, &res); + if (!tv_check(&res, 0, 100 * 1000L)) { + printf("timeval_diff failed at ms1 - ms0.\n"); + goto fail; + } + + tv_diff(&us0, &us1, &res); + if (!tv_check(&res, -1, 999900)) { + printf("timeval_diff failed at us0 - us1.\n"); + goto fail; + } + + tv_diff(&us1, &us0, &res); + if (!tv_check(&res, 0, 100)) { + printf("timeval_diff failed at us1 - us0.\n"); + goto fail; + } + + tv_diff(&s0, &ms0, &res); + if (!tv_check(&res, 99, 900 * 1000L)) { + printf("timeval_diff failed at s0 - ms0.\n"); + goto fail; + } + + tv_diff(&s0, &us0, &res); + if (!tv_check(&res, 99, 999900)) { + printf("timeval_diff failed at s0 - us0.\n"); + goto fail; + } + + tv_diff(&ms0, &us0, &res); + if (!tv_check(&res, 0, 99900)) { + printf("timeval_diff failed at ms0 - us0.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_ts_diff_time(void) +{ + struct timespec s0 = TIMESPEC_INIT_S (100); + struct timespec s1 = TIMESPEC_INIT_S (200); + struct timespec ms0 = TIMESPEC_INIT_MS(100); + struct timespec ms1 = TIMESPEC_INIT_MS(200); + struct timespec us0 = TIMESPEC_INIT_US(100); + struct timespec us1 = TIMESPEC_INIT_US(200); + struct timespec ns0 = TIMESPEC_INIT_NS(100); + struct timespec ns1 = TIMESPEC_INIT_NS(200); + + TEST_START(); + + if (ts_diff_ms(&s0, &s1) != -100 * 1000L) { + printf("timespec_diff_ms failed at s0 - s1.\n"); + goto fail; + } + + if (ts_diff_ms(&s1, &s0) != 100 * 1000L) { + printf("timespec_diff_ms failed at s1 - s0.\n"); + goto fail; + } + + if (ts_diff_us(&s0, &s1) != -100 * MILLION) { + printf("timespec_diff_us failed at s1 - s0.\n"); + goto fail; + } + + if (ts_diff_us(&s1, &s0) != 100 * MILLION) { + printf("timespec_diff_us failed at s0 - s1.\n"); + goto fail; + } + + if (ts_diff_ns(&s0, &s1) != -100 * BILLION) { + printf("timespec_diff_ns failed at s0 - s1.\n"); + goto fail; + } + + if (ts_diff_ns(&s1, &s0) != 100 * BILLION) { + printf("timespec_diff_ns failed at s1 - s0.\n"); + goto fail; + } + + if (ts_diff_ms(&ms0, &ms1) != -100) { + printf("timespec_diff_ms failed at ms0 - ms1.\n"); + goto fail; + } + + if (ts_diff_ms(&ms1, &ms0) != 100) { + printf("timespec_diff_ms failed at ms1 - ms0.\n"); + goto fail; + } + + if (ts_diff_us(&ms0, &ms1) != -100 * 1000L) { + printf("timespec_diff_us failed at ms0 - ms1.\n"); + goto fail; + } + + if (ts_diff_us(&ms1, &ms0) != 100 * 1000L) { + printf("timespec_diff_us failed at ms1 - ms0.\n"); + goto fail; + } + + if (ts_diff_ns(&ms0, &ms1) != -100 * MILLION) { + printf("timespec_diff_ns failed at ms0 - ms1.\n"); + goto fail; + } + + if (ts_diff_ns(&ms1, &ms0) != 100 * MILLION) { + printf("timespec_diff_ns failed at ms1 - ms0.\n"); + goto fail; + } + + if (ts_diff_ms(&us0, &us1) != 0) { + printf("timespec_diff_ms failed at us0 - us1.\n"); + goto fail; + } + + if (ts_diff_ms(&us1, &us0) != 0) { + printf("timespec_diff_ms failed at us1 - us0.\n"); + goto fail; + } + + if (ts_diff_us(&us0, &us1) != -100) { + printf("timespec_diff_us failed at us0 - us1.\n"); + goto fail; + } + + if (ts_diff_us(&us1, &us0) != 100) { + printf("timespec_diff_us failed at us1 - us0.\n"); + goto fail; + } + + if (ts_diff_ns(&us0, &us1) != -100 * 1000L) { + printf("timespec_diff_ns failed at us0 - us1.\n"); + goto fail; + } + + if (ts_diff_ns(&us1, &us0) != 100 * 1000L) { + printf("timespec_diff_ns failed at us1 - us0.\n"); + goto fail; + } + + if (ts_diff_ms(&ns0, &ns1) != 0) { + printf("timespec_diff_ms failed at ns0 - ns1.\n"); + goto fail; + } + + if (ts_diff_ms(&ns1, &ns0) != 0) { + printf("timespec_diff_ms failed at ns1 - ns0.\n"); + goto fail; + } + + if (ts_diff_us(&ns0, &ns1) != 0) { + printf("timespec_diff_us failed at ns0 - ns1.\n"); + goto fail; + } + + if (ts_diff_us(&ns1, &ns0) != 0) { + printf("timespec_diff_us failed at ns1 - ns0.\n"); + goto fail; + } + + if (ts_diff_ns(&ns0, &ns1) != -100) { + printf("timespec_diff_ns failed at ns0 - ns1.\n"); + goto fail; + } + + if (ts_diff_ns(&ns1, &ns0) != 100) { + printf("timespec_diff_ns failed at ns1 - ns0.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tv_diff_time(void) +{ + struct timeval s0 = TIMEVAL_INIT_S (100); + struct timeval s1 = TIMEVAL_INIT_S (200); + struct timeval ms0 = TIMEVAL_INIT_MS(100); + struct timeval ms1 = TIMEVAL_INIT_MS(200); + struct timeval us0 = TIMEVAL_INIT_US(100); + struct timeval us1 = TIMEVAL_INIT_US(200); + + TEST_START(); + + if (tv_diff_ms(&s0, &s1) != -100 * 1000L) { + printf("timeval_diff_ms failed at s0 - s1.\n"); + goto fail; + } + + if (tv_diff_ms(&s1, &s0) != 100 * 1000L) { + printf("timeval_diff_ms failed at s1 - s0.\n"); + goto fail; + } + + if (tv_diff_us(&s0, &s1) != -100 * MILLION) { + printf("timeval_diff_us failed at s0 - s1.\n"); + goto fail; + } + + if (tv_diff_us(&s1, &s0) != 100 * MILLION) { + printf("timeval_diff_us failed at s1 - s0.\n"); + goto fail; + } + + if (tv_diff_ms(&ms0, &ms1) != -100) { + printf("timeval_diff_ms failed at ms0 - ms1.\n"); + goto fail; + } + + if (tv_diff_ms(&ms1, &ms0) != 100) { + printf("timeval_diff_ms failed at ms1 - ms0.\n"); + goto fail; + } + + if (tv_diff_us(&ms0, &ms1) != -100 * 1000L) { + printf("timeval_diff_us failed at ms0 - ms1.\n"); + goto fail; + } + + if (tv_diff_us(&ms1, &ms0) != 100 * 1000L) { + printf("timeval_diff_us failed at ms1 - ms0.\n"); + goto fail; + } + + if (tv_diff_ms(&us0, &us1) != 0) { + printf("timeval_diff_ms failed at us0 - us1.\n"); + goto fail; + } + + if (tv_diff_ms(&us1, &us0) != 0) { + printf("timeval_diff_ms failed at us1 - us0.\n"); + goto fail; + } + + if (tv_diff_us(&us0, &us1) != -100) { + printf("timeval_diff_us failed at us0 - us1.\n"); + goto fail; + } + + if (tv_diff_us(&us1, &us0) != 100) { + printf("timeval_diff_us failed at us1 - us0.\n"); + goto fail; + } + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int time_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_time_ts_init(); + ret |= test_time_tv_init(); + ret |= test_ts_diff(); + ret |= test_tv_diff(); + ret |= test_ts_diff_time(); + ret |= test_tv_diff_time(); + + return ret; +} diff --git a/src/lib/tests/time_utils_test.c b/src/lib/tests/time_utils_test.c deleted file mode 100644 index c384d272..00000000 --- a/src/lib/tests/time_utils_test.c +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * Test of the time utilities - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -#define _POSIX_C_SOURCE 200809L - -#include <ouroboros/time_utils.h> - -#include <stdio.h> - -static void ts_print(struct timespec * s) -{ - printf("timespec is %zd:%ld.\n", (ssize_t) s->tv_sec, s->tv_nsec); -} - -static void tv_print(struct timeval * v) -{ - printf("timeval is %zd:%zu.\n", (ssize_t) v->tv_sec, (size_t) v->tv_usec); -} - -static void ts_init(struct timespec * s, - time_t sec, - time_t nsec) -{ - s->tv_sec = sec; - s->tv_nsec = nsec; -} - -static void tv_init(struct timeval * v, - time_t sec, - time_t usec) -{ - v->tv_sec = sec; - v->tv_usec = usec; -} - -static int ts_check(struct timespec * s, - time_t sec, - time_t nsec) -{ - return s->tv_sec == sec && s->tv_nsec == nsec; -} - -static int tv_check(struct timeval * v, - time_t sec, - time_t usec) -{ - return v->tv_sec == sec && v->tv_usec == usec; -} - -int time_utils_test(int argc, - char ** argv) -{ - struct timespec s0; - struct timespec s1; - struct timespec s2; - - struct timeval v0; - struct timeval v1; - struct timeval v2; - - (void) argc; - (void) argv; - - ts_init(&s0, 0, 0); - ts_init(&s1, 5, 0); - - ts_add(&s0, &s1, &s2); - if (!ts_check(&s2, 5, 0)) { - printf("ts_add failed.\n"); - ts_print(&s2); - return -1; - } - - tv_init(&v0, 0, 0); - tv_init(&v1, 5, 0); - - tv_add(&v0, &v1, &v2); - if (!tv_check(&v2, 5, 0)) { - printf("tv_add failed.\n"); - tv_print(&v2); - return -1; - } - - ts_init(&s0, 0, 500 * MILLION); - ts_init(&s1, 0, 600 * MILLION); - - ts_add(&s0, &s1, &s2); - if (!ts_check(&s2, 1, 100 * MILLION)) { - printf("ts_add with nano overflow failed.\n"); - ts_print(&s2); - return -1; - } - - tv_init(&v0, 0, 500 * 1000); - tv_init(&v1, 0, 600 * 1000); - - tv_add(&v0, &v1, &v2); - if (!tv_check(&v2, 1, 100 * 1000)) { - printf("tv_add with nano overflow failed.\n"); - tv_print(&v2); - return -1; - } - - ts_init(&s0, 0, 0); - ts_init(&s1, 5, 0); - - ts_diff(&s0, &s1, &s2); - if (!ts_check(&s2, -5, 0)) { - printf("ts_diff failed.\n"); - ts_print(&s2); - return -1; - } - - tv_init(&v0, 0, 0); - tv_init(&v1, 5, 0); - - tv_diff(&v0, &v1, &v2); - if (!tv_check(&v2, -5, 0)) { - printf("tv_diff failed.\n"); - tv_print(&v2); - return -1; - } - - ts_init(&s0, 0, 500 * MILLION); - ts_init(&s1, 0, 600 * MILLION); - - ts_diff(&s0, &s1, &s2); - if (!ts_check(&s2, -1, 900 * MILLION)) { - printf("ts_diff with nano underflow failed.\n"); - ts_print(&s2); - return -1; - } - - tv_init(&v0, 0, 500 * 1000); - tv_init(&v1, 0, 600 * 1000); - - tv_diff(&v0, &v1, &v2); - if (!tv_check(&v2, -1, 900 * 1000)) { - printf("tv_diff with nano underflow failed.\n"); - tv_print(&v2); - return -1; - } - - return 0; -} diff --git a/src/lib/tests/tpm_test.c b/src/lib/tests/tpm_test.c new file mode 100644 index 00000000..7cc049cd --- /dev/null +++ b/src/lib/tests/tpm_test.c @@ -0,0 +1,104 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Tests for the threadpool manager + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + + +#include <ouroboros/tpm.h> + +#include <test/test.h> + +static void * test_func(void * o) +{ + (void) o; + + while(1) + sleep(1); + + return NULL; +} + +static int test_tpm_create_destroy(void) +{ + struct tpm *tpm; + + TEST_START(); + + tpm = tpm_create(2, 2, &test_func, NULL); + if (tpm == NULL) { + printf("Failed to initialize TPM.\n"); + goto fail; + } + + tpm_destroy(tpm); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tpm_start_stop(void * (* fn)(void *), + void * o) +{ + struct tpm *tpm; + + TEST_START(); + + tpm = tpm_create(2, 2, fn, o); + if (tpm == NULL) { + printf("Failed to initialize TPM.\n"); + goto fail_create; + } + + if (tpm_start(tpm) < 0) { + printf("Failed to start TPM.\n"); + goto fail_start; + } + + tpm_stop(tpm); + + tpm_destroy(tpm); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_start: + tpm_destroy(tpm); + fail_create: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int tpm_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_tpm_create_destroy(); + ret |= test_tpm_start_stop(&test_func, NULL); + + return ret; +} diff --git a/src/lib/tests/tw_test.c b/src/lib/tests/tw_test.c new file mode 100644 index 00000000..32c302c4 --- /dev/null +++ b/src/lib/tests/tw_test.c @@ -0,0 +1,663 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Generic timing-wheel tests + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#define _DEFAULT_SOURCE +#else +#define _POSIX_C_SOURCE 200809L +#endif + +#include "config.h" + +#include <test/test.h> + +#include <ouroboros/time.h> +#include <ouroboros/tw.h> + +#include <stdint.h> +#include <stdio.h> +#include <time.h> + +struct payload { + struct tw_entry tw; + int fired; +}; + +struct cancel_payload { + struct tw_entry tw; + int fired; + struct tw_entry * sibling; +}; + +struct repost_payload { + struct tw_entry tw; + int fired; + struct payload * sibling; + uint64_t repost_at; +}; + +static void cb_count(void * arg) +{ + struct payload * p = arg; + p->fired++; +} + +static void cb_cancel_sibling(void * arg) +{ + struct cancel_payload * p = arg; + p->fired++; + tw_cancel(p->sibling); +} + +static void cb_repost_sibling(void * arg) +{ + struct repost_payload * p = arg; + p->fired++; + tw_post(&p->sibling->tw, p->repost_at, cb_count, p->sibling); +} + +static uint64_t now_ns(void) +{ + struct timespec ts; + clock_gettime(PTHREAD_COND_CLOCK, &ts); + return TS_TO_UINT64(ts); +} + +static void sleep_ns(uint64_t ns) +{ + struct timespec ts; + UINT64_TO_TS(ns, &ts); + nanosleep(&ts, NULL); +} + +static int test_tw_init_fini(void) +{ + TEST_START(); + + if (tw_init() < 0) { + printf("tw_init failed.\n"); + goto fail; + } + + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tw_post_fires_after_deadline(void) +{ + struct payload p; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&p.tw); + p.fired = 0; + + tw_post(&p.tw, now_ns() + 5 * MILLION, cb_count, &p); + + sleep_ns(20 * MILLION); + tw_move(); + + if (p.fired != 1) { + printf("expected 1 fire, got %d\n", p.fired); + goto fail_post; + } + + tw_cancel(&p.tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + tw_cancel(&p.tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tw_no_fire_before_deadline(void) +{ + struct payload p; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&p.tw); + p.fired = 0; + + tw_post(&p.tw, now_ns() + 100 * MILLION, cb_count, &p); + + sleep_ns(2 * MILLION); + tw_move(); + + if (p.fired != 0) { + printf("expected 0 fires, got %d\n", p.fired); + goto fail_post; + } + + tw_cancel(&p.tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + tw_cancel(&p.tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tw_cancel_prevents_fire(void) +{ + struct payload p; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&p.tw); + p.fired = 0; + + tw_post(&p.tw, now_ns() + 5 * MILLION, cb_count, &p); + tw_cancel(&p.tw); + + sleep_ns(20 * MILLION); + tw_move(); + + if (p.fired != 0) { + printf("cancelled entry fired %d times\n", p.fired); + goto fail_init; + } + + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tw_cancel_unposted_is_noop(void) +{ + struct tw_entry e; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&e); + tw_cancel(&e); + tw_cancel(&e); + + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tw_fire_only_once(void) +{ + struct payload p; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&p.tw); + p.fired = 0; + + tw_post(&p.tw, now_ns() + 3 * MILLION, cb_count, &p); + + sleep_ns(20 * MILLION); + tw_move(); + tw_move(); + tw_move(); + + if (p.fired != 1) { + printf("expected 1 fire, got %d after 3 moves\n", p.fired); + goto fail_post; + } + + tw_cancel(&p.tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + tw_cancel(&p.tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Multi-level: post a level-1 (>= 256ms) deadline; should still fire. */ +static int test_tw_post_level1_fires(void) +{ + struct payload p; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&p.tw); + p.fired = 0; + + tw_post(&p.tw, now_ns() + 300 * MILLION, cb_count, &p); + + if (p.tw.lvl != 1) { + printf("expected level 1 placement, got %zu\n", p.tw.lvl); + goto fail_post; + } + + sleep_ns(320 * MILLION); + tw_move(); + + if (p.fired != 1) { + printf("level-1 entry didn't fire (got %d)\n", p.fired); + goto fail_post; + } + + tw_cancel(&p.tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + tw_cancel(&p.tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +static int test_tw_many_entries_all_fire(void) +{ + struct payload pl[16]; + size_t i; + size_t total = 0; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + for (i = 0; i < 16; ++i) { + tw_init_entry(&pl[i].tw); + pl[i].fired = 0; + tw_post(&pl[i].tw, now_ns() + (1 + i) * MILLION, + cb_count, &pl[i]); + } + + sleep_ns(40 * MILLION); + tw_move(); + + for (i = 0; i < 16; ++i) + total += pl[i].fired; + + if (total != 16) { + printf("expected 16 fires, got %zu\n", total); + goto fail_post; + } + + for (i = 0; i < 16; ++i) + tw_cancel(&pl[i].tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + for (i = 0; i < 16; ++i) + tw_cancel(&pl[i].tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* tw_next_expiry signals empty wheel via tv_nsec == -1. */ +static int test_tw_next_expiry_empty(void) +{ + struct timespec out; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_next_expiry(&out); + if (out.tv_nsec != -1) { + printf("expected tv_nsec=-1, got %ld\n", (long) out.tv_nsec); + goto fail_init; + } + + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_init: + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* tw_next_expiry returns a deadline within the right ballpark. */ +static int test_tw_next_expiry_returns_deadline(void) +{ + struct payload p; + struct timespec out; + uint64_t target; + uint64_t out_ns; + int64_t skew; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&p.tw); + p.fired = 0; + + target = now_ns() + 50 * MILLION; + tw_post(&p.tw, target, cb_count, &p); + + tw_next_expiry(&out); + out_ns = TS_TO_UINT64(out); + + /* Level-0 quantization gives ±1 slot of skew. */ + skew = (int64_t)(out_ns) - (int64_t)(target); + if (skew < -2 * MILLION || skew > 4 * MILLION) { + printf("deadline not in -2..+4 ms, skew=%ld ns\n", (long) skew); + goto fail_post; + } + + tw_cancel(&p.tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + tw_cancel(&p.tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Repost: fire, then post again. */ +static int test_tw_repost_after_fire(void) +{ + struct payload p; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&p.tw); + p.fired = 0; + + tw_post(&p.tw, now_ns() + 3 * MILLION, cb_count, &p); + sleep_ns(20 * MILLION); + tw_move(); + if (p.fired != 1) { + printf("first fire missed\n"); + goto fail_post; + } + + tw_post(&p.tw, now_ns() + 3 * MILLION, cb_count, &p); + sleep_ns(20 * MILLION); + tw_move(); + if (p.fired != 2) { + printf("second fire missed (fired=%d)\n", p.fired); + goto fail_post; + } + + tw_cancel(&p.tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + tw_cancel(&p.tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Double-post replaces the schedule; only the second fires. */ +static int test_tw_double_post_replaces(void) +{ + struct payload p; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&p.tw); + p.fired = 0; + + tw_post(&p.tw, now_ns() + 30 * MILLION, cb_count, &p); + tw_post(&p.tw, now_ns() + 3 * MILLION, cb_count, &p); + + sleep_ns(20 * MILLION); + tw_move(); + + if (p.fired != 1) { + printf("expected 1 fire after replace, got %d\n", p.fired); + goto fail_post; + } + + sleep_ns(40 * MILLION); + tw_move(); + + if (p.fired != 1) { + printf("first schedule fired after replace (got %d)\n", + p.fired); + goto fail_post; + } + + tw_cancel(&p.tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + tw_cancel(&p.tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Fire callback may safely cancel a sibling in the same slot. */ +static int test_tw_fire_cancels_sibling(void) +{ + struct cancel_payload a; + struct payload b; + uint64_t deadline; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&a.tw); + tw_init_entry(&b.tw); + a.fired = 0; + a.sibling = &b.tw; + b.fired = 0; + + deadline = now_ns() + 3 * MILLION; + tw_post(&a.tw, deadline, cb_cancel_sibling, &a); + tw_post(&b.tw, deadline, cb_count, &b); + + sleep_ns(20 * MILLION); + tw_move(); + + if (a.fired != 1) { + printf("a expected 1 fire, got %d\n", a.fired); + goto fail_post; + } + if (b.fired != 0) { + printf("b should not have fired (got %d)\n", b.fired); + goto fail_post; + } + + tw_cancel(&a.tw); + tw_cancel(&b.tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + tw_cancel(&a.tw); + tw_cancel(&b.tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +/* Fire callback may safely repost a sibling to a future slot. */ +static int test_tw_fire_posts_sibling(void) +{ + struct repost_payload a; + struct payload b; + uint64_t deadline; + + TEST_START(); + + if (tw_init() < 0) + goto fail; + + tw_init_entry(&a.tw); + tw_init_entry(&b.tw); + a.fired = 0; + a.sibling = &b; + a.repost_at = now_ns() + 30 * MILLION; + b.fired = 0; + + deadline = now_ns() + 3 * MILLION; + tw_post(&a.tw, deadline, cb_repost_sibling, &a); + tw_post(&b.tw, deadline, cb_count, &b); + + sleep_ns(20 * MILLION); + tw_move(); + + if (a.fired != 1) { + printf("a expected 1 fire, got %d\n", a.fired); + goto fail_post; + } + if (b.fired != 0) { + printf("b fired before reposted deadline (got %d)\n", + b.fired); + goto fail_post; + } + + sleep_ns(25 * MILLION); + tw_move(); + + if (b.fired != 1) { + printf("b expected 1 fire after repost, got %d\n", + b.fired); + goto fail_post; + } + + tw_cancel(&a.tw); + tw_cancel(&b.tw); + tw_fini(); + + TEST_SUCCESS(); + + return TEST_RC_SUCCESS; + fail_post: + tw_cancel(&a.tw); + tw_cancel(&b.tw); + tw_fini(); + fail: + TEST_FAIL(); + return TEST_RC_FAIL; +} + +int tw_test(int argc, + char ** argv) +{ + int ret = 0; + + (void) argc; + (void) argv; + + ret |= test_tw_init_fini(); + ret |= test_tw_post_fires_after_deadline(); + ret |= test_tw_no_fire_before_deadline(); + ret |= test_tw_cancel_prevents_fire(); + ret |= test_tw_cancel_unposted_is_noop(); + ret |= test_tw_fire_only_once(); + ret |= test_tw_post_level1_fires(); + ret |= test_tw_many_entries_all_fire(); + ret |= test_tw_next_expiry_empty(); + ret |= test_tw_next_expiry_returns_deadline(); + ret |= test_tw_repost_after_fire(); + ret |= test_tw_double_post_replaces(); + ret |= test_tw_fire_cancels_sibling(); + ret |= test_tw_fire_posts_sibling(); + + return ret; +} diff --git a/src/lib/timerwheel.c b/src/lib/timerwheel.c deleted file mode 100644 index a9f3c72a..00000000 --- a/src/lib/timerwheel.c +++ /dev/null @@ -1,429 +0,0 @@ -/* - * Ouroboros - Copyright (C) 2016 - 2020 - * - * Timerwheel - * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * version 2.1 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., http://www.fsf.org/about/contact/. - */ - -#include <ouroboros/list.h> - -/* Overflow limits range to about 6 hours. */ -#define ts_to_ns(ts) (ts.tv_sec * BILLION + ts.tv_nsec) -#define ts_to_rxm_slot(ts) (ts_to_ns(ts) >> RXMQ_RES) -#define ts_to_ack_slot(ts) (ts_to_ns(ts) >> ACKQ_RES) - -struct rxm { - struct list_head next; - uint32_t seqno; -#ifdef RXM_BUFFER_ON_HEAP - uint8_t * pkt; - size_t pkt_len; -#else - struct shm_du_buff * sdb; - uint8_t * head; - uint8_t * tail; -#endif - time_t t0; /* Time when original was sent (us). */ - size_t mul; /* RTO multiplier. */ - struct frcti * frcti; - int fd; - int flow_id; /* Prevent rtx when fd reused. */ -}; - -struct ack { - struct list_head next; - struct frcti * frcti; - int fd; - int flow_id; -}; - -struct { - /* - * At a 1 ms min resolution, every level bumps the - * resolution by a factor of 16. - */ - struct list_head rxms[RXMQ_LVLS][RXMQ_SLOTS]; - - struct list_head acks[ACKQ_SLOTS]; - bool map[ACKQ_SLOTS][PROG_MAX_FLOWS]; - - size_t prv_rxm; /* Last processed rxm slot at lvl 0. */ - size_t prv_ack; /* Last processed ack slot. */ - pthread_mutex_t lock; - - bool in_use; -} rw; - -static void timerwheel_fini(void) -{ - size_t i; - size_t j; - struct list_head * p; - struct list_head * h; - - pthread_mutex_lock(&rw.lock); - - for (i = 0; i < RXMQ_LVLS; ++i) { - for (j = 0; j < RXMQ_SLOTS; j++) { - list_for_each_safe(p, h, &rw.rxms[i][j]) { - struct rxm * rxm; - rxm = list_entry(p, struct rxm, next); - list_del(&rxm->next); -#ifdef RXM_BUFFER_ON_HEAP - free(rxm->pkt); -#else - shm_du_buff_ack(rxm->sdb); - ipcp_sdb_release(rxm->sdb); -#endif - free(rxm); - } - } - } - - for (i = 0; i < ACKQ_SLOTS; ++i) { - list_for_each_safe(p, h, &rw.acks[i]) { - struct ack * a = list_entry(p, struct ack, next); - list_del(&a->next); - free(a); - } - } - - pthread_mutex_unlock(&rw.lock); - - pthread_mutex_destroy(&rw.lock); -} - -static int timerwheel_init(void) -{ - struct timespec now; - size_t i; - size_t j; - - if (pthread_mutex_init(&rw.lock, NULL)) - return -1; - - clock_gettime(PTHREAD_COND_CLOCK, &now); - - rw.prv_rxm = (ts_to_rxm_slot(now) - 1) & (RXMQ_SLOTS - 1); - for (i = 0; i < RXMQ_LVLS; ++i) { - for (j = 0; j < RXMQ_SLOTS; ++j) - list_head_init(&rw.rxms[i][j]); - } - - rw.prv_ack = (ts_to_ack_slot(now) - 1) & (ACKQ_SLOTS - 1); - for (i = 0; i < ACKQ_SLOTS; ++i) - list_head_init(&rw.acks[i]); - - return 0; -} - -static void timerwheel_move(void) -{ - struct timespec now; - struct list_head * p; - struct list_head * h; - size_t rxm_slot; - size_t ack_slot; - size_t i; - size_t j; - - if (!__sync_bool_compare_and_swap(&rw.in_use, true, true)) - return; - - pthread_mutex_lock(&rw.lock); - - pthread_cleanup_push((void (*) (void *)) pthread_mutex_unlock, - (void *) &rw.lock); - - clock_gettime(PTHREAD_COND_CLOCK, &now); - - rxm_slot = ts_to_ns(now) >> RXMQ_RES; - j = rw.prv_rxm; - rw.prv_rxm = rxm_slot & (RXMQ_SLOTS - 1); - - for (i = 0; i < RXMQ_LVLS; ++i) { - size_t j_max_slot = rxm_slot & (RXMQ_SLOTS - 1); - if (j_max_slot < j) - j_max_slot += RXMQ_SLOTS; - - while (j++ < j_max_slot) { - list_for_each_safe(p, h, - &rw.rxms[i][j & (RXMQ_SLOTS - 1)]) { - struct rxm * r; - struct frct_cr * snd_cr; - struct frct_cr * rcv_cr; - size_t rslot; - ssize_t idx; - struct shm_du_buff * sdb; - uint8_t * head; - struct flow * f; - uint32_t snd_lwe; - uint32_t rcv_lwe; - time_t rto; - - r = list_entry(p, struct rxm, next); - - list_del(&r->next); - - snd_cr = &r->frcti->snd_cr; - rcv_cr = &r->frcti->rcv_cr; - f = &ai.flows[r->fd]; -#ifndef RXM_BUFFER_ON_HEAP - shm_du_buff_ack(r->sdb); -#endif - if (f->frcti == NULL - || f->flow_id != r->flow_id) - goto cleanup; - - pthread_rwlock_wrlock(&r->frcti->lock); - - snd_lwe = snd_cr->lwe; - rcv_lwe = rcv_cr->lwe; - rto = r->frcti->rto; - - pthread_rwlock_unlock(&r->frcti->lock); - - /* Has been ack'd, remove. */ - if ((int) (r->seqno - snd_lwe) < 0) - goto cleanup; - - /* Check for r-timer expiry. */ - if (ts_to_ns(now) - r->t0 > r->frcti->r) - goto flow_down; - - if (r->frcti->probe - && (r->frcti->rttseq + 1) == r->seqno) - r->frcti->probe = false; -#ifdef RXM_BLOCKING - #ifdef RXM_BUFFER_ON_HEAP - if (ipcp_sdb_reserve(&sdb, r->pkt_len)) - #else - if (ipcp_sdb_reserve(&sdb, r->tail - r->head)) - #endif -#else - #ifdef RXM_BUFFER_ON_HEAP - if (shm_rdrbuff_alloc(ai.rdrb, r->pkt_len, NULL, - &sdb)) - #else - if (shm_rdrbuff_alloc(ai.rdrb, - r->tail - r->head, NULL, - &sdb)) - #endif -#endif - goto reschedule; /* rbuff full */ - idx = shm_du_buff_get_idx(sdb); - - head = shm_du_buff_head(sdb); -#ifdef RXM_BUFFER_ON_HEAP - memcpy(head, r->pkt, r->pkt_len); -#else - memcpy(head, r->head, r->tail - r->head); - ipcp_sdb_release(r->sdb); - r->sdb = sdb; - r->head = head; - r->tail = shm_du_buff_tail(sdb); - shm_du_buff_wait_ack(sdb); -#endif - /* Retransmit the copy. */ - ((struct frct_pci *) head)->ackno = - hton32(rcv_lwe); -#ifdef RXM_BLOCKING - if (shm_rbuff_write_b(f->tx_rb, idx, NULL) == 0) -#else - if (shm_rbuff_write(f->tx_rb, idx) == 0) -#endif - shm_flow_set_notify(f->set, f->flow_id, - FLOW_PKT); - reschedule: - r->mul++; - - /* Schedule at least in the next time slot. */ - rslot = (rxm_slot - + MAX(((rto * r->mul) >> RXMQ_RES), 1)) - & (RXMQ_SLOTS - 1); - - list_add_tail(&r->next, &rw.rxms[i][rslot]); - - continue; - - flow_down: - shm_rbuff_set_acl(f->tx_rb, ACL_FLOWDOWN); - shm_rbuff_set_acl(f->rx_rb, ACL_FLOWDOWN); - cleanup: -#ifdef RXM_BUFFER_ON_HEAP - free(r->pkt); -#else - ipcp_sdb_release(r->sdb); -#endif - free(r); - } - } - /* Move up a level in the wheel. */ - rxm_slot >>= RXMQ_BUMP; - j >>= RXMQ_BUMP; - } - - ack_slot = ts_to_ack_slot(now) & (ACKQ_SLOTS - 1) ; - - j = rw.prv_ack; - - if (ack_slot < j) - ack_slot += ACKQ_SLOTS; - - while (j++ < ack_slot) { - list_for_each_safe(p, h, &rw.acks[j & (ACKQ_SLOTS - 1)]) { - struct ack * a; - struct flow * f; - - a = list_entry(p, struct ack, next); - - list_del(&a->next); - - f = &ai.flows[a->fd]; - - rw.map[j & (ACKQ_SLOTS - 1)][a->fd] = false; - - if (f->flow_id == a->flow_id && f->frcti != NULL) - send_frct_pkt(a->frcti); - - free(a); - - } - } - - rw.prv_ack = ack_slot & (ACKQ_SLOTS - 1); - - pthread_cleanup_pop(true); -} - -static int timerwheel_rxm(struct frcti * frcti, - uint32_t seqno, - struct shm_du_buff * sdb) -{ - struct timespec now; - struct rxm * r; - size_t slot; - size_t lvl = 0; - time_t rto_slot; - - r = malloc(sizeof(*r)); - if (r == NULL) - return -ENOMEM; - - clock_gettime(PTHREAD_COND_CLOCK, &now); - - r->t0 = ts_to_ns(now); - r->mul = 0; - r->seqno = seqno; - r->frcti = frcti; -#ifdef RXM_BUFFER_ON_HEAP - r->pkt_len = shm_du_buff_tail(sdb) - shm_du_buff_head(sdb); - r->pkt = malloc(r->pkt_len); - if (r->pkt == NULL) { - free(r); - return -ENOMEM; - } - memcpy(r->pkt, shm_du_buff_head(sdb), r->pkt_len); -#else - r->sdb = sdb; - r->head = shm_du_buff_head(sdb); - r->tail = shm_du_buff_tail(sdb); -#endif - pthread_rwlock_rdlock(&r->frcti->lock); - - rto_slot = frcti->rto >> RXMQ_RES; - slot = r->t0 >> RXMQ_RES; - - r->fd = frcti->fd; - r->flow_id = ai.flows[r->fd].flow_id; - - pthread_rwlock_unlock(&r->frcti->lock); - - while (rto_slot >= RXMQ_SLOTS) { - ++lvl; - rto_slot >>= RXMQ_BUMP; - slot >>= RXMQ_BUMP; - } - - if (lvl >= RXMQ_LVLS) { /* Out of timerwheel range. */ -#ifdef RXM_BUFFER_ON_HEAP - free(r->pkt); -#endif - free(r); - return -EPERM; - } - - slot = (slot + rto_slot) & (RXMQ_SLOTS - 1); - - pthread_mutex_lock(&rw.lock); - - list_add_tail(&r->next, &rw.rxms[lvl][slot]); -#ifndef RXM_BUFFER_ON_HEAP - shm_du_buff_wait_ack(sdb); -#endif - pthread_mutex_unlock(&rw.lock); - - __sync_bool_compare_and_swap(&rw.in_use, false, true); - - return 0; -} - -static int timerwheel_ack(int fd, - struct frcti * frcti) -{ - struct timespec now; - struct ack * a; - size_t slot; - - a = malloc(sizeof(*a)); - if (a == NULL) - return -ENOMEM; - - clock_gettime(PTHREAD_COND_CLOCK, &now); - - slot = DELT_ACK >> ACKQ_RES; - if (slot >= ACKQ_SLOTS) { /* Out of timerwheel range. */ - free(a); - return -EPERM; - } - - slot = (((ts_to_ns(now) + DELT_ACK) >> ACKQ_RES) + 1) - & (ACKQ_SLOTS - 1); - - a->fd = fd; - a->frcti = frcti; - a->flow_id = ai.flows[fd].flow_id; - - pthread_mutex_lock(&rw.lock); - - if (rw.map[slot][fd]) { - pthread_mutex_unlock(&rw.lock); - free(a); - return 0; - } - - rw.map[slot][fd] = true; - - list_add_tail(&a->next, &rw.acks[slot]); - - pthread_mutex_unlock(&rw.lock); - - __sync_bool_compare_and_swap(&rw.in_use, false, true); - - return 0; -} diff --git a/src/lib/tpm.c b/src/lib/tpm.c index ff0a22df..9229ea1a 100644 --- a/src/lib/tpm.c +++ b/src/lib/tpm.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Threadpool management * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -26,21 +26,32 @@ #include <ouroboros/errno.h> #include <ouroboros/list.h> -#include <ouroboros/time_utils.h> +#include <ouroboros/pthread.h> +#include <ouroboros/time.h> #include <ouroboros/tpm.h> -#include <pthread.h> -#include <stdlib.h> +#ifdef CONFIG_OUROBOROS_DEBUG +#define OUROBOROS_PREFIX "tpm" +#include <ouroboros/logs.h> +#endif + #include <assert.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> -#define TPM_TIMEOUT 1000 +#define TPM_TIMEOUT 1000 struct pthr_el { struct list_head next; bool kill; bool busy; - + bool wait; +#ifdef CONFIG_OUROBOROS_DEBUG + struct timespec start; + struct timespec last; +#endif pthread_t thr; }; @@ -63,16 +74,45 @@ struct tpm { enum tpm_state state; pthread_cond_t cond; - pthread_mutex_t lock; + pthread_mutex_t mtx; pthread_t mgr; }; +#ifdef CONFIG_OUROBOROS_DEBUG +#define BETWEEN(a, x, y) ((a) > (x) && (a) <= (y)) +static void tpm_debug_thread(struct pthr_el * e) +{ + struct timespec now; + time_t diff; + time_t intv; + + if (e->wait || !e->busy) + return; + + clock_gettime(CLOCK_REALTIME, &now); + + diff = ts_diff_ms(&now, &e->start) / 1000; + intv = ts_diff_ms(&now, &e->last) / 1000; + + (void) diff; /* Never read if both build options off (0) */ + (void) intv; /* Never read if report option off (0) */ + + if (BETWEEN(TPM_DEBUG_REPORT_INTERVAL, 0, intv)) { + log_dbg("Thread %d:%lx running for %ld s.\n", + getpid(), (unsigned long) e->thr, diff); + e->last = now; + } + + if (BETWEEN(TPM_DEBUG_ABORT_TIMEOUT, 0, diff)) + assert(false); /* TODO: Grab a coffee and fire up GDB */ +} +#endif + static void tpm_join(struct tpm * tpm) { struct list_head * p; struct list_head * h; - list_for_each_safe(p, h, &tpm->pool) { struct pthr_el * e = list_entry(p, struct pthr_el, next); if (tpm->state != TPM_RUNNING) { @@ -86,15 +126,18 @@ static void tpm_join(struct tpm * tpm) list_for_each_safe(p, h, &tpm->pool) { struct pthr_el * e = list_entry(p, struct pthr_el, next); +#ifdef CONFIG_OUROBOROS_DEBUG + tpm_debug_thread(e); +#endif if (e->kill) { pthread_t thr = e->thr; list_del(&e->next); free(e); - pthread_mutex_unlock(&tpm->lock); + pthread_mutex_unlock(&tpm->mtx); pthread_join(thr, NULL); - pthread_mutex_lock(&tpm->lock); + pthread_mutex_lock(&tpm->mtx); } } } @@ -114,57 +157,59 @@ static void tpm_kill(struct tpm * tpm) } } -static void * tpmgr(void * o) +static int __tpm(struct tpm * tpm) { struct timespec dl; - struct timespec to = {(TPM_TIMEOUT / 1000), - (TPM_TIMEOUT % 1000) * MILLION}; - struct tpm * tpm = (struct tpm *) o; - - while (true) { - clock_gettime(PTHREAD_COND_CLOCK, &dl); - ts_add(&dl, &to, &dl); + struct timespec to = TIMESPEC_INIT_MS(TPM_TIMEOUT); - pthread_mutex_lock(&tpm->lock); + clock_gettime(PTHREAD_COND_CLOCK, &dl); + ts_add(&dl, &to, &dl); - if (tpm->state != TPM_RUNNING) { - tpm_join(tpm); - pthread_mutex_unlock(&tpm->lock); - break; - } + pthread_mutex_lock(&tpm->mtx); + if (tpm->state != TPM_RUNNING) { tpm_join(tpm); + pthread_mutex_unlock(&tpm->mtx); + return -1; + } - if (tpm->cur - tpm->wrk < tpm->min) { - size_t i; - for (i = 0; i < tpm->inc; ++i) { - struct pthr_el * e = malloc(sizeof(*e)); - if (e == NULL) - break; + tpm_join(tpm); - e->kill = false; - e->busy = false; + if (tpm->cur - tpm->wrk < tpm->min) { + size_t i; + for (i = 0; i < tpm->inc; ++i) { + struct pthr_el * e = malloc(sizeof(*e)); + if (e == NULL) + break; - if (pthread_create(&e->thr, NULL, - tpm->func, tpm->o)) { - free(e); - break; - } + memset(e, 0, sizeof(*e)); - list_add(&e->next, &tpm->pool); + if (pthread_create(&e->thr, NULL, tpm->func, tpm->o)) { + free(e); + break; } - tpm->cur += i; + list_add(&e->next, &tpm->pool); } - if (pthread_cond_timedwait(&tpm->cond, &tpm->lock, &dl) - == ETIMEDOUT) - if (tpm->cur - tpm->wrk > tpm->min) - tpm_kill(tpm); - - pthread_mutex_unlock(&tpm->lock); + tpm->cur += i; } + pthread_cleanup_push(__cleanup_mutex_unlock, &tpm->mtx); + + if (pthread_cond_timedwait(&tpm->cond, &tpm->mtx, &dl) == ETIMEDOUT) + if (tpm->cur - tpm->wrk > tpm->min) + tpm_kill(tpm); + + pthread_cleanup_pop(true); + + return 0; +} + +static void * tpmgr(void * o) +{ + while (__tpm((struct tpm *) o) == 0); + return (void *) 0; } @@ -176,11 +221,14 @@ struct tpm * tpm_create(size_t min, struct tpm * tpm; pthread_condattr_t cattr; + assert(func != NULL); + assert(inc > 0); + tpm = malloc(sizeof(*tpm)); if (tpm == NULL) goto fail_malloc; - if (pthread_mutex_init(&tpm->lock, NULL)) + if (pthread_mutex_init(&tpm->mtx, NULL)) goto fail_lock; if (pthread_condattr_init(&cattr)) @@ -209,7 +257,7 @@ struct tpm * tpm_create(size_t min, fail_cond: pthread_condattr_destroy(&cattr); fail_cattr: - pthread_mutex_destroy(&tpm->lock); + pthread_mutex_destroy(&tpm->mtx); fail_lock: free(tpm); fail_malloc: @@ -218,34 +266,39 @@ struct tpm * tpm_create(size_t min, int tpm_start(struct tpm * tpm) { - pthread_mutex_lock(&tpm->lock); + pthread_mutex_lock(&tpm->mtx); if (pthread_create(&tpm->mgr, NULL, tpmgr, tpm)) { - pthread_mutex_unlock(&tpm->lock); + pthread_mutex_unlock(&tpm->mtx); return -1; } tpm->state = TPM_RUNNING; - pthread_mutex_unlock(&tpm->lock); + pthread_mutex_unlock(&tpm->mtx); return 0; } void tpm_stop(struct tpm * tpm) { - pthread_mutex_lock(&tpm->lock); + pthread_mutex_lock(&tpm->mtx); + + if (tpm->state != TPM_RUNNING) { + pthread_mutex_unlock(&tpm->mtx); + return; + } tpm->state = TPM_NULL; - pthread_mutex_unlock(&tpm->lock); + pthread_mutex_unlock(&tpm->mtx); + + pthread_join(tpm->mgr, NULL); } void tpm_destroy(struct tpm * tpm) { - pthread_join(tpm->mgr, NULL); - - pthread_mutex_destroy(&tpm->lock); + pthread_mutex_destroy(&tpm->mtx); pthread_cond_destroy(&tpm->cond); free(tpm); @@ -261,40 +314,61 @@ static struct pthr_el * tpm_pthr_el(struct tpm * tpm, e = list_entry(p, struct pthr_el, next); if (e->thr == thr) return e; - } return NULL; } -void tpm_inc(struct tpm * tpm) +void tpm_begin_work(struct tpm * tpm) { struct pthr_el * e; - pthread_mutex_lock(&tpm->lock); +#ifdef CONFIG_OUROBOROS_DEBUG + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); +#endif + + pthread_mutex_lock(&tpm->mtx); e = tpm_pthr_el(tpm, pthread_self()); if (e != NULL) { - e->busy = false; - --tpm->wrk; + e->busy = true; + ++tpm->wrk; +#ifdef CONFIG_OUROBOROS_DEBUG + e->start = now; + e->last = now; +#endif } - pthread_mutex_unlock(&tpm->lock); + pthread_cond_signal(&tpm->cond); + + pthread_mutex_unlock(&tpm->mtx); +} + +void tpm_wait_work(struct tpm * tpm) +{ + struct pthr_el * e; + + pthread_mutex_lock(&tpm->mtx); + + e = tpm_pthr_el(tpm, pthread_self()); + if (e != NULL) + e->wait = true; + + pthread_mutex_unlock(&tpm->mtx); } -void tpm_dec(struct tpm * tpm) +void tpm_end_work(struct tpm * tpm) { struct pthr_el * e; - pthread_mutex_lock(&tpm->lock); + pthread_mutex_lock(&tpm->mtx); e = tpm_pthr_el(tpm, pthread_self()); if (e != NULL) { - e->busy = true; - ++tpm->wrk; + e->busy = false; + --tpm->wrk; } - pthread_cond_signal(&tpm->cond); - - pthread_mutex_unlock(&tpm->lock); + pthread_mutex_unlock(&tpm->mtx); } diff --git a/src/lib/tw.c b/src/lib/tw.c new file mode 100644 index 00000000..ccde7dd1 --- /dev/null +++ b/src/lib/tw.c @@ -0,0 +1,307 @@ +/* + * Ouroboros - Copyright (C) 2016 - 2026 + * + * Generic deadline-ordered callback queue (timing wheel) + * + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., http://www.fsf.org/about/contact/. + */ + +#if defined(__linux__) || defined(__CYGWIN__) +#define _DEFAULT_SOURCE +#else +#define _POSIX_C_SOURCE 200809L +#endif + +#include "config.h" + +#include <ouroboros/list.h> +#include <ouroboros/pthread.h> +#include <ouroboros/time.h> +#include <ouroboros/tw.h> + +#include <assert.h> +#include <stdbool.h> +#include <stdint.h> + +/* 3 levels × 256 slots, 1 ms / 16 ms / 256 ms per-slot resolution. */ +#define TW_LVLS 3 +#define TW_SLOTS 256 +#define TW_BUMP 4 +#define TW_RES 20 /* 2^20 ns ≈ 1 ms per slot at level 0. */ + +#define TW_SLOT(x) ((x) & (TW_SLOTS - 1)) + +static struct { + struct list_head levels[TW_LVLS][TW_SLOTS]; + size_t prv[TW_LVLS]; + pthread_mutex_t mtx; + pthread_mutex_t move_mtx; + bool initialised; +} tw; + +static size_t tw_lvl_res(size_t lvl) +{ + return TW_RES + TW_BUMP * lvl; +} + +/* Smallest level whose slot range covers the deadline. */ +static size_t tw_pick_lvl(uint64_t now_ns, + uint64_t deadline_ns) +{ + uint64_t delta; + size_t lvl; + + delta = deadline_ns > now_ns ? deadline_ns - now_ns : 0; + lvl = 0; + + while (lvl < TW_LVLS - 1 && (delta >> tw_lvl_res(lvl)) >= TW_SLOTS) + ++lvl; + + return lvl; +} + +static size_t tw_slot(uint64_t ns, + size_t lvl) +{ + return TW_SLOT(ns >> tw_lvl_res(lvl)); +} + +int tw_init(void) +{ + struct timespec now; + size_t i; + size_t j; + + assert(!tw.initialised); + + if (pthread_mutex_init(&tw.mtx, NULL)) + goto fail_mtx; + + if (pthread_mutex_init(&tw.move_mtx, NULL)) + goto fail_move_mtx; + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + for (i = 0; i < TW_LVLS; ++i) { + tw.prv[i] = TW_SLOT(tw_slot(TS_TO_UINT64(now), i) - 1); + for (j = 0; j < TW_SLOTS; ++j) + list_head_init(&tw.levels[i][j]); + } + + tw.initialised = true; + + return 0; + + fail_move_mtx: + pthread_mutex_destroy(&tw.mtx); + fail_mtx: + return -1; +} + +void tw_fini(void) +{ + size_t i; + size_t j; + + assert(tw.initialised); + + for (i = 0; i < TW_LVLS; ++i) { + for (j = 0; j < TW_SLOTS; ++j) + assert(list_is_empty(&tw.levels[i][j])); + } + + pthread_mutex_destroy(&tw.move_mtx); + pthread_mutex_destroy(&tw.mtx); + + tw.initialised = false; +} + +void tw_init_entry(struct tw_entry * e) +{ + list_head_init(&e->next); + + e->deadline_ns = 0; + e->fire = NULL; + e->arg = NULL; + e->lvl = 0; +} + +void tw_post(struct tw_entry * e, + uint64_t deadline_ns, + tw_fire_fn_t fire, + void * arg) +{ + struct timespec now; + size_t lvl; + size_t slot; + + assert(tw.initialised); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + + lvl = tw_pick_lvl(TS_TO_UINT64(now), deadline_ns); + /* +1 so deadline <= slot_start; lands later in slot. */ + slot = TW_SLOT(tw_slot(deadline_ns, lvl) + 1); + + e->deadline_ns = deadline_ns; + e->fire = fire; + e->arg = arg; + e->lvl = lvl; + + pthread_mutex_lock(&tw.mtx); + + if (!list_is_empty(&e->next)) + list_del(&e->next); + + list_add_tail(&e->next, &tw.levels[lvl][slot]); + + pthread_mutex_unlock(&tw.mtx); +} + +void tw_cancel(struct tw_entry * e) +{ + if (e == NULL) + return; + + assert(tw.initialised); + + pthread_mutex_lock(&tw.mtx); + + if (!list_is_empty(&e->next)) { + list_del(&e->next); + list_head_init(&e->next); + } + + pthread_mutex_unlock(&tw.mtx); +} + +void tw_move(void) +{ + struct timespec now; + struct list_head deferred; + struct list_head * p; + uint64_t now_ns; + size_t i; + size_t j; + size_t cur; + + assert(tw.initialised); + + if (pthread_mutex_trylock(&tw.move_mtx) != 0) + return; + + pthread_cleanup_push(__cleanup_mutex_unlock, &tw.move_mtx); + + pthread_mutex_lock(&tw.mtx); + + pthread_cleanup_push(__cleanup_mutex_unlock, &tw.mtx); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + for (i = 0; i < TW_LVLS; ++i) { + cur = tw_slot(now_ns, i); + + j = tw.prv[i]; + if (cur < j) + cur += TW_SLOTS; + + while (j++ < cur) { + size_t s = TW_SLOT(j); + + /* Pop-front so fire may mutate any entry. */ + list_head_init(&deferred); + + while (!list_is_empty(&tw.levels[i][s])) { + struct tw_entry * e; + p = tw.levels[i][s].nxt; + e = list_entry(p, struct tw_entry, next); + list_del(&e->next); + + if (e->deadline_ns > now_ns) { + list_add_tail(&e->next, &deferred); + continue; + } + + pthread_mutex_unlock(&tw.mtx); + e->fire(e->arg); + pthread_mutex_lock(&tw.mtx); + } + + while (!list_is_empty(&deferred)) { + p = deferred.nxt; + list_del(p); + list_add_tail(p, &tw.levels[i][s]); + } + } + + tw.prv[i] = TW_SLOT(cur); + } + + pthread_cleanup_pop(true); /* tw.mtx */ + pthread_cleanup_pop(true); /* tw.move_mtx */ +} + +/* Earliest pending deadline at level lvl, INT64_MAX if level is empty. */ +static int64_t tw_lvl_earliest(size_t lvl, + uint64_t now_ns) +{ + size_t cur = tw_slot(now_ns, lvl); + size_t j; + + for (j = 1; j <= TW_SLOTS; ++j) { + size_t s = TW_SLOT(cur + j); + + if (list_is_empty(&tw.levels[lvl][s])) + continue; + + return (int64_t)(now_ns + ((uint64_t) j << tw_lvl_res(lvl))); + } + + return INT64_MAX; +} + +void tw_next_expiry(struct timespec * out) +{ + struct timespec now; + uint64_t now_ns; + int64_t earliest = INT64_MAX; + size_t i; + + assert(tw.initialised); + + clock_gettime(PTHREAD_COND_CLOCK, &now); + now_ns = TS_TO_UINT64(now); + + pthread_mutex_lock(&tw.mtx); + + for (i = 0; i < TW_LVLS; ++i) { + int64_t dl = tw_lvl_earliest(i, now_ns); + if (dl < earliest) + earliest = dl; + } + + pthread_mutex_unlock(&tw.mtx); + + if (earliest == INT64_MAX) { + /* Empty wheel: tv_nsec=-1 is an invalid normalised value. */ + out->tv_sec = 0; + out->tv_nsec = -1; + } else { + UINT64_TO_TS((uint64_t) earliest, out); + } +} diff --git a/src/lib/utils.c b/src/lib/utils.c index e3024283..6b49cc00 100644 --- a/src/lib/utils.c +++ b/src/lib/utils.c @@ -1,10 +1,10 @@ /* - * Ouroboros - Copyright (C) 2016 - 2020 + * Ouroboros - Copyright (C) 2016 - 2026 * * Handy utilities * - * Dimitri Staessens <dimitri.staessens@ugent.be> - * Sander Vrijders <sander.vrijders@ugent.be> + * Dimitri Staessens <dimitri@ouroboros.rocks> + * Sander Vrijders <sander@ouroboros.rocks> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public License @@ -20,29 +20,48 @@ * Foundation, Inc., http://www.fsf.org/about/contact/. */ +#define _DEFAULT_SOURCE + +#include "config.h" + +#include <ouroboros/utils.h> + +#include <ctype.h> +#include <grp.h> +#include <pwd.h> #include <stdlib.h> #include <string.h> +int bufcmp(const buffer_t * a, + const buffer_t * b) +{ + if (a->len != b->len) + return a->len < b->len ? -1 : 1; + + return memcmp(a->data, b->data, a->len); +} + + int n_digits(unsigned i) { - int n = 1; + int n = 1; - while (i > 9) { - ++n; - i /= 10; - } + while (i > 9) { + ++n; + i /= 10; + } - return n; + return n; } -char * path_strip(char * src) +char * path_strip(const char * src) { - char * dst = NULL; + char * dst; if (src == NULL) return NULL; - dst = src + strlen(src); + dst = (char *) src + strlen(src); while (dst > src && *dst != '/') --dst; @@ -52,3 +71,143 @@ char * path_strip(char * src) return dst; } + +char * trim_whitespace(char * str) +{ + char * end; + + while (isspace((unsigned char) *str)) + str++; + + if (*str == '\0') + return str; + + /* Trim trailing space */ + end = str + strlen(str) - 1; + while (end > str && isspace((unsigned char)*end)) + *end-- = '\0'; + + return str; +} + +size_t argvlen(const char ** argv) +{ + size_t argc = 0; + + if (argv == NULL) + return 0; + + while (*argv++ != NULL) + argc++; + + return argc; +} + +void argvfree(char ** argv) +{ + char ** argv_dup; + + if (argv == NULL) + return; + + argv_dup = argv; + while (*argv_dup != NULL) + free(*(argv_dup++)); + + free(argv); +} + +char ** argvdup(char ** argv) +{ + int argc = 0; + char ** argv_dup = argv; + int i; + + if (argv == NULL) + return NULL; + + while (*(argv_dup++) != NULL) + argc++; + + argv_dup = malloc((argc + 1) * sizeof(*argv_dup)); + if (argv_dup == NULL) + return NULL; + + for (i = 0; i < argc; ++i) { + argv_dup[i] = strdup(argv[i]); + if (argv_dup[i] == NULL) { + argvfree(argv_dup); + return NULL; + } + } + + argv_dup[argc] = NULL; + + return argv_dup; +} + +bool is_ouroboros_member_uid(uid_t uid) +{ + struct group * grp; + struct passwd * pw; +#ifdef __APPLE__ + unsigned int gid; + int * groups = NULL; +#else + gid_t gid; + gid_t * groups = NULL; +#endif + int ngroups; + int i; + + /* Root is always privileged */ + if (uid == 0) + return true; + + grp = getgrnam("ouroboros"); + if (grp == NULL) + return false; + + gid = grp->gr_gid; + + pw = getpwuid(uid); + if (pw == NULL) + return false; + + if (pw->pw_gid == gid) + return true; + + ngroups = 0; + getgrouplist(pw->pw_name, pw->pw_gid, NULL, &ngroups); + if (ngroups <= 0) + return false; + + groups = malloc(ngroups * sizeof(*groups)); + if (groups == NULL) + return false; + + if (getgrouplist(pw->pw_name, pw->pw_gid, groups, &ngroups) < 0) { + free(groups); + return false; + } + + for (i = 0; i < ngroups; i++) { +#ifdef __APPLE__ + if (groups[i] == (int) gid) { +#else + if (groups[i] == gid) { +#endif + free(groups); + return true; + } + } + + free(groups); + + return false; +} + +bool is_ouroboros_member(void) +{ + return is_ouroboros_member_uid(getuid()); +} |