From 5c56ff31b6a56d2520345f49f89deda54ccc71b4 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Sun, 3 May 2026 14:38:10 -0700
Subject: [PATCH 01/25] Add FFI benchmark tool

---
 boringtun/benches/ffi_benches/.gitignore      |   3 +
 boringtun/benches/ffi_benches/Makefile        |  28 ++
 boringtun/benches/ffi_benches/main.c          | 200 ++++++++++++
 .../benches/ffi_benches/wg_bench_client.c     | 307 ++++++++++++++++++
 .../benches/ffi_benches/wg_bench_client.h     |  38 +++
 5 files changed, 576 insertions(+)
 create mode 100644 boringtun/benches/ffi_benches/.gitignore
 create mode 100644 boringtun/benches/ffi_benches/Makefile
 create mode 100644 boringtun/benches/ffi_benches/main.c
 create mode 100644 boringtun/benches/ffi_benches/wg_bench_client.c
 create mode 100644 boringtun/benches/ffi_benches/wg_bench_client.h

diff --git a/boringtun/benches/ffi_benches/.gitignore b/boringtun/benches/ffi_benches/.gitignore
new file mode 100644
index 000000000..c23e1d0c7
--- /dev/null
+++ b/boringtun/benches/ffi_benches/.gitignore
@@ -0,0 +1,3 @@
+*.o
+ffi-bench
+release/
diff --git a/boringtun/benches/ffi_benches/Makefile b/boringtun/benches/ffi_benches/Makefile
new file mode 100644
index 000000000..c271efa7e
--- /dev/null
+++ b/boringtun/benches/ffi_benches/Makefile
@@ -0,0 +1,28 @@
+SRCDIR := $(dir $(lastword ${MAKEFILE_LIST}))
+PKGDIR := $(realpath ${SRCDIR}/../..)
+OBJDIR := $(shell pwd)
+
+release/libboringtun.a: ${PKGDIR}/Cargo.toml
+	cd ${PKGDIR} && cargo build --lib --release --target-dir ${OBJDIR} --features ffi-bindings
+	echo -n "$@:" > release/libboringtun-fixup.d
+	cat release/libboringtun.d | cut -d: -f2- >> release/libboringtun-fixup.d
+
+-include release/libboringtun-fixup.d
+
+BENCH_CFLAGS := -I ${PKGDIR}/src
+BENCH_SRCS := main.c wg_bench_client.c
+BENCH_OBJS := $(patsubst %.c,%.o,${BENCH_SRCS})
+
+%.o : %.c
+	${CC} ${CFLAGS} ${BENCH_CFLAGS} -c -o $@ $<
+
+ffi-bench: ${BENCH_OBJS} release/libboringtun.a
+	${CC} ${LDLAGS} -o $@ $^
+
+clean:
+	rm -f ${OBJDIR}/*.o
+	rm -rf ${OBJDIR}/release
+	rm -f ${OBJDIR}/ffi-bench
+
+.PHONY: clean
+.DEFAULT_GOAL := ffi-bench
diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
new file mode 100644
index 000000000..59513789c
--- /dev/null
+++ b/boringtun/benches/ffi_benches/main.c
@@ -0,0 +1,200 @@
+
+
+#include <errno.h>
+#include <getopt.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <asm/termbits.h>  /* Definition of TIOC*WINSZ constants */
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <time.h> 
+#include <pthread.h>
+#include <unistd.h>
+
+#include "wireguard_ffi.h"
+#include "wg_bench_client.h"
+
+static void handle_signal(int sig) {
+    switch (sig) {
+        case SIGINT:
+            fprintf(stderr, "benchmark interrupted\n");
+            break;
+
+        case SIGTERM:
+            fprintf(stderr, "benchmark terminated\n");
+            break;
+    }
+}
+
+static double timespec_elapsed(const struct timespec *a, const struct timespec* b) {
+    double result = (a->tv_sec - b->tv_sec) * 1000000000.0;
+    return (double)(result + a->tv_nsec - b->tv_nsec) / 1000000000.0;
+}
+
+static void print_header() {
+    printf("%012s %012s %012s %012s %08s  %-12s\n",
+           "TXPKT", "RXPKT", "TXBYTES", "RXBYTES", "ERRORS", "TRANSFERRED");
+}
+
+static void print_stats(const struct wg_bench_statistics* st, double elapsed) {
+    uintmax_t total_errors = 0;
+    uintmax_t total_bytes = atomic_load(&st->tx_bytes) + atomic_load(&st->rx_bytes);
+    for (int i = 0; i < WG_BENCH_MAX_ERRORS; i++) {
+        total_errors += atomic_load(&st->errors[i]);
+    }
+
+    // Estimate the total throughput.
+    char xfer[32];
+    if (total_bytes > 1000000000) {
+        snprintf(xfer, sizeof(xfer), "%.3F GB", total_bytes / 1000000000.0);
+    } else if (total_bytes > 1000000) {
+        snprintf(xfer, sizeof(xfer), "%.3F MB", total_bytes / 1000000.0);
+    } else if (total_bytes > 1000) {
+        snprintf(xfer, sizeof(xfer), "%.3F kB", total_bytes / 1000.0);
+    } else {
+        snprintf(xfer, sizeof(xfer), "%.3F B", total_bytes);
+    }
+
+    char tpbuf[32];
+    double throughput = (double)total_bytes / elapsed;
+    if (throughput > 1000000000) {
+        snprintf(tpbuf, sizeof(tpbuf), "%s (%.3F GB/s)", xfer, throughput / 1000000000.0);
+    } else if (throughput > 1000000) {
+        snprintf(tpbuf, sizeof(tpbuf), "%s (%.3F MB/s)", xfer, throughput / 1000000.0);
+    } else if (throughput > 1000) {
+        snprintf(tpbuf, sizeof(tpbuf), "%s (%.3F kB/s)", xfer, throughput / 1000.0);
+    } else {
+        snprintf(tpbuf, sizeof(tpbuf), "%s (%.3F B/s)", xfer, throughput);
+    }
+
+    // Prepare the status to write.
+    char linebuf[96];
+    int len = snprintf(linebuf, sizeof(linebuf), "\r%12lu %12lu %12lu %12lu %8lu  %-s",
+                       atomic_load(&st->tx_packets), atomic_load(&st->rx_packets),
+                       atomic_load(&st->tx_bytes), atomic_load(&st->rx_bytes),
+                       total_errors, tpbuf);
+
+    struct winsize ws;
+    if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) == 0) {
+        // This is a terminal.
+        if (ws.ws_col < sizeof(linebuf)) {
+            len = ws.ws_col - 1;
+        } else {
+            memset(linebuf + len, ' ', sizeof(linebuf) - len);
+            len = sizeof(linebuf);
+        }
+        write(STDOUT_FILENO, linebuf, len);
+
+    } else {
+        // Some other file.
+        linebuf[len] = '\n';
+        linebuf[len+1] = '\0';
+        puts(linebuf+1);
+    }
+}
+
+static void print_usage(FILE* fp, const char* name) {
+    fprintf(fp, "Usage: %s [OPTIONS]\n");
+    fprintf(fp, "Run FFI benchmarks for the boringtun library.\n");
+    fprintf(fp, "\n");
+    fprintf(fp, "Options:\n");
+    fprintf(fp, "\t--jobs, -j NUM   create NUM parallel threads\n");
+    fprintf(fp, "\t--help, -h       display this message and exit\n");
+}
+
+int main(int argc, char* argv[]) {
+    const char* shortopts = "hj:";
+    const struct option longopts[] = {
+        {"help", no_argument,       0, 'h'},
+        {"jobs", required_argument, 0, 'j'},
+        {NULL, 0, 0, 0}
+    };
+    unsigned int num_workers = 1;
+
+    // Parse options
+    while (true) {
+        int index;
+        int opt = getopt_long(argc, argv, shortopts, longopts, &index);
+        if (opt < 0) {
+            break;
+        }
+
+        char* endp;
+        switch (opt) {
+            case 'j':
+                num_workers = strtoul(optarg, &endp, 10);
+                if (*endp != '\0' || (num_workers == 0)) {
+                    fprintf(stderr, "Invalid thread count: %s\n", optarg);
+                    return 1;
+                }
+                break;
+
+            case 'h':
+                print_usage(stdout, argv[0]);
+                return 0;
+            
+            default:
+                break;
+        }
+    }
+
+    srand(time(0));
+
+    // This thread should handle signals.
+    struct sigaction action = {
+        .sa_handler = handle_signal,
+    };
+    sigaction(SIGINT, &action, NULL);
+    sigaction(SIGTERM, &action, NULL);
+
+    // Create a socket pair for the two clients to communicate over.
+    int sv[2];
+    socketpair(AF_UNIX, SOCK_DGRAM, 0, sv);
+
+    // Create two benchmark clients.
+    struct wg_bench_client* a = wg_bench_create(sv[0]);
+    struct wg_bench_client* b = wg_bench_create(sv[1]);
+
+    // Connect the two clients.
+    wg_bench_connect(a, b->pubkey);
+    wg_bench_connect(b, a->pubkey);
+
+    struct timespec start;
+    struct timespec end;
+    clock_gettime(CLOCK_MONOTONIC, &start);
+    end.tv_sec = start.tv_sec + 10;
+    end.tv_nsec = start.tv_nsec;
+    print_header();
+
+    // Launch workers.
+    wg_bench_start_handshake(a);
+    for (int i = 0; i < num_workers; i++) {
+        wg_bench_start_recv(a);
+        wg_bench_start_recv(b);
+        wg_bench_start_send(a);
+        wg_bench_start_send(b);
+    }
+
+    do {
+        struct timespec now;
+        struct wg_bench_statistics stats;
+        clock_gettime(CLOCK_MONOTONIC, &now);
+
+        // Fetch and render the statistics.
+        memset(&stats, 0, sizeof(stats));
+        wg_bench_fetch_stats(a, &stats);
+        wg_bench_fetch_stats(b, &stats);
+        print_stats(&stats, timespec_elapsed(&now, &start));
+
+        // Check for the end condition.
+        if (end.tv_sec > now.tv_sec) continue;
+        if (end.tv_sec < now.tv_sec) break;
+        if (end.tv_nsec < now.tv_nsec) break;
+    } while(usleep(100000) == 0);
+    printf("\n");
+
+    wg_bench_close(a);
+    wg_bench_close(b);
+}
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.c b/boringtun/benches/ffi_benches/wg_bench_client.c
new file mode 100644
index 000000000..261387156
--- /dev/null
+++ b/boringtun/benches/ffi_benches/wg_bench_client.c
@@ -0,0 +1,307 @@
+#include "wg_bench_client.h"
+
+#include <arpa/inet.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#define WG_BENCH_MTU 2048
+
+static void wg_worker_sigmask() {
+    sigset_t sigset;
+    sigemptyset(&sigset);
+    sigaddset(&sigset, SIGINT);
+    sigaddset(&sigset, SIGTERM);
+    pthread_sigmask(SIG_BLOCK, &sigset, NULL);
+}
+
+static void* wg_bench_background(void *arg) {
+    struct wg_bench_client *client = (struct wg_bench_client *)arg;
+    struct wireguard_result result;
+    uint8_t ciphertext[WG_BENCH_MTU + 32];
+
+    wg_worker_sigmask();
+    while (true) {
+        // Process timeouts and state updates.
+        result = wireguard_tick(client->tunnel, ciphertext, sizeof(ciphertext));
+        switch (result.op) {
+            case WIREGUARD_DONE:
+                // Sleep for a tick before trying again.
+                usleep(100000);
+                break;
+
+            case WIREGUARD_ERROR:
+                fprintf(stderr, "worker tick error: %zu\n", result.size);
+                break;
+
+            case WRITE_TO_NETWORK:
+                if (send(client->fd, ciphertext, result.size, MSG_DONTWAIT) < 0) {
+                    if (errno == EAGAIN) break;
+                    fprintf(stderr, "worker tick write: %s\n", strerror(errno));
+                }
+                break;
+
+            case WRITE_TO_TUNNEL_IPV4:
+            case WRITE_TO_TUNNEL_IPV6:
+                // not expected
+                fprintf(stderr, "worker tick tunnel");
+                break;
+            
+            default:
+                fprintf(stderr, "worker tick unknown: %d\n", result.op);
+                break;
+        }
+    }
+}
+
+struct wg_bench_iphdr {
+    uint8_t vlen;
+    uint8_t qos;
+    uint16_t tot_len;
+    uint16_t id;
+    uint16_t frag;
+    uint8_t ttl;
+    uint8_t protocol;
+    uint16_t cksum;
+    uint32_t saddr;
+    uint32_t daddr;
+};
+
+struct wg_bench_udphdr {
+    uint16_t sport;
+    uint16_t dport;
+    uint16_t length;
+    uint16_t cksum;
+    uint8_t  dgram[];
+};
+
+static void* wg_bench_send_worker(void *arg) {
+    struct wg_bench_client *client = (struct wg_bench_client *)arg;
+    struct wireguard_result result;
+    uint8_t ciphertext[WG_BENCH_MTU + 32];
+    uint8_t plaintext[WG_BENCH_MTU];
+
+    struct wg_bench_iphdr *ip = (struct wg_bench_iphdr*)plaintext;
+    struct wg_bench_udphdr *udp = (struct wg_bench_udphdr*)(ip+1);
+    const in_addr_t src = inet_addr("172.16.0.123");
+    const in_addr_t dst = inet_addr("172.16.0.1");
+
+    // Generate a sample IPv4 packet header.
+    memset(ip, 0, sizeof(struct wg_bench_iphdr));
+    ip->vlen = 0x45;
+    ip->ttl = 64;
+    ip->protocol = IPPROTO_UDP;
+    ip->saddr = htonl(src);
+    ip->daddr = htonl(dst);
+    udp->sport = 0x1234;
+    udp->dport = 0x5678;
+    udp->cksum = 0;
+
+    while (true) {
+        // Check if we are shutting down.
+        pthread_testcancel();
+
+        // Pick a random datagram size between 512-1024 and fill with a random byte.
+        int dsize = 512 + (rand() % 512);
+        int pktlen = sizeof(struct wg_bench_iphdr) + sizeof(struct wg_bench_udphdr) + dsize;
+        memset(udp->dgram, rand() & 0xff, dsize);
+        udp->length = htons(sizeof(struct wg_bench_udphdr) + dsize);
+        udp->cksum = 0;
+        ip->tot_len = htons(pktlen);
+        ip->cksum = 0;
+
+        // Encrypt the packet.
+        result = wireguard_write(client->tunnel, plaintext, pktlen, ciphertext, sizeof(ciphertext));
+        switch (result.op) {
+            case WIREGUARD_DONE:
+                break;
+
+            case WIREGUARD_ERROR:
+                if (result.size < WG_BENCH_MAX_ERRORS) {
+                    atomic_fetch_add(&client->stats.errors[result.size], 1);
+                }
+                fprintf(stderr, "worker encrypt error: %zu\n", result.size);
+                break;
+
+            case WRITE_TO_NETWORK:
+                // Not expected in this case.
+                atomic_fetch_add(&client->stats.tx_packets, 1);
+                atomic_fetch_add(&client->stats.tx_bytes, pktlen);
+                if (send(client->fd, ciphertext, result.size, MSG_DONTWAIT) < 0) {
+                    if (errno == EAGAIN) continue;
+                    if (!client->worker_shutdown) {
+                        fprintf(stderr, "worker tx error: %s\n", strerror(errno));
+                    }
+                    return NULL;
+                }
+                break;
+
+            case WRITE_TO_TUNNEL_IPV4:
+            case WRITE_TO_TUNNEL_IPV6:
+                // Not expected in this case.
+                fprintf(stderr, "worker encrypt tunnel\n");
+                break;
+
+            default:
+                fprintf(stderr, "worker encrypt unknown: %d\n", result.op);
+                break;
+        }
+    }
+
+    return NULL;
+}
+
+static void* wg_bench_recv_worker(void *arg) {
+    struct wg_bench_client *client = (struct wg_bench_client *)arg;
+    struct wireguard_result result;
+    uint8_t ciphertext[WG_BENCH_MTU + 32];
+    uint8_t plaintext[WG_BENCH_MTU];
+
+    wg_worker_sigmask();
+    while (true) {
+        // Read a packet.
+        ssize_t rx = recv(client->fd, ciphertext, sizeof(ciphertext), MSG_DONTWAIT);
+        if (rx == 0) {
+            fprintf(stderr, "worker shutdown\n");
+            return NULL;
+        }
+        if (rx < 0) {
+            if (errno == EAGAIN) continue;
+            if (!client->worker_shutdown) {
+                fprintf(stderr, "worker rx error: %s\n", strerror(errno));
+            }
+            return NULL;
+        }
+
+        // Decrypt the packet.
+        result = wireguard_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
+        switch (result.op) {
+            case WIREGUARD_DONE:
+                continue;
+            
+            case WIREGUARD_ERROR:
+                if (result.size < WG_BENCH_MAX_ERRORS) {
+                    atomic_fetch_add(&client->stats.errors[result.size], 1);
+                }
+                continue;
+
+            case WRITE_TO_NETWORK:
+                if (send(client->fd, plaintext, result.size, MSG_DONTWAIT) < 0) {
+                    //fprintf(stderr, "worker reply error: %s\n", strerror(errno));
+                }
+                continue;
+
+            case WRITE_TO_TUNNEL_IPV4:
+            case WRITE_TO_TUNNEL_IPV6:
+                atomic_fetch_add(&client->stats.rx_packets, 1);
+                atomic_fetch_add(&client->stats.rx_bytes, result.size);
+                break;
+            
+            default:
+                continue;
+        }
+    }
+
+    return NULL;
+}
+
+struct wg_bench_client* wg_bench_create(int fd) {
+    struct wg_bench_client* client = calloc(sizeof(struct wg_bench_client), 1);
+    if (!client) {
+        return NULL;
+    }
+    client->secret = x25519_secret_key();
+    client->pubkey = x25519_key_to_base64(x25519_public_key(client->secret));
+    client->fd = fd;
+    return client;
+}
+
+void wg_bench_connect(struct wg_bench_client* client, const char* pubkey) {
+    const char* statickey = x25519_key_to_base64(client->secret);
+    client->tunnel = new_tunnel(statickey, pubkey, NULL, 5, rand() & 0xffffff);
+    x25519_key_to_str_free(statickey);
+
+    // Start the background worker to drive wireguard_tick();
+    client->worker_count = 1;
+    pthread_create(&client->workers[0], NULL, wg_bench_background, client);
+}
+
+void wg_bench_start_handshake(struct wg_bench_client* client) {
+    uint8_t handshake[154];
+    struct wireguard_result result;
+    result = wireguard_force_handshake(client->tunnel, handshake, sizeof(handshake));
+    switch (result.op) {
+        case WIREGUARD_DONE:
+            break;
+        
+        case WIREGUARD_ERROR:
+            if (result.size < WG_BENCH_MAX_ERRORS) {
+                atomic_fetch_add(&client->stats.errors[result.size], 1);
+            }
+            break;
+
+        case WRITE_TO_NETWORK:
+            if (send(client->fd, handshake, result.size, MSG_DONTWAIT) < 0) {
+                //fprintf(stderr, "worker reply error: %s\n", strerror(errno));
+            }
+            break;
+
+        case WRITE_TO_TUNNEL_IPV4:
+        case WRITE_TO_TUNNEL_IPV6:
+            atomic_fetch_add(&client->stats.rx_packets, 1);
+            atomic_fetch_add(&client->stats.rx_bytes, result.size);
+            break;
+        
+        default:
+            break;
+    }
+}
+
+void wg_bench_start_recv(struct wg_bench_client* client) {
+    if (client->worker_count >= WG_BENCH_MAX_THREADS) {
+        return;
+    }
+    if (pthread_create(&client->workers[client->worker_count], NULL, wg_bench_recv_worker, client) != 0) {
+        return;
+    }
+    client->worker_count++;
+}
+
+void wg_bench_start_send(struct wg_bench_client* client) {
+    if (client->worker_count >= WG_BENCH_MAX_THREADS) {
+        return;
+    }
+    if (pthread_create(&client->workers[client->worker_count], NULL, wg_bench_send_worker, client) != 0) {
+        return;
+    }
+    client->worker_count++;
+}
+
+void wg_bench_fetch_stats(const struct wg_bench_client* client, struct wg_bench_statistics* st) {
+    atomic_fetch_add(&st->tx_packets, atomic_load(&client->stats.tx_packets));
+    atomic_fetch_add(&st->tx_bytes, atomic_load(&client->stats.tx_bytes));
+    atomic_fetch_add(&st->rx_packets, atomic_load(&client->stats.rx_packets));
+    atomic_fetch_add(&st->rx_bytes, atomic_load(&client->stats.rx_bytes));
+    for (int i = 0; i < WG_BENCH_MAX_ERRORS; i++) {
+        atomic_fetch_add(&st->errors[i], atomic_load(&client->stats.errors[i]));
+    }
+}
+
+void wg_bench_close(struct wg_bench_client* client) {
+    // Signal that we are shutting down and terminate workers.
+    client->worker_shutdown = 1;
+    for (int i = 0; i < client->worker_count; i++) {
+        pthread_cancel(client->workers[i]);
+        pthread_join(client->workers[i], NULL);
+    }
+
+    shutdown(client->fd, SHUT_RDWR);
+    close(client->fd);
+
+    x25519_key_to_str_free(client->pubkey);
+    tunnel_free(client->tunnel);
+}
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.h b/boringtun/benches/ffi_benches/wg_bench_client.h
new file mode 100644
index 000000000..b7ba8f122
--- /dev/null
+++ b/boringtun/benches/ffi_benches/wg_bench_client.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include <pthread.h>
+#include <stdatomic.h>
+#include "wireguard_ffi.h"
+
+#define WG_BENCH_MAX_THREADS 256
+#define WG_BENCH_MAX_ERRORS 32
+
+struct wg_bench_statistics {
+    atomic_uintmax_t tx_packets;
+    atomic_uintmax_t tx_bytes;
+    atomic_uintmax_t rx_packets;
+    atomic_uintmax_t rx_bytes;
+    atomic_uintmax_t errors[WG_BENCH_MAX_ERRORS];
+};
+
+struct wg_bench_client {
+    struct wireguard_tunnel *tunnel;
+    struct x25519_key secret;
+    const char *pubkey;
+    int fd;
+
+    int       worker_shutdown;
+    pthread_t worker_count;
+    pthread_t workers[WG_BENCH_MAX_THREADS];
+
+    // Statistics.
+    struct wg_bench_statistics stats;
+};
+
+struct wg_bench_client* wg_bench_create(int fd);
+void wg_bench_connect(struct wg_bench_client* client, const char* pubkey);
+void wg_bench_start_handshake(struct wg_bench_client* client);
+void wg_bench_start_send(struct wg_bench_client* client);
+void wg_bench_start_recv(struct wg_bench_client* client);
+void wg_bench_fetch_stats(const struct wg_bench_client* client, struct wg_bench_statistics* stats);
+void wg_bench_close(struct wg_bench_client* client);

From bf15bd684f664773ca5a25e1218a76eb962950e3 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Sun, 3 May 2026 15:58:59 -0700
Subject: [PATCH 02/25] Add error and CPU load reporting

---
 boringtun/benches/ffi_benches/main.c | 105 ++++++++++++++++++++-------
 1 file changed, 77 insertions(+), 28 deletions(-)

diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
index 59513789c..53ae2c726 100644
--- a/boringtun/benches/ffi_benches/main.c
+++ b/boringtun/benches/ffi_benches/main.c
@@ -34,47 +34,52 @@ static double timespec_elapsed(const struct timespec *a, const struct timespec*
 }
 
 static void print_header() {
-    printf("%012s %012s %012s %012s %08s  %-12s\n",
-           "TXPKT", "RXPKT", "TXBYTES", "RXBYTES", "ERRORS", "TRANSFERRED");
+    printf("%012s %012s %08s %08s  %-12s\n",
+           "TXPKT", "RXPKT", "ERRORS", "CPULOAD", "TRANSFERRED");
 }
 
-static void print_stats(const struct wg_bench_statistics* st, double elapsed) {
+static const char* print_bytes(uintmax_t value, char* buffer, size_t bufsize) {
+    const char* suffix = NULL;
+    double vfloat;
+    if (value > 1000000000) {
+        suffix = "GB";
+        vfloat = value / 1000000000.0;
+    } else if (value > 1000000) {
+        suffix = "MB";
+        vfloat = value / 1000000.0;
+    } else if (value > 1000.0) {
+        suffix = "kB";
+        vfloat = value / 1000.0;
+    } else {
+        suffix = "B";
+        vfloat = (double)value;
+    }
+
+    int len = snprintf(buffer, bufsize, "%.3F %s", vfloat, suffix);
+    return buffer;
+}
+
+static void print_stats(const struct wg_bench_statistics* st, double walltime, double cputime) {
     uintmax_t total_errors = 0;
     uintmax_t total_bytes = atomic_load(&st->tx_bytes) + atomic_load(&st->rx_bytes);
     for (int i = 0; i < WG_BENCH_MAX_ERRORS; i++) {
         total_errors += atomic_load(&st->errors[i]);
     }
 
+    char loadbuf[16];
+    snprintf(loadbuf, sizeof(loadbuf), "%.1F%%", 100.0 * cputime / walltime);
+
     // Estimate the total throughput.
     char xfer[32];
-    if (total_bytes > 1000000000) {
-        snprintf(xfer, sizeof(xfer), "%.3F GB", total_bytes / 1000000000.0);
-    } else if (total_bytes > 1000000) {
-        snprintf(xfer, sizeof(xfer), "%.3F MB", total_bytes / 1000000.0);
-    } else if (total_bytes > 1000) {
-        snprintf(xfer, sizeof(xfer), "%.3F kB", total_bytes / 1000.0);
-    } else {
-        snprintf(xfer, sizeof(xfer), "%.3F B", total_bytes);
-    }
-
     char tpbuf[32];
-    double throughput = (double)total_bytes / elapsed;
-    if (throughput > 1000000000) {
-        snprintf(tpbuf, sizeof(tpbuf), "%s (%.3F GB/s)", xfer, throughput / 1000000000.0);
-    } else if (throughput > 1000000) {
-        snprintf(tpbuf, sizeof(tpbuf), "%s (%.3F MB/s)", xfer, throughput / 1000000.0);
-    } else if (throughput > 1000) {
-        snprintf(tpbuf, sizeof(tpbuf), "%s (%.3F kB/s)", xfer, throughput / 1000.0);
-    } else {
-        snprintf(tpbuf, sizeof(tpbuf), "%s (%.3F B/s)", xfer, throughput);
-    }
+    uintmax_t throughput = total_bytes / walltime;
 
     // Prepare the status to write.
     char linebuf[96];
-    int len = snprintf(linebuf, sizeof(linebuf), "\r%12lu %12lu %12lu %12lu %8lu  %-s",
+    int len = snprintf(linebuf, sizeof(linebuf), "\r%12lu %12lu %8lu %08s  %s (%s)",
                        atomic_load(&st->tx_packets), atomic_load(&st->rx_packets),
-                       atomic_load(&st->tx_bytes), atomic_load(&st->rx_bytes),
-                       total_errors, tpbuf);
+                       total_errors, loadbuf, print_bytes(total_bytes, xfer, sizeof(xfer)),
+                       print_bytes(total_bytes / walltime, tpbuf, sizeof(tpbuf)));
 
     struct winsize ws;
     if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) == 0) {
@@ -95,6 +100,42 @@ static void print_stats(const struct wg_bench_statistics* st, double elapsed) {
     }
 }
 
+static void print_errors(const struct wg_bench_statistics* st) {
+    // From errors.rs
+    const char* names[] = {
+        "DestinationBufferTooSmall",
+        "IncorrectPacketLength",
+        "UnexpectedPacket",
+        "WrongPacketType",
+        "WrongIndex",
+        "WrongKey",
+        "InvalidTai64nTimestamp",
+        "WrongTai64nTimestamp",
+        "InvalidMac",
+        "InvalidAeadTag",
+        "InvalidCounter",
+        "DuplicateCounter",
+        "InvalidPacket",
+        "NoCurrentSession",
+        "LockFailed",
+        "ConnectionExpired",
+        "UnderLoad",
+    };
+    const int maxerr = sizeof(names)/sizeof(char*);
+
+    int maxname = 0;
+    for (int i = 0; i < maxerr; i++) {
+        if (strlen(names[i]) > maxname) {
+            maxname = strlen(names[i]);
+        }
+    }
+
+    printf("\nError Report:\n");
+    for (int i = 0; i < maxerr; i++) {
+        printf("   %*s: %lu\n", maxname, names[i], atomic_load(&st->errors[i]));
+    }
+}
+
 static void print_usage(FILE* fp, const char* name) {
     fprintf(fp, "Usage: %s [OPTIONS]\n");
     fprintf(fp, "Run FFI benchmarks for the boringtun library.\n");
@@ -162,8 +203,10 @@ int main(int argc, char* argv[]) {
     wg_bench_connect(b, a->pubkey);
 
     struct timespec start;
+    struct timespec cpustart;
     struct timespec end;
     clock_gettime(CLOCK_MONOTONIC, &start);
+    clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &cpustart);
     end.tv_sec = start.tv_sec + 10;
     end.tv_nsec = start.tv_nsec;
     print_header();
@@ -177,23 +220,29 @@ int main(int argc, char* argv[]) {
         wg_bench_start_send(b);
     }
 
+    struct wg_bench_statistics stats;
     do {
         struct timespec now;
-        struct wg_bench_statistics stats;
+        struct timespec cpu;
         clock_gettime(CLOCK_MONOTONIC, &now);
+        clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &cpu);
 
         // Fetch and render the statistics.
         memset(&stats, 0, sizeof(stats));
         wg_bench_fetch_stats(a, &stats);
         wg_bench_fetch_stats(b, &stats);
-        print_stats(&stats, timespec_elapsed(&now, &start));
+        print_stats(&stats, timespec_elapsed(&now, &start), timespec_elapsed(&cpu, &cpustart));
 
         // Check for the end condition.
         if (end.tv_sec > now.tv_sec) continue;
         if (end.tv_sec < now.tv_sec) break;
         if (end.tv_nsec < now.tv_nsec) break;
     } while(usleep(100000) == 0);
+
     printf("\n");
+    wg_bench_fetch_stats(a, &stats);
+    wg_bench_fetch_stats(b, &stats);
+    print_errors(&stats);
 
     wg_bench_close(a);
     wg_bench_close(b);

From 7ec07fa5962d92790aa70225367f655a4bb3889b Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Sun, 3 May 2026 17:31:58 -0700
Subject: [PATCH 03/25] Generate lot messages from boringtun callback

---
 boringtun/benches/ffi_benches/main.c | 82 ++++++++++++++++++++--------
 1 file changed, 60 insertions(+), 22 deletions(-)

diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
index 53ae2c726..9cd8662aa 100644
--- a/boringtun/benches/ffi_benches/main.c
+++ b/boringtun/benches/ffi_benches/main.c
@@ -1,14 +1,18 @@
 
 
 #include <errno.h>
+#include <fcntl.h>
 #include <getopt.h>
+#include <limits.h>
 #include <signal.h>
+#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <asm/termbits.h>  /* Definition of TIOC*WINSZ constants */
 #include <sys/ioctl.h>
 #include <sys/socket.h>
+#include <sys/time.h>
 #include <time.h> 
 #include <pthread.h>
 #include <unistd.h>
@@ -16,14 +20,46 @@
 #include "wireguard_ffi.h"
 #include "wg_bench_client.h"
 
+// Exiting because of a signal.
+static int caught_sigint = 0;
+
+static void wg_printf(const char* format, ...) {
+    struct winsize ws;
+    if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) == 0) {
+        // If on a terminal - clear the line and reset before printing.
+        char fmtbuf[ws.ws_col + strlen(format) + 3];
+        fmtbuf[0] = '\r';
+        memset(&fmtbuf[1], ' ', ws.ws_col);
+        fmtbuf[ws.ws_col+1] = '\r';
+        strcpy(&fmtbuf[2+ws.ws_col], format);
+
+        va_list args;
+        va_start(args, format);
+        vprintf(fmtbuf, args);
+        va_end(args);
+    } else {
+        // Otherwise, just print it.
+        va_list args;
+        va_start(args, format);
+        vprintf(format, args);
+        va_end(args);
+    }
+}
+
+static void wg_print_msg(const char* msg) {
+    wg_printf("%s", msg);
+}
+
 static void handle_signal(int sig) {
     switch (sig) {
         case SIGINT:
-            fprintf(stderr, "benchmark interrupted\n");
+            wg_printf("benchmark interrupted\n");
+            caught_sigint = 1;
             break;
 
         case SIGTERM:
-            fprintf(stderr, "benchmark terminated\n");
+            wg_printf("benchmark terminated\n");
+            caught_sigint = 1;
             break;
     }
 }
@@ -33,11 +69,6 @@ static double timespec_elapsed(const struct timespec *a, const struct timespec*
     return (double)(result + a->tv_nsec - b->tv_nsec) / 1000000000.0;
 }
 
-static void print_header() {
-    printf("%012s %012s %08s %08s  %-12s\n",
-           "TXPKT", "RXPKT", "ERRORS", "CPULOAD", "TRANSFERRED");
-}
-
 static const char* print_bytes(uintmax_t value, char* buffer, size_t bufsize) {
     const char* suffix = NULL;
     double vfloat;
@@ -75,9 +106,10 @@ static void print_stats(const struct wg_bench_statistics* st, double walltime, d
     uintmax_t throughput = total_bytes / walltime;
 
     // Prepare the status to write.
-    char linebuf[96];
-    int len = snprintf(linebuf, sizeof(linebuf), "\r%12lu %12lu %8lu %08s  %s (%s)",
-                       atomic_load(&st->tx_packets), atomic_load(&st->rx_packets),
+    char linebuf[120];
+    int len = snprintf(linebuf, sizeof(linebuf),
+                       "\r   tx:%-12lu rx:%-12lu drops:%-8lu err:%-8lu load:%08s  %s transferred (%s/s)",
+                       atomic_load(&st->tx_packets), atomic_load(&st->rx_packets), atomic_load(&st->tx_drops),
                        total_errors, loadbuf, print_bytes(total_bytes, xfer, sizeof(xfer)),
                        print_bytes(total_bytes / walltime, tpbuf, sizeof(tpbuf)));
 
@@ -91,7 +123,6 @@ static void print_stats(const struct wg_bench_statistics* st, double walltime, d
             len = sizeof(linebuf);
         }
         write(STDOUT_FILENO, linebuf, len);
-
     } else {
         // Some other file.
         linebuf[len] = '\n';
@@ -130,9 +161,9 @@ static void print_errors(const struct wg_bench_statistics* st) {
         }
     }
 
-    printf("\nError Report:\n");
+    wg_printf("\nError Report:\n");
     for (int i = 0; i < maxerr; i++) {
-        printf("   %*s: %lu\n", maxname, names[i], atomic_load(&st->errors[i]));
+        wg_printf("   %*s: %lu\n", maxname, names[i], atomic_load(&st->errors[i]));
     }
 }
 
@@ -182,6 +213,7 @@ int main(int argc, char* argv[]) {
     }
 
     srand(time(0));
+    set_logging_function(wg_print_msg);
 
     // This thread should handle signals.
     struct sigaction action = {
@@ -209,7 +241,6 @@ int main(int argc, char* argv[]) {
     clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &cpustart);
     end.tv_sec = start.tv_sec + 10;
     end.tv_nsec = start.tv_nsec;
-    print_header();
 
     // Launch workers.
     wg_bench_start_handshake(a);
@@ -220,10 +251,10 @@ int main(int argc, char* argv[]) {
         wg_bench_start_send(b);
     }
 
+    struct timespec now;
+    struct timespec cpu;
     struct wg_bench_statistics stats;
-    do {
-        struct timespec now;
-        struct timespec cpu;
+    while (!caught_sigint) {
         clock_gettime(CLOCK_MONOTONIC, &now);
         clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &cpu);
 
@@ -234,15 +265,22 @@ int main(int argc, char* argv[]) {
         print_stats(&stats, timespec_elapsed(&now, &start), timespec_elapsed(&cpu, &cpustart));
 
         // Check for the end condition.
-        if (end.tv_sec > now.tv_sec) continue;
-        if (end.tv_sec < now.tv_sec) break;
-        if (end.tv_nsec < now.tv_nsec) break;
-    } while(usleep(100000) == 0);
+        if (end.tv_sec < now.tv_sec) {
+            break;
+        } else if ((end.tv_sec == now.tv_sec) && (end.tv_nsec < now.tv_nsec)) {
+            break;
+        }
+
+        // Sleep for more data.
+        usleep(100000);
+    }
 
-    printf("\n");
+    clock_gettime(CLOCK_MONOTONIC, &now);
+    clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &cpu);
     wg_bench_fetch_stats(a, &stats);
     wg_bench_fetch_stats(b, &stats);
     print_errors(&stats);
+    print_stats(&stats, timespec_elapsed(&now, &start), timespec_elapsed(&cpu, &cpustart));
 
     wg_bench_close(a);
     wg_bench_close(b);

From 0cda5f6539b45c357ebfbdb0814d6656d25e83cf Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Sun, 3 May 2026 17:32:56 -0700
Subject: [PATCH 04/25] Implement read-only try_encapsulate and try_decapsulate
 methods

---
 boringtun/src/ffi/mod.rs      | 62 +++++++++++++++++++------
 boringtun/src/noise/mod.rs    | 81 ++++++++++++++++++++++++++-------
 boringtun/src/noise/timers.rs | 85 ++++++++++++++++-------------------
 boringtun/src/wireguard_ffi.h | 12 +++++
 4 files changed, 162 insertions(+), 78 deletions(-)

diff --git a/boringtun/src/ffi/mod.rs b/boringtun/src/ffi/mod.rs
index 1e5a2a9f3..9590e8173 100644
--- a/boringtun/src/ffi/mod.rs
+++ b/boringtun/src/ffi/mod.rs
@@ -11,7 +11,7 @@ use crate::x25519::{PublicKey, StaticSecret};
 use base64::{decode, encode};
 use hex::encode as encode_hex;
 use libc::{raise, SIGSEGV};
-use parking_lot::Mutex;
+use parking_lot::RwLock;
 use rand_core::OsRng;
 use tracing;
 use tracing_subscriber::fmt;
@@ -247,7 +247,7 @@ pub unsafe extern "C" fn new_tunnel(
     preshared_key: *const c_char,
     keep_alive: u16,
     index: u32,
-) -> *mut Mutex<Tunn> {
+) -> *mut RwLock<Tunn> {
     let c_str = CStr::from_ptr(static_private);
     let static_private = match c_str.to_str() {
         Err(_) => return ptr::null_mut(),
@@ -292,7 +292,7 @@ pub unsafe extern "C" fn new_tunnel(
         Some(keep_alive)
     };
 
-    let tunnel = Box::new(Mutex::new(Tunn::new(
+    let tunnel = Box::new(RwLock::new(Tunn::new(
         private_key,
         public_key,
         preshared_key,
@@ -313,7 +313,7 @@ pub unsafe extern "C" fn new_tunnel(
 
 /// Drops the Tunn object
 #[no_mangle]
-pub unsafe extern "C" fn tunnel_free(tunnel: *mut Mutex<Tunn>) {
+pub unsafe extern "C" fn tunnel_free(tunnel: *mut RwLock<Tunn>) {
     drop(Box::from_raw(tunnel));
 }
 
@@ -321,45 +321,79 @@ pub unsafe extern "C" fn tunnel_free(tunnel: *mut Mutex<Tunn>) {
 /// For more details check noise::tunnel_to_network functions.
 #[no_mangle]
 pub unsafe extern "C" fn wireguard_write(
-    tunnel: *const Mutex<Tunn>,
+    tunnel: *const RwLock<Tunn>,
     src: *const u8,
     src_size: u32,
     dst: *mut u8,
     dst_size: u32,
 ) -> wireguard_result {
-    let mut tunnel = tunnel.as_ref().unwrap().lock();
+    let mut tunnel = tunnel.as_ref().unwrap().write();
     // Slices are not owned, and therefore will not be freed by Rust
     let src = slice::from_raw_parts(src, src_size as usize);
     let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
     wireguard_result::from(tunnel.encapsulate(src, dst))
 }
 
+/// Write an IP packet from the tunnel interface.
+/// For more details check noise::tunnel_to_network functions.
+#[no_mangle]
+pub unsafe extern "C" fn wireguard_try_write(
+    tunnel: *const RwLock<Tunn>,
+    src: *const u8,
+    src_size: u32,
+    dst: *mut u8,
+    dst_size: u32,
+) -> wireguard_result {
+    let tunnel = tunnel.as_ref().unwrap().read();
+    // Slices are not owned, and therefore will not be freed by Rust
+    let src = slice::from_raw_parts(src, src_size as usize);
+    let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
+    wireguard_result::from(tunnel.try_encapsulate(src, dst))
+}
+
 /// Read a UDP packet from the server.
 /// For more details check noise::network_to_tunnel functions.
 #[no_mangle]
 pub unsafe extern "C" fn wireguard_read(
-    tunnel: *const Mutex<Tunn>,
+    tunnel: *const RwLock<Tunn>,
     src: *const u8,
     src_size: u32,
     dst: *mut u8,
     dst_size: u32,
 ) -> wireguard_result {
-    let mut tunnel = tunnel.as_ref().unwrap().lock();
+    let mut tunnel = tunnel.as_ref().unwrap().write();
     // Slices are not owned, and therefore will not be freed by Rust
     let src = slice::from_raw_parts(src, src_size as usize);
     let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
     wireguard_result::from(tunnel.decapsulate(None, src, dst))
 }
 
+/// Read a UDP packet from the server.
+/// For more details check noise::network_to_tunnel functions.
+#[no_mangle]
+pub unsafe extern "C" fn wireguard_try_read(
+    tunnel: *const RwLock<Tunn>,
+    src: *const u8,
+    src_size: u32,
+    dst: *mut u8,
+    dst_size: u32,
+) -> wireguard_result {
+    let tunnel = tunnel.as_ref().unwrap().read();
+    // Slices are not owned, and therefore will not be freed by Rust
+    let src = slice::from_raw_parts(src, src_size as usize);
+    let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
+    wireguard_result::from(tunnel.try_decapsulate(src, dst))
+}
+
 /// This is a state keeping function, that need to be called periodically.
 /// Recommended interval: 100ms.
 #[no_mangle]
 pub unsafe extern "C" fn wireguard_tick(
-    tunnel: *const Mutex<Tunn>,
+    tunnel: *const RwLock<Tunn>,
     dst: *mut u8,
     dst_size: u32,
 ) -> wireguard_result {
-    let mut tunnel = tunnel.as_ref().unwrap().lock();
+    let mut tunnel = tunnel.as_ref().unwrap().write();
     // Slices are not owned, and therefore will not be freed by Rust
     let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
     wireguard_result::from(tunnel.update_timers(dst))
@@ -368,11 +402,11 @@ pub unsafe extern "C" fn wireguard_tick(
 /// Force the tunnel to initiate a new handshake, dst buffer must be at least 148 byte long.
 #[no_mangle]
 pub unsafe extern "C" fn wireguard_force_handshake(
-    tunnel: *const Mutex<Tunn>,
+    tunnel: *const RwLock<Tunn>,
     dst: *mut u8,
     dst_size: u32,
 ) -> wireguard_result {
-    let mut tunnel = tunnel.as_ref().unwrap().lock();
+    let mut tunnel = tunnel.as_ref().unwrap().write();
     // Slices are not owned, and therefore will not be freed by Rust
     let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
     wireguard_result::from(tunnel.format_handshake_initiation(dst, true))
@@ -383,8 +417,8 @@ pub unsafe extern "C" fn wireguard_force_handshake(
 /// Number of data bytes encapsulated
 /// Number of data bytes decapsulated
 #[no_mangle]
-pub unsafe extern "C" fn wireguard_stats(tunnel: *const Mutex<Tunn>) -> stats {
-    let tunnel = tunnel.as_ref().unwrap().lock();
+pub unsafe extern "C" fn wireguard_stats(tunnel: *const RwLock<Tunn>) -> stats {
+    let tunnel = tunnel.as_ref().unwrap().read();
     let (time, tx_bytes, rx_bytes, estimated_loss, estimated_rtt) = tunnel.stats();
     stats {
         time_since_last_handshake: time.map(|t| t.as_secs() as i64).unwrap_or(-1),
diff --git a/boringtun/src/noise/mod.rs b/boringtun/src/noise/mod.rs
index 76e377b63..114bb2977 100644
--- a/boringtun/src/noise/mod.rs
+++ b/boringtun/src/noise/mod.rs
@@ -18,6 +18,7 @@ use std::collections::VecDeque;
 use std::convert::{TryFrom, TryInto};
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::Duration;
 
 /// The default value to use for rate limiting, when no other rate limiter is defined
@@ -63,13 +64,13 @@ pub struct Tunn {
     /// The N_SESSIONS most recent sessions, index is session id modulo N_SESSIONS
     sessions: [Option<session::Session>; N_SESSIONS],
     /// Index of most recently used session
-    current: usize,
+    current: AtomicUsize,
     /// Queue to store blocked packets
     packet_queue: VecDeque<Vec<u8>>,
     /// Keeps tabs on the expiring timers
     timers: timers::Timers,
-    tx_bytes: usize,
-    rx_bytes: usize,
+    tx_bytes: AtomicUsize,
+    rx_bytes: AtomicUsize,
     rate_limiter: Arc<RateLimiter>,
 }
 
@@ -241,14 +242,14 @@ impl Tunn {
         }
     }
 
-    /// Encapsulate a single packet from the tunnel interface.
+    /// Encapsulate a single packet from the tunnel interface or queue it.
     /// Returns TunnResult.
     ///
     /// # Panics
     /// Panics if dst buffer is too small.
     /// Size of dst should be at least src.len() + 32, and no less than 148 bytes.
     pub fn encapsulate<'a>(&mut self, src: &[u8], dst: &'a mut [u8]) -> TunnResult<'a> {
-        let current = self.current;
+        let current = self.current.load(Ordering::Relaxed);
         if let Some(ref session) = self.sessions[current % N_SESSIONS] {
             // Send the packet using an established session
             let packet = session.format_packet_data(src, dst);
@@ -257,14 +258,31 @@ impl Tunn {
             if !src.is_empty() {
                 self.timer_tick(TimerName::TimeLastDataPacketSent);
             }
-            self.tx_bytes += src.len();
+            self.tx_bytes.fetch_add(src.len(), Ordering::Relaxed);
             return TunnResult::WriteToNetwork(packet);
         }
 
         // If there is no session, queue the packet for future retry
         self.queue_packet(src);
         // Initiate a new handshake if none is in progress
-        self.format_handshake_initiation(dst, false)
+        return self.format_handshake_initiation(dst, false);
+    }
+
+    pub fn try_encapsulate<'a>(&self, src: &[u8], dst: &'a mut [u8]) -> TunnResult<'a> {
+        let current = self.current.load(Ordering::Relaxed);
+        if let Some(ref session) = self.sessions[current % N_SESSIONS] {
+            // Send the packet using an established session
+            let packet = session.format_packet_data(src, dst);
+            self.timer_tick(TimerName::TimeLastPacketSent);
+            // Exclude Keepalive packets from timer update.
+            if !src.is_empty() {
+                self.timer_tick(TimerName::TimeLastDataPacketSent);
+            }
+            self.tx_bytes.fetch_add(src.len(), Ordering::Relaxed);
+            return TunnResult::WriteToNetwork(packet);
+        }
+
+        TunnResult::Done
     }
 
     /// Receives a UDP datagram from the network and parses it.
@@ -301,6 +319,33 @@ impl Tunn {
         self.handle_verified_packet(packet, dst)
     }
 
+    /// Receives a UDP datagram from the network and parses it.
+    /// Returns TunnResult.
+    ///
+    /// This is a subset of decapsulate that only accepts PacketData, but can be called without needing
+    /// to acquire a write lock on the tunnel state. Callers should verify the packet type before calling
+    /// this method.
+    pub fn try_decapsulate<'a>(
+        &self,
+        datagram: &[u8],
+        dst: &'a mut [u8],
+    ) -> TunnResult<'a> {
+        // Dequeueing not supported, caller should use decapsulate() instead.
+        if datagram.is_empty() {
+            return TunnResult::Done;
+        }
+
+        // Handle the packet if, and only if, it's a data packet.
+        if let Ok(packet) = Tunn::parse_incoming_packet(datagram) {
+            match packet {
+                Packet::PacketData(p) => self.handle_data(p, dst).unwrap_or_else(TunnResult::from),
+                _ => TunnResult::Done
+            }
+        } else {
+            TunnResult::Done
+        }
+    }
+
     pub(crate) fn handle_verified_packet<'a>(
         &mut self,
         packet: Packet,
@@ -387,24 +432,26 @@ impl Tunn {
     }
 
     /// Update the index of the currently used session, if needed
-    fn set_current_session(&mut self, new_idx: usize) {
-        let cur_idx = self.current;
+    fn set_current_session(&self, new_idx: usize) {
+        let cur_idx = self.current.load(Ordering::Relaxed);
         if cur_idx == new_idx {
             // There is nothing to do, already using this session, this is the common case
             return;
         }
+        
         if self.sessions[cur_idx % N_SESSIONS].is_none()
             || self.timers.session_timers[new_idx % N_SESSIONS]
                 >= self.timers.session_timers[cur_idx % N_SESSIONS]
         {
-            self.current = new_idx;
-            tracing::debug!(message = "New session", session = new_idx);
+            if let Ok(idx) = self.current.compare_exchange(cur_idx, new_idx, Ordering::Acquire, Ordering::Relaxed) {
+                tracing::debug!(message = "New session", session = idx);
+            }
         }
     }
 
     /// Decrypts a data packet, and stores the decapsulated packet in dst.
     fn handle_data<'a>(
-        &mut self,
+        &self,
         packet: PacketData,
         dst: &'a mut [u8],
     ) -> Result<TunnResult<'a>, WireGuardError> {
@@ -461,7 +508,7 @@ impl Tunn {
 
     /// Check if an IP packet is v4 or v6, truncate to the length indicated by the length field
     /// Returns the truncated packet and the source IP as TunnResult
-    fn validate_decapsulated_packet<'a>(&mut self, packet: &'a mut [u8]) -> TunnResult<'a> {
+    fn validate_decapsulated_packet<'a>(&self, packet: &'a mut [u8]) -> TunnResult<'a> {
         let (computed_len, src_ip_address) = match packet.len() {
             0 => return TunnResult::Done, // This is keepalive, and not an error
             _ if packet[0] >> 4 == 4 && packet.len() >= IPV4_MIN_HEADER_SIZE => {
@@ -498,7 +545,7 @@ impl Tunn {
         }
 
         self.timer_tick(TimerName::TimeLastDataPacketReceived);
-        self.rx_bytes += computed_len;
+        self.rx_bytes.fetch_add(computed_len, Ordering::Relaxed);
 
         match src_ip_address {
             IpAddr::V4(addr) => TunnResult::WriteToTunnelV4(&mut packet[..computed_len], addr),
@@ -541,7 +588,7 @@ impl Tunn {
     }
 
     fn estimate_loss(&self) -> f32 {
-        let session_idx = self.current;
+        let session_idx = self.current.load(Ordering::Relaxed);
 
         let mut weight = 9.0;
         let mut cur_avg = 0.0;
@@ -576,8 +623,8 @@ impl Tunn {
     /// * Data bytes received
     pub fn stats(&self) -> (Option<Duration>, usize, usize, f32, Option<u32>) {
         let time = self.time_since_last_handshake();
-        let tx_bytes = self.tx_bytes;
-        let rx_bytes = self.rx_bytes;
+        let tx_bytes = self.tx_bytes.load(Ordering::Relaxed);
+        let rx_bytes = self.rx_bytes.load(Ordering::Relaxed);
         let loss = self.estimate_loss();
         let rtt = self.handshake.last_rtt;
 
diff --git a/boringtun/src/noise/timers.rs b/boringtun/src/noise/timers.rs
index 6b91d5767..d657f0966 100644
--- a/boringtun/src/noise/timers.rs
+++ b/boringtun/src/noise/timers.rs
@@ -3,8 +3,7 @@
 
 use super::errors::WireGuardError;
 use crate::noise::{Tunn, TunnResult};
-use std::mem;
-use std::ops::{Index, IndexMut};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 
 use std::time::Duration;
 
@@ -25,8 +24,6 @@ const COOKIE_EXPIRATION_TIME: Duration = Duration::from_secs(120);
 
 #[derive(Debug)]
 pub enum TimerName {
-    /// Current time, updated each call to `update_timers`
-    TimeCurrent,
     /// Time when last handshake was completed
     TimeSessionEstablished,
     /// Time the last attempt for a new handshake began
@@ -54,12 +51,14 @@ pub struct Timers {
     is_initiator: bool,
     /// Start time of the tunnel
     time_started: Instant,
-    timers: [Duration; TimerName::Top as usize],
+    /// Current time, updated each call to `update_timers`
+    current: Duration,
+    timers: [AtomicU64; TimerName::Top as usize],
     pub(super) session_timers: [Duration; super::N_SESSIONS],
     /// Did we receive data without sending anything back?
-    want_keepalive: bool,
+    want_keepalive: AtomicBool,
     /// Did we send data without hearing back?
-    want_handshake: bool,
+    want_handshake: AtomicBool,
     persistent_keepalive: usize,
     /// Should this timer call reset rr function (if not a shared rr instance)
     pub(super) should_reset_rr: bool,
@@ -70,10 +69,11 @@ impl Timers {
         Timers {
             is_initiator: false,
             time_started: Instant::now(),
+            current: Default::default(),
             timers: Default::default(),
             session_timers: Default::default(),
-            want_keepalive: Default::default(),
-            want_handshake: Default::default(),
+            want_keepalive: AtomicBool::new(false),
+            want_handshake: AtomicBool::new(false),
             persistent_keepalive: usize::from(persistent_keepalive.unwrap_or(0)),
             should_reset_rr: reset_rr,
         }
@@ -87,43 +87,35 @@ impl Timers {
     // so the reference time frame is the same
     pub(super) fn clear(&mut self) {
         let now = Instant::now().duration_since(self.time_started);
+        let msec = now.as_millis() as u64;
         for t in &mut self.timers[..] {
-            *t = now;
+            t.store(msec, Ordering::Release);
         }
-        self.want_handshake = false;
-        self.want_keepalive = false;
-    }
-}
-
-impl Index<TimerName> for Timers {
-    type Output = Duration;
-    fn index(&self, index: TimerName) -> &Duration {
-        &self.timers[index as usize]
-    }
-}
-
-impl IndexMut<TimerName> for Timers {
-    fn index_mut(&mut self, index: TimerName) -> &mut Duration {
-        &mut self.timers[index as usize]
+        self.want_handshake.store(false, Ordering::Release);
+        self.want_keepalive.store(false, Ordering::Release);
     }
 }
 
 impl Tunn {
-    pub(super) fn timer_tick(&mut self, timer_name: TimerName) {
+    pub(super) fn timer_tick(&self, timer_name: TimerName) {
         match timer_name {
             TimeLastPacketReceived => {
-                self.timers.want_keepalive = true;
-                self.timers.want_handshake = false;
+                self.timers.want_keepalive.store(true, Ordering::Release);
+                self.timers.want_handshake.store(false, Ordering::Release);
             }
             TimeLastPacketSent => {
-                self.timers.want_handshake = true;
-                self.timers.want_keepalive = false;
+                self.timers.want_handshake.store(true, Ordering::Release);
+                self.timers.want_keepalive.store(false, Ordering::Release);
             }
             _ => {}
         }
 
-        let time = self.timers[TimeCurrent];
-        self.timers[timer_name] = time;
+        let msecs = self.timers.current.as_millis() as u64;
+        self.timers.timers[timer_name as usize].store(msecs, Ordering::Release);
+    }
+
+    pub(super) fn timer_fetch(&self, timer_name: TimerName) -> Duration {
+        Duration::from_millis(self.timers.timers[timer_name as usize].load(Ordering::Acquire))
     }
 
     pub(super) fn timer_tick_session_established(
@@ -132,8 +124,7 @@ impl Tunn {
         session_idx: usize,
     ) {
         self.timer_tick(TimeSessionEstablished);
-        self.timers.session_timers[session_idx % crate::noise::N_SESSIONS] =
-            self.timers[TimeCurrent];
+        self.timers.session_timers[session_idx % crate::noise::N_SESSIONS] = self.timers.current;
         self.timers.is_initiator = is_initiator;
     }
 
@@ -178,17 +169,17 @@ impl Tunn {
         // All the times are counted from tunnel initiation, for efficiency our timers are rounded
         // to a second, as there is no real benefit to having highly accurate timers.
         let now = time.duration_since(self.timers.time_started);
-        self.timers[TimeCurrent] = now;
+        self.timers.current = now;
 
         self.update_session_timers(now);
 
         // Load timers only once:
-        let session_established = self.timers[TimeSessionEstablished];
-        let handshake_started = self.timers[TimeLastHandshakeStarted];
-        let aut_packet_received = self.timers[TimeLastPacketReceived];
-        let aut_packet_sent = self.timers[TimeLastPacketSent];
-        let data_packet_received = self.timers[TimeLastDataPacketReceived];
-        let data_packet_sent = self.timers[TimeLastDataPacketSent];
+        let session_established = self.timer_fetch(TimeSessionEstablished);
+        let handshake_started = self.timer_fetch(TimeLastHandshakeStarted);
+        let aut_packet_received = self.timer_fetch(TimeLastPacketReceived);
+        let aut_packet_sent = self.timer_fetch(TimeLastPacketSent);
+        let data_packet_received = self.timer_fetch(TimeLastDataPacketReceived);
+        let data_packet_sent = self.timer_fetch(TimeLastDataPacketSent);
         let persistent_keepalive = self.timers.persistent_keepalive;
 
         {
@@ -198,7 +189,7 @@ impl Tunn {
 
             // Clear cookie after COOKIE_EXPIRATION_TIME
             if self.handshake.has_cookie()
-                && now - self.timers[TimeCookieReceived] >= COOKIE_EXPIRATION_TIME
+                && now - self.timer_fetch(TimeCookieReceived) >= COOKIE_EXPIRATION_TIME
             {
                 self.handshake.clear_cookie();
             }
@@ -270,7 +261,7 @@ impl Tunn {
                 // we initiate a new handshake.
                 if data_packet_sent > aut_packet_received
                     && now - aut_packet_received >= KEEPALIVE_TIMEOUT + REKEY_TIMEOUT
-                    && mem::replace(&mut self.timers.want_handshake, false)
+                    && self.timers.want_handshake.swap(false, Ordering::AcqRel)
                 {
                     tracing::warn!("HANDSHAKE(KEEPALIVE + REKEY_TIMEOUT)");
                     handshake_initiation_required = true;
@@ -281,7 +272,7 @@ impl Tunn {
                     // to the given peer in KEEPALIVE ms, we send an empty packet.
                     if data_packet_received > aut_packet_sent
                         && now - aut_packet_sent >= KEEPALIVE_TIMEOUT
-                        && mem::replace(&mut self.timers.want_keepalive, false)
+                        && self.timers.want_keepalive.swap(false, Ordering::AcqRel)
                     {
                         tracing::debug!("KEEPALIVE(KEEPALIVE_TIMEOUT)");
                         keepalive_required = true;
@@ -289,7 +280,7 @@ impl Tunn {
 
                     // Persistent KEEPALIVE
                     if persistent_keepalive > 0
-                        && (now - self.timers[TimePersistentKeepalive]
+                        && (now - self.timer_fetch(TimePersistentKeepalive)
                             >= Duration::from_secs(persistent_keepalive as _))
                     {
                         tracing::debug!("KEEPALIVE(PERSISTENT_KEEPALIVE)");
@@ -312,10 +303,10 @@ impl Tunn {
     }
 
     pub fn time_since_last_handshake(&self) -> Option<Duration> {
-        let current_session = self.current;
+        let current_session = self.current.load(Ordering::Relaxed);
         if self.sessions[current_session % super::N_SESSIONS].is_some() {
             let duration_since_tun_start = Instant::now().duration_since(self.timers.time_started);
-            let duration_since_session_established = self.timers[TimeSessionEstablished];
+            let duration_since_session_established = self.timer_fetch(TimeSessionEstablished);
 
             Some(duration_since_tun_start - duration_since_session_established)
         } else {
diff --git a/boringtun/src/wireguard_ffi.h b/boringtun/src/wireguard_ffi.h
index 5cdd90125..724d8fce6 100644
--- a/boringtun/src/wireguard_ffi.h
+++ b/boringtun/src/wireguard_ffi.h
@@ -89,12 +89,24 @@ struct wireguard_result wireguard_write(const struct wireguard_tunnel *tunnel,
                                         uint8_t *dst,
                                         uint32_t dst_size);
 
+struct wireguard_result wireguard_try_write(const struct wireguard_tunnel *tunnel,
+                                            const uint8_t *src,
+                                            uint32_t src_size,
+                                            uint8_t *dst,
+                                            uint32_t dst_size);
+
 struct wireguard_result wireguard_read(const struct wireguard_tunnel *tunnel,
                                        const uint8_t *src,
                                        uint32_t src_size,
                                        uint8_t *dst,
                                        uint32_t dst_size);
 
+struct wireguard_result wireguard_try_read(const struct wireguard_tunnel *tunnel,
+                                           const uint8_t *src,
+                                           uint32_t src_size,
+                                           uint8_t *dst,
+                                           uint32_t dst_size);
+
 struct wireguard_result wireguard_tick(const struct wireguard_tunnel *tunnel,
                                        uint8_t *dst,
                                        uint32_t dst_size);

From 97e60dfa6a44d5a48bd65f07c8a0a0606d6242db Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Sun, 3 May 2026 17:33:39 -0700
Subject: [PATCH 05/25] Switch benchmark to the non-blocking FFI methods

---
 .../benches/ffi_benches/wg_bench_client.c     | 25 ++++++++++++-------
 .../benches/ffi_benches/wg_bench_client.h     |  1 +
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/boringtun/benches/ffi_benches/wg_bench_client.c b/boringtun/benches/ffi_benches/wg_bench_client.c
index 261387156..159828498 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.c
+++ b/boringtun/benches/ffi_benches/wg_bench_client.c
@@ -40,8 +40,8 @@ static void* wg_bench_background(void *arg) {
 
             case WRITE_TO_NETWORK:
                 if (send(client->fd, ciphertext, result.size, MSG_DONTWAIT) < 0) {
+                    atomic_fetch_add(&client->stats.tx_drops, 1);
                     if (errno == EAGAIN) break;
-                    fprintf(stderr, "worker tick write: %s\n", strerror(errno));
                 }
                 break;
 
@@ -92,7 +92,7 @@ static void* wg_bench_send_worker(void *arg) {
 
     // Generate a sample IPv4 packet header.
     memset(ip, 0, sizeof(struct wg_bench_iphdr));
-    ip->vlen = 0x45;
+    ip->vlen = 0x40 + (sizeof(struct wg_bench_iphdr) / 4);
     ip->ttl = 64;
     ip->protocol = IPPROTO_UDP;
     ip->saddr = htonl(src);
@@ -115,7 +115,7 @@ static void* wg_bench_send_worker(void *arg) {
         ip->cksum = 0;
 
         // Encrypt the packet.
-        result = wireguard_write(client->tunnel, plaintext, pktlen, ciphertext, sizeof(ciphertext));
+        result = wireguard_try_write(client->tunnel, plaintext, pktlen, ciphertext, sizeof(ciphertext));
         switch (result.op) {
             case WIREGUARD_DONE:
                 break;
@@ -131,7 +131,8 @@ static void* wg_bench_send_worker(void *arg) {
                 // Not expected in this case.
                 atomic_fetch_add(&client->stats.tx_packets, 1);
                 atomic_fetch_add(&client->stats.tx_bytes, pktlen);
-                if (send(client->fd, ciphertext, result.size, MSG_DONTWAIT) < 0) {
+                if (send(client->fd, ciphertext, result.size, 0) < 0) {
+                    atomic_fetch_add(&client->stats.tx_drops, 1);
                     if (errno == EAGAIN) continue;
                     if (!client->worker_shutdown) {
                         fprintf(stderr, "worker tx error: %s\n", strerror(errno));
@@ -157,7 +158,6 @@ static void* wg_bench_send_worker(void *arg) {
 
 static void* wg_bench_recv_worker(void *arg) {
     struct wg_bench_client *client = (struct wg_bench_client *)arg;
-    struct wireguard_result result;
     uint8_t ciphertext[WG_BENCH_MTU + 32];
     uint8_t plaintext[WG_BENCH_MTU];
 
@@ -177,8 +177,14 @@ static void* wg_bench_recv_worker(void *arg) {
             return NULL;
         }
 
-        // Decrypt the packet.
-        result = wireguard_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
+        struct wireguard_result result;
+        if (ciphertext[0] == 0x04) {
+            // Fast path - decrypt data packets without locking.
+            result = wireguard_try_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
+        } else {
+            // Slow path - handle handshake and state changes while locking.
+            result = wireguard_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
+        }
         switch (result.op) {
             case WIREGUARD_DONE:
                 continue;
@@ -191,7 +197,7 @@ static void* wg_bench_recv_worker(void *arg) {
 
             case WRITE_TO_NETWORK:
                 if (send(client->fd, plaintext, result.size, MSG_DONTWAIT) < 0) {
-                    //fprintf(stderr, "worker reply error: %s\n", strerror(errno));
+                    atomic_fetch_add(&client->stats.tx_drops, 1);
                 }
                 continue;
 
@@ -246,7 +252,7 @@ void wg_bench_start_handshake(struct wg_bench_client* client) {
 
         case WRITE_TO_NETWORK:
             if (send(client->fd, handshake, result.size, MSG_DONTWAIT) < 0) {
-                //fprintf(stderr, "worker reply error: %s\n", strerror(errno));
+                atomic_fetch_add(&client->stats.tx_drops, 1);
             }
             break;
 
@@ -283,6 +289,7 @@ void wg_bench_start_send(struct wg_bench_client* client) {
 
 void wg_bench_fetch_stats(const struct wg_bench_client* client, struct wg_bench_statistics* st) {
     atomic_fetch_add(&st->tx_packets, atomic_load(&client->stats.tx_packets));
+    atomic_fetch_add(&st->tx_drops, atomic_load(&client->stats.tx_drops));
     atomic_fetch_add(&st->tx_bytes, atomic_load(&client->stats.tx_bytes));
     atomic_fetch_add(&st->rx_packets, atomic_load(&client->stats.rx_packets));
     atomic_fetch_add(&st->rx_bytes, atomic_load(&client->stats.rx_bytes));
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.h b/boringtun/benches/ffi_benches/wg_bench_client.h
index b7ba8f122..6282600a5 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.h
+++ b/boringtun/benches/ffi_benches/wg_bench_client.h
@@ -9,6 +9,7 @@
 
 struct wg_bench_statistics {
     atomic_uintmax_t tx_packets;
+    atomic_uintmax_t tx_drops;
     atomic_uintmax_t tx_bytes;
     atomic_uintmax_t rx_packets;
     atomic_uintmax_t rx_bytes;

From 47bfb78e85c9d17f6c4577d2b2dfef85355c52aa Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Mon, 4 May 2026 11:27:15 -0700
Subject: [PATCH 06/25] Fix the build on macOS

---
 boringtun/benches/ffi_benches/Makefile          | 9 +++++++--
 boringtun/benches/ffi_benches/main.c            | 6 +++---
 boringtun/benches/ffi_benches/wg_bench_client.c | 2 +-
 boringtun/benches/ffi_benches/wg_bench_client.h | 3 ++-
 4 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/boringtun/benches/ffi_benches/Makefile b/boringtun/benches/ffi_benches/Makefile
index c271efa7e..f9def5104 100644
--- a/boringtun/benches/ffi_benches/Makefile
+++ b/boringtun/benches/ffi_benches/Makefile
@@ -1,11 +1,16 @@
 SRCDIR := $(dir $(lastword ${MAKEFILE_LIST}))
 PKGDIR := $(realpath ${SRCDIR}/../..)
 OBJDIR := $(shell pwd)
+HOST_OS := $(shell uname)
+
+ifeq (${HOST_OS},Darwin)
+MACOSX_MAJOR_VERSION := $(shell sw_vers --productVersion | cut -d. -f1)
+export MACOSX_DEPLOYMENT_TARGET=${MACOSX_MAJOR_VERSION}.0
+endif
 
 release/libboringtun.a: ${PKGDIR}/Cargo.toml
 	cd ${PKGDIR} && cargo build --lib --release --target-dir ${OBJDIR} --features ffi-bindings
-	echo -n "$@:" > release/libboringtun-fixup.d
-	cat release/libboringtun.d | cut -d: -f2- >> release/libboringtun-fixup.d
+	@echo "$@: $$(cut -d: -f2- release/libboringtun.d)" > release/libboringtun-fixup.d
 
 -include release/libboringtun-fixup.d
 
diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
index 9cd8662aa..06229cf4a 100644
--- a/boringtun/benches/ffi_benches/main.c
+++ b/boringtun/benches/ffi_benches/main.c
@@ -9,7 +9,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <asm/termbits.h>  /* Definition of TIOC*WINSZ constants */
 #include <sys/ioctl.h>
 #include <sys/socket.h>
 #include <sys/time.h>
@@ -108,7 +107,7 @@ static void print_stats(const struct wg_bench_statistics* st, double walltime, d
     // Prepare the status to write.
     char linebuf[120];
     int len = snprintf(linebuf, sizeof(linebuf),
-                       "\r   tx:%-12lu rx:%-12lu drops:%-8lu err:%-8lu load:%08s  %s transferred (%s/s)",
+                       "\r   tx:%-12lu rx:%-12lu drops:%-8lu err:%-8lu load:%8s  %s transferred (%s/s)",
                        atomic_load(&st->tx_packets), atomic_load(&st->rx_packets), atomic_load(&st->tx_drops),
                        total_errors, loadbuf, print_bytes(total_bytes, xfer, sizeof(xfer)),
                        print_bytes(total_bytes / walltime, tpbuf, sizeof(tpbuf)));
@@ -168,7 +167,7 @@ static void print_errors(const struct wg_bench_statistics* st) {
 }
 
 static void print_usage(FILE* fp, const char* name) {
-    fprintf(fp, "Usage: %s [OPTIONS]\n");
+    fprintf(fp, "Usage: %s [OPTIONS]\n", name);
     fprintf(fp, "Run FFI benchmarks for the boringtun library.\n");
     fprintf(fp, "\n");
     fprintf(fp, "Options:\n");
@@ -275,6 +274,7 @@ int main(int argc, char* argv[]) {
         usleep(100000);
     }
 
+    memset(&stats, 0, sizeof(stats));
     clock_gettime(CLOCK_MONOTONIC, &now);
     clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &cpu);
     wg_bench_fetch_stats(a, &stats);
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.c b/boringtun/benches/ffi_benches/wg_bench_client.c
index 159828498..6899c8b72 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.c
+++ b/boringtun/benches/ffi_benches/wg_bench_client.c
@@ -128,12 +128,12 @@ static void* wg_bench_send_worker(void *arg) {
                 break;
 
             case WRITE_TO_NETWORK:
-                // Not expected in this case.
                 atomic_fetch_add(&client->stats.tx_packets, 1);
                 atomic_fetch_add(&client->stats.tx_bytes, pktlen);
                 if (send(client->fd, ciphertext, result.size, 0) < 0) {
                     atomic_fetch_add(&client->stats.tx_drops, 1);
                     if (errno == EAGAIN) continue;
+                    if (errno == ENOBUFS) continue;
                     if (!client->worker_shutdown) {
                         fprintf(stderr, "worker tx error: %s\n", strerror(errno));
                     }
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.h b/boringtun/benches/ffi_benches/wg_bench_client.h
index 6282600a5..9227a7418 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.h
+++ b/boringtun/benches/ffi_benches/wg_bench_client.h
@@ -22,8 +22,9 @@ struct wg_bench_client {
     const char *pubkey;
     int fd;
 
+    // The packet send and receive worker pool.
     int       worker_shutdown;
-    pthread_t worker_count;
+    int       worker_count;
     pthread_t workers[WG_BENCH_MAX_THREADS];
 
     // Statistics.

From 468886874d396110a69822833729b5f4fccbc453 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Mon, 4 May 2026 13:02:19 -0700
Subject: [PATCH 07/25] Use named sockets and move creation/ownership into
 wg_bench_client

---
 boringtun/benches/ffi_benches/main.c          |  13 +-
 .../benches/ffi_benches/wg_bench_client.c     | 140 ++++++++++++------
 .../benches/ffi_benches/wg_bench_client.h     |  10 +-
 3 files changed, 112 insertions(+), 51 deletions(-)

diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
index 06229cf4a..817a4a734 100644
--- a/boringtun/benches/ffi_benches/main.c
+++ b/boringtun/benches/ffi_benches/main.c
@@ -60,6 +60,10 @@ static void handle_signal(int sig) {
             wg_printf("benchmark terminated\n");
             caught_sigint = 1;
             break;
+        
+        case SIGHUP:
+            // Do nothing.
+            break;
     }
 }
 
@@ -220,14 +224,11 @@ int main(int argc, char* argv[]) {
     };
     sigaction(SIGINT, &action, NULL);
     sigaction(SIGTERM, &action, NULL);
-
-    // Create a socket pair for the two clients to communicate over.
-    int sv[2];
-    socketpair(AF_UNIX, SOCK_DGRAM, 0, sv);
+    sigaction(SIGHUP, &action, NULL);
 
     // Create two benchmark clients.
-    struct wg_bench_client* a = wg_bench_create(sv[0]);
-    struct wg_bench_client* b = wg_bench_create(sv[1]);
+    struct wg_bench_client* a = wg_bench_create();
+    struct wg_bench_client* b = wg_bench_create();
 
     // Connect the two clients.
     wg_bench_connect(a, b->pubkey);
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.c b/boringtun/benches/ffi_benches/wg_bench_client.c
index 6899c8b72..d61caf2a2 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.c
+++ b/boringtun/benches/ffi_benches/wg_bench_client.c
@@ -7,6 +7,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/socket.h>
+#include <sys/un.h>
 #include <unistd.h>
 
 #define WG_BENCH_MTU 2048
@@ -17,6 +18,10 @@ static void wg_worker_sigmask() {
     sigaddset(&sigset, SIGINT);
     sigaddset(&sigset, SIGTERM);
     pthread_sigmask(SIG_BLOCK, &sigset, NULL);
+
+    sigemptyset(&sigset);
+    sigaddset(&sigset, SIGHUP);
+    pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 }
 
 static void* wg_bench_background(void *arg) {
@@ -24,13 +29,31 @@ static void* wg_bench_background(void *arg) {
     struct wireguard_result result;
     uint8_t ciphertext[WG_BENCH_MTU + 32];
 
+    // Create a separate socket for this thread.
+    int fd = socket(AF_UNIX, SOCK_DGRAM, 0);
+    if (fd < 0) {
+        fprintf(stderr, "background socket error: %s\n", strerror(errno));
+        return NULL;
+    }
+    if (connect(fd, (const struct sockaddr*)&client->peer, sizeof(client->peer)) < 0) {
+        fprintf(stderr, "background connect error: %s\n", strerror(errno));
+        close(fd);
+        return NULL;
+    }
+
     wg_worker_sigmask();
     while (true) {
+        if (client->worker_handshake) {
+            client->worker_handshake = false;
+            result = wireguard_force_handshake(client->tunnel, ciphertext, sizeof(ciphertext));
+        } else {
+            result = wireguard_tick(client->tunnel, ciphertext, sizeof(ciphertext));
+        }
+
         // Process timeouts and state updates.
-        result = wireguard_tick(client->tunnel, ciphertext, sizeof(ciphertext));
         switch (result.op) {
             case WIREGUARD_DONE:
-                // Sleep for a tick before trying again.
+                // Sleep for 100ms before trying again.
                 usleep(100000);
                 break;
 
@@ -39,9 +62,10 @@ static void* wg_bench_background(void *arg) {
                 break;
 
             case WRITE_TO_NETWORK:
-                if (send(client->fd, ciphertext, result.size, MSG_DONTWAIT) < 0) {
+                if (send(fd, ciphertext, result.size, MSG_DONTWAIT) < 0) {
                     atomic_fetch_add(&client->stats.tx_drops, 1);
                     if (errno == EAGAIN) break;
+                    fprintf(stderr, "worker tick send: %s\n", strerror(errno));
                 }
                 break;
 
@@ -101,10 +125,19 @@ static void* wg_bench_send_worker(void *arg) {
     udp->dport = 0x5678;
     udp->cksum = 0;
 
-    while (true) {
-        // Check if we are shutting down.
-        pthread_testcancel();
+    // Create a separate socket for the send workflows.
+    int fd = socket(AF_UNIX, SOCK_DGRAM, 0);
+    if (fd < 0) {
+        fprintf(stderr, "worker socket error: %s\n", strerror(errno));
+        return NULL;
+    }
+    if (connect(fd, (const struct sockaddr*)&client->peer, sizeof(client->peer)) < 0) {
+        fprintf(stderr, "worker connect error: %s\n", strerror(errno));
+        close(fd);
+        return NULL;
+    }
 
+    while (!client->worker_shutdown) {
         // Pick a random datagram size between 512-1024 and fill with a random byte.
         int dsize = 512 + (rand() % 512);
         int pktlen = sizeof(struct wg_bench_iphdr) + sizeof(struct wg_bench_udphdr) + dsize;
@@ -115,7 +148,8 @@ static void* wg_bench_send_worker(void *arg) {
         ip->cksum = 0;
 
         // Encrypt the packet.
-        result = wireguard_try_write(client->tunnel, plaintext, pktlen, ciphertext, sizeof(ciphertext));
+        result = wireguard_try_write(client->tunnel, plaintext, pktlen,
+                                     ciphertext, sizeof(ciphertext));
         switch (result.op) {
             case WIREGUARD_DONE:
                 break;
@@ -130,13 +164,14 @@ static void* wg_bench_send_worker(void *arg) {
             case WRITE_TO_NETWORK:
                 atomic_fetch_add(&client->stats.tx_packets, 1);
                 atomic_fetch_add(&client->stats.tx_bytes, pktlen);
-                if (send(client->fd, ciphertext, result.size, 0) < 0) {
+                if (send(fd, ciphertext, result.size, 0) < 0) {
                     atomic_fetch_add(&client->stats.tx_drops, 1);
                     if (errno == EAGAIN) continue;
                     if (errno == ENOBUFS) continue;
                     if (!client->worker_shutdown) {
                         fprintf(stderr, "worker tx error: %s\n", strerror(errno));
                     }
+                    close(fd);
                     return NULL;
                 }
                 break;
@@ -158,11 +193,13 @@ static void* wg_bench_send_worker(void *arg) {
 
 static void* wg_bench_recv_worker(void *arg) {
     struct wg_bench_client *client = (struct wg_bench_client *)arg;
+    const struct sockaddr* peer = (const struct sockaddr*)&client->peer;
+    socklen_t peerlen = sizeof(client->peer);
     uint8_t ciphertext[WG_BENCH_MTU + 32];
     uint8_t plaintext[WG_BENCH_MTU];
 
     wg_worker_sigmask();
-    while (true) {
+    while (!client->worker_shutdown) {
         // Read a packet.
         ssize_t rx = recv(client->fd, ciphertext, sizeof(ciphertext), MSG_DONTWAIT);
         if (rx == 0) {
@@ -196,7 +233,7 @@ static void* wg_bench_recv_worker(void *arg) {
                 continue;
 
             case WRITE_TO_NETWORK:
-                if (send(client->fd, plaintext, result.size, MSG_DONTWAIT) < 0) {
+                if (sendto(client->fd, plaintext, result.size, MSG_DONTWAIT, peer, peerlen) < 0) {
                     atomic_fetch_add(&client->stats.tx_drops, 1);
                 }
                 continue;
@@ -206,7 +243,7 @@ static void* wg_bench_recv_worker(void *arg) {
                 atomic_fetch_add(&client->stats.rx_packets, 1);
                 atomic_fetch_add(&client->stats.rx_bytes, result.size);
                 break;
-            
+
             default:
                 continue;
         }
@@ -215,14 +252,52 @@ static void* wg_bench_recv_worker(void *arg) {
     return NULL;
 }
 
-struct wg_bench_client* wg_bench_create(int fd) {
+// Fill a sockaddr_un with the named pipe for a given public key.
+static struct sockaddr* wg_bench_sockaddr(const char* pubkey, struct sockaddr_un *addr) {
+    memset(addr, 0, sizeof(struct sockaddr_un));
+    addr->sun_family = AF_UNIX;
+#ifdef SUN_LEN
+    addr->sun_len = sizeof(struct sockaddr_un);
+#endif
+
+    // Set the name to /tmp/ffi-bench-<base64key>.sock, using URL-safe encoding.
+    strcpy(addr->sun_path, "/tmp/ffi-bench-");
+    char *urlsafe = strchr(addr->sun_path, '\0');
+    while (*pubkey != '\0') {
+        char c = *pubkey++;
+        if (c == '+') *urlsafe++ = '-';
+        else if (c == '/') *urlsafe++ = '_';
+        else if (c != '=') *urlsafe++ = c;
+    }
+    strcat(urlsafe, ".sock");
+
+    return (struct sockaddr*)addr;
+}
+
+struct wg_bench_client* wg_bench_create() {
     struct wg_bench_client* client = calloc(sizeof(struct wg_bench_client), 1);
     if (!client) {
         return NULL;
     }
     client->secret = x25519_secret_key();
     client->pubkey = x25519_key_to_base64(x25519_public_key(client->secret));
-    client->fd = fd;
+
+    // Create a named datagram pipe for handling packets.
+    client->fd = socket(AF_UNIX, SOCK_DGRAM, 0);
+    if (client->fd < 0) {
+        x25519_key_to_str_free(client->pubkey);
+        free(client);
+        return NULL;
+    }
+
+    struct sockaddr* sa = wg_bench_sockaddr(client->pubkey, &client->addr);
+    if (bind(client->fd, sa, sizeof(client->addr)) < 0) {
+        x25519_key_to_str_free(client->pubkey);
+        close(client->fd);
+        free(client);
+        return NULL;
+    }
+
     return client;
 }
 
@@ -231,40 +306,15 @@ void wg_bench_connect(struct wg_bench_client* client, const char* pubkey) {
     client->tunnel = new_tunnel(statickey, pubkey, NULL, 5, rand() & 0xffffff);
     x25519_key_to_str_free(statickey);
 
+    wg_bench_sockaddr(pubkey, &client->peer);
+
     // Start the background worker to drive wireguard_tick();
-    client->worker_count = 1;
-    pthread_create(&client->workers[0], NULL, wg_bench_background, client);
+    pthread_create(&client->background, NULL, wg_bench_background, client);
 }
 
 void wg_bench_start_handshake(struct wg_bench_client* client) {
-    uint8_t handshake[154];
-    struct wireguard_result result;
-    result = wireguard_force_handshake(client->tunnel, handshake, sizeof(handshake));
-    switch (result.op) {
-        case WIREGUARD_DONE:
-            break;
-        
-        case WIREGUARD_ERROR:
-            if (result.size < WG_BENCH_MAX_ERRORS) {
-                atomic_fetch_add(&client->stats.errors[result.size], 1);
-            }
-            break;
-
-        case WRITE_TO_NETWORK:
-            if (send(client->fd, handshake, result.size, MSG_DONTWAIT) < 0) {
-                atomic_fetch_add(&client->stats.tx_drops, 1);
-            }
-            break;
-
-        case WRITE_TO_TUNNEL_IPV4:
-        case WRITE_TO_TUNNEL_IPV6:
-            atomic_fetch_add(&client->stats.rx_packets, 1);
-            atomic_fetch_add(&client->stats.rx_bytes, result.size);
-            break;
-        
-        default:
-            break;
-    }
+    client->worker_handshake = true;
+    pthread_kill(client->background, SIGHUP);
 }
 
 void wg_bench_start_recv(struct wg_bench_client* client) {
@@ -301,8 +351,12 @@ void wg_bench_fetch_stats(const struct wg_bench_client* client, struct wg_bench_
 void wg_bench_close(struct wg_bench_client* client) {
     // Signal that we are shutting down and terminate workers.
     client->worker_shutdown = 1;
+    pthread_cancel(client->background);
+    pthread_kill(client->background, SIGHUP);
+    pthread_join(client->background, NULL);
     for (int i = 0; i < client->worker_count; i++) {
         pthread_cancel(client->workers[i]);
+        pthread_kill(client->workers[i], SIGHUP);
         pthread_join(client->workers[i], NULL);
     }
 
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.h b/boringtun/benches/ffi_benches/wg_bench_client.h
index 9227a7418..73add4497 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.h
+++ b/boringtun/benches/ffi_benches/wg_bench_client.h
@@ -2,6 +2,7 @@
 
 #include <pthread.h>
 #include <stdatomic.h>
+#include <sys/un.h>
 #include "wireguard_ffi.h"
 
 #define WG_BENCH_MAX_THREADS 256
@@ -20,18 +21,23 @@ struct wg_bench_client {
     struct wireguard_tunnel *tunnel;
     struct x25519_key secret;
     const char *pubkey;
-    int fd;
+
+    int                 fd;
+    struct sockaddr_un  addr;
+    struct sockaddr_un  peer;
 
     // The packet send and receive worker pool.
+    int       worker_handshake;
     int       worker_shutdown;
     int       worker_count;
+    pthread_t background;
     pthread_t workers[WG_BENCH_MAX_THREADS];
 
     // Statistics.
     struct wg_bench_statistics stats;
 };
 
-struct wg_bench_client* wg_bench_create(int fd);
+struct wg_bench_client* wg_bench_create();
 void wg_bench_connect(struct wg_bench_client* client, const char* pubkey);
 void wg_bench_start_handshake(struct wg_bench_client* client);
 void wg_bench_start_send(struct wg_bench_client* client);

From 55a4edfcfd2d044ab4b918174a4ccc00b6d0cb57 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Mon, 4 May 2026 16:59:14 -0700
Subject: [PATCH 08/25] Improve profiling support

---
 boringtun/benches/ffi_benches/Makefile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/boringtun/benches/ffi_benches/Makefile b/boringtun/benches/ffi_benches/Makefile
index f9def5104..923db3432 100644
--- a/boringtun/benches/ffi_benches/Makefile
+++ b/boringtun/benches/ffi_benches/Makefile
@@ -18,11 +18,16 @@ BENCH_CFLAGS := -I ${PKGDIR}/src
 BENCH_SRCS := main.c wg_bench_client.c
 BENCH_OBJS := $(patsubst %.c,%.o,${BENCH_SRCS})
 
+# Profiling support
+BENCH_CFLAGS += -g -fno-omit-frame-pointer
+BENCH_LDFLAGS := -g -fno-omit-frame-pointer
+release/libboringtun.a: export RUSTFLAGS=-C force-frame-pointers=y
+
 %.o : %.c
 	${CC} ${CFLAGS} ${BENCH_CFLAGS} -c -o $@ $<
 
 ffi-bench: ${BENCH_OBJS} release/libboringtun.a
-	${CC} ${LDLAGS} -o $@ $^
+	${CC} ${LDLAGS} ${BENCH_LDFLAGS} -o $@ $^
 
 clean:
 	rm -f ${OBJDIR}/*.o

From 10e4b608ad9ebffdfb5573cdbb92a4ebdf32a62a Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Mon, 4 May 2026 17:00:35 -0700
Subject: [PATCH 09/25] Try using kqueue/epoll

---
 boringtun/benches/ffi_benches/main.c          |  21 +-
 .../benches/ffi_benches/wg_bench_client.c     | 299 ++++++++++++------
 .../benches/ffi_benches/wg_bench_client.h     |   3 +-
 3 files changed, 202 insertions(+), 121 deletions(-)

diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
index 817a4a734..7ad7da42f 100644
--- a/boringtun/benches/ffi_benches/main.c
+++ b/boringtun/benches/ffi_benches/main.c
@@ -111,26 +111,17 @@ static void print_stats(const struct wg_bench_statistics* st, double walltime, d
     // Prepare the status to write.
     char linebuf[120];
     int len = snprintf(linebuf, sizeof(linebuf),
-                       "\r   tx:%-12lu rx:%-12lu drops:%-8lu err:%-8lu load:%8s  %s transferred (%s/s)",
+                       "   tx:%-12lu rx:%-12lu drops:%-8lu err:%-8lu load:%8s  %s transferred (%s/s)",
                        atomic_load(&st->tx_packets), atomic_load(&st->rx_packets), atomic_load(&st->tx_drops),
                        total_errors, loadbuf, print_bytes(total_bytes, xfer, sizeof(xfer)),
                        print_bytes(total_bytes / walltime, tpbuf, sizeof(tpbuf)));
-
     struct winsize ws;
     if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) == 0) {
-        // This is a terminal.
-        if (ws.ws_col < sizeof(linebuf)) {
-            len = ws.ws_col - 1;
-        } else {
-            memset(linebuf + len, ' ', sizeof(linebuf) - len);
-            len = sizeof(linebuf);
-        }
-        write(STDOUT_FILENO, linebuf, len);
+        // If we are on an interactive terminal, refresh the status line
+        dprintf(STDOUT_FILENO, "\r%s%*s\r", linebuf, ws.ws_col - len - 1, "");
     } else {
         // Some other file.
-        linebuf[len] = '\n';
-        linebuf[len+1] = '\0';
-        puts(linebuf+1);
+        printf("%s\n", linebuf);
     }
 }
 
@@ -245,8 +236,8 @@ int main(int argc, char* argv[]) {
     // Launch workers.
     wg_bench_start_handshake(a);
     for (int i = 0; i < num_workers; i++) {
-        wg_bench_start_recv(a);
-        wg_bench_start_recv(b);
+        wg_bench_start_worker(a);
+        wg_bench_start_worker(b);
         wg_bench_start_send(a);
         wg_bench_start_send(b);
     }
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.c b/boringtun/benches/ffi_benches/wg_bench_client.c
index d61caf2a2..ef4c28431 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.c
+++ b/boringtun/benches/ffi_benches/wg_bench_client.c
@@ -2,6 +2,7 @@
 
 #include <arpa/inet.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -10,6 +11,13 @@
 #include <sys/un.h>
 #include <unistd.h>
 
+#ifdef __linux
+#include <sys/epoll.h>
+#include <sys/timerfd.h>
+#else
+#include <sys/event.h>
+#endif
+
 #define WG_BENCH_MTU 2048
 
 static void wg_worker_sigmask() {
@@ -24,64 +32,6 @@ static void wg_worker_sigmask() {
     pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 }
 
-static void* wg_bench_background(void *arg) {
-    struct wg_bench_client *client = (struct wg_bench_client *)arg;
-    struct wireguard_result result;
-    uint8_t ciphertext[WG_BENCH_MTU + 32];
-
-    // Create a separate socket for this thread.
-    int fd = socket(AF_UNIX, SOCK_DGRAM, 0);
-    if (fd < 0) {
-        fprintf(stderr, "background socket error: %s\n", strerror(errno));
-        return NULL;
-    }
-    if (connect(fd, (const struct sockaddr*)&client->peer, sizeof(client->peer)) < 0) {
-        fprintf(stderr, "background connect error: %s\n", strerror(errno));
-        close(fd);
-        return NULL;
-    }
-
-    wg_worker_sigmask();
-    while (true) {
-        if (client->worker_handshake) {
-            client->worker_handshake = false;
-            result = wireguard_force_handshake(client->tunnel, ciphertext, sizeof(ciphertext));
-        } else {
-            result = wireguard_tick(client->tunnel, ciphertext, sizeof(ciphertext));
-        }
-
-        // Process timeouts and state updates.
-        switch (result.op) {
-            case WIREGUARD_DONE:
-                // Sleep for 100ms before trying again.
-                usleep(100000);
-                break;
-
-            case WIREGUARD_ERROR:
-                fprintf(stderr, "worker tick error: %zu\n", result.size);
-                break;
-
-            case WRITE_TO_NETWORK:
-                if (send(fd, ciphertext, result.size, MSG_DONTWAIT) < 0) {
-                    atomic_fetch_add(&client->stats.tx_drops, 1);
-                    if (errno == EAGAIN) break;
-                    fprintf(stderr, "worker tick send: %s\n", strerror(errno));
-                }
-                break;
-
-            case WRITE_TO_TUNNEL_IPV4:
-            case WRITE_TO_TUNNEL_IPV6:
-                // not expected
-                fprintf(stderr, "worker tick tunnel");
-                break;
-            
-            default:
-                fprintf(stderr, "worker tick unknown: %d\n", result.op);
-                break;
-        }
-    }
-}
-
 struct wg_bench_iphdr {
     uint8_t vlen;
     uint8_t qos;
@@ -191,63 +141,145 @@ static void* wg_bench_send_worker(void *arg) {
     return NULL;
 }
 
-static void* wg_bench_recv_worker(void *arg) {
-    struct wg_bench_client *client = (struct wg_bench_client *)arg;
+static void wg_bench_tick(struct wg_bench_client *client) {
     const struct sockaddr* peer = (const struct sockaddr*)&client->peer;
     socklen_t peerlen = sizeof(client->peer);
-    uint8_t ciphertext[WG_BENCH_MTU + 32];
-    uint8_t plaintext[WG_BENCH_MTU];
-
-    wg_worker_sigmask();
-    while (!client->worker_shutdown) {
-        // Read a packet.
-        ssize_t rx = recv(client->fd, ciphertext, sizeof(ciphertext), MSG_DONTWAIT);
-        if (rx == 0) {
-            fprintf(stderr, "worker shutdown\n");
-            return NULL;
-        }
-        if (rx < 0) {
-            if (errno == EAGAIN) continue;
-            if (!client->worker_shutdown) {
-                fprintf(stderr, "worker rx error: %s\n", strerror(errno));
-            }
-            return NULL;
-        }
 
+    while (true) {
+        uint8_t ciphertext[WG_BENCH_MTU + 32];
         struct wireguard_result result;
-        if (ciphertext[0] == 0x04) {
-            // Fast path - decrypt data packets without locking.
-            result = wireguard_try_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
-        } else {
-            // Slow path - handle handshake and state changes while locking.
-            result = wireguard_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
-        }
+        result = wireguard_tick(client->tunnel, ciphertext, sizeof(ciphertext));
+
+        // Process timeouts and state updates.
         switch (result.op) {
             case WIREGUARD_DONE:
-                continue;
-            
+                return;
+
             case WIREGUARD_ERROR:
-                if (result.size < WG_BENCH_MAX_ERRORS) {
-                    atomic_fetch_add(&client->stats.errors[result.size], 1);
-                }
-                continue;
+                fprintf(stderr, "worker tick error: %zu\n", result.size);
+                return;
 
             case WRITE_TO_NETWORK:
-                if (sendto(client->fd, plaintext, result.size, MSG_DONTWAIT, peer, peerlen) < 0) {
+                if (sendto(client->fd, ciphertext, result.size, MSG_DONTWAIT, peer, sizeof(client->peer)) < 0) {
                     atomic_fetch_add(&client->stats.tx_drops, 1);
                 }
                 continue;
 
             case WRITE_TO_TUNNEL_IPV4:
             case WRITE_TO_TUNNEL_IPV6:
-                atomic_fetch_add(&client->stats.rx_packets, 1);
-                atomic_fetch_add(&client->stats.rx_bytes, result.size);
+                // not expected
+                fprintf(stderr, "worker tick tunnel");
                 break;
-
+            
             default:
-                continue;
+                fprintf(stderr, "worker tick unknown: %d\n", result.op);
+                return;
         }
     }
+}
+
+static void wg_bench_recv(struct wg_bench_client *client) {
+    uint8_t ciphertext[WG_BENCH_MTU + 32];
+    uint8_t plaintext[WG_BENCH_MTU];
+    struct wireguard_result result;
+
+    const struct sockaddr* peer = (const struct sockaddr*)&client->peer;
+    socklen_t peerlen = sizeof(client->peer);
+
+    // Read a packet.
+    ssize_t rx = recv(client->fd, ciphertext, sizeof(ciphertext), 0);
+    if (rx == 0) {
+        fprintf(stderr, "worker shutdown\n");
+        return;
+    }
+    if (rx < 0) {
+        if (!client->worker_shutdown) {
+            //fprintf(stderr, "worker rx error: %s\n", strerror(errno));
+        }
+        return;
+    }
+
+    if (ciphertext[0] == 0x04) {
+        // Fast path - decrypt data packets without locking.
+        result = wireguard_try_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
+    } else {
+        // Slow path - handle handshake and state changes while locking.
+        result = wireguard_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
+    }
+
+    switch (result.op) {
+        case WIREGUARD_DONE:
+            break;
+        
+        case WIREGUARD_ERROR:
+            if (result.size < WG_BENCH_MAX_ERRORS) {
+                atomic_fetch_add(&client->stats.errors[result.size], 1);
+            }
+            break;
+
+        case WRITE_TO_NETWORK:
+            // This is not expected, but I guess it's possible
+            if (sendto(client->fd, plaintext, result.size, MSG_DONTWAIT, peer, peerlen) < 0) {
+                atomic_fetch_add(&client->stats.tx_drops, 1);
+            }
+            break;
+
+        case WRITE_TO_TUNNEL_IPV4:
+        case WRITE_TO_TUNNEL_IPV6:
+            atomic_fetch_add(&client->stats.rx_packets, 1);
+            atomic_fetch_add(&client->stats.rx_bytes, result.size);
+            break;
+
+        default:
+            break;
+    }
+}
+
+static void* wg_bench_worker(void *arg) {
+    struct wg_bench_client *client = (struct wg_bench_client *)arg;
+
+    wg_worker_sigmask();
+    while (!client->worker_shutdown) {
+#ifdef __linux
+        // Wait for an event to handle.
+        struct epoll_event ev[16];
+        int maxev = sizeof(ev)/sizeof(struct epoll_event);
+        int nev = epoll_wait(client->queue, ev, maxev, -1);
+        if (nfds == -1) {
+            if (errno == EINTR) continue;
+            fprintf(stderr, "worker epoll error: %s\n", strerror(errno));
+            continue;
+        }
+    
+        // Handle events.
+        for (int i = 0; i < nev; i++) {
+            if (ev[i].data.fd < 0) {
+                wg_bench_tick(client);
+            } else if (ev[i].data.fd == client->fd) {
+                wg_bench_recv(client);
+            }
+        }
+#else
+        // Wait for an event to handle.
+        struct kevent kev[16];
+        int maxev = sizeof(kev)/sizeof(struct kevent);
+        int nev = kevent(client->queue, NULL, 0, kev, maxev, NULL);
+        if (nev < 0) {
+            if (errno == EINTR) continue;
+            fprintf(stderr, "worker kevent error: %s\n", strerror(errno));
+            continue;
+        }
+
+        // Handle events.
+        for (int i = 0; i < nev; i++) {
+            if (kev[i].filter == EVFILT_TIMER) {
+                wg_bench_tick(client);
+            } else if (kev[i].filter == EVFILT_READ) {
+                wg_bench_recv(client);
+            }
+        }
+#endif
+    }
 
     return NULL;
 }
@@ -298,6 +330,12 @@ struct wg_bench_client* wg_bench_create() {
         return NULL;
     }
 
+    fcntl(client->fd, F_SETFL, fcntl(client->fd, F_GETFL) | O_NONBLOCK);
+#ifdef __linux
+    client->queue = epoll_create1(0);
+#else
+    client->queue = kqueue();
+#endif
     return client;
 }
 
@@ -308,20 +346,72 @@ void wg_bench_connect(struct wg_bench_client* client, const char* pubkey) {
 
     wg_bench_sockaddr(pubkey, &client->peer);
 
-    // Start the background worker to drive wireguard_tick();
-    pthread_create(&client->background, NULL, wg_bench_background, client);
+    // Begin packet processing.
+#ifdef __linux
+    struct epoll_event ev;
+    ev.events = EPOLLIN;
+    ev.data.fd = client->fd;
+    epoll_ctl(client->queue, EPOLL_CTL_ADD, client->fd, &ev);
+
+    int timer = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK);
+    ev.events = EPOLLIN;
+    ev.data.fd = -1;
+    epoll_ctl(client->queue, EPOLL_CTL_ADD, timer, &ev);
+
+    struct itimerspec itspec;
+    itspec.it_value.tv_sec = 0;
+    itspec.it_value.tv_nsec = 100000000;
+    itspec.it_interval.tv_sec = 0;
+    itspec.it_interval.tv_nsec = 100000000;
+    timerfd_settime(timer, 0, &itspec, NULL);
+#else
+    struct kevent kev[2];
+    EV_SET(&kev[0], client->fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
+    EV_SET(&kev[1], 1, EVFILT_TIMER, EV_ADD, 0, 100, NULL);
+    kevent(client->queue, kev, 2, NULL, 0, NULL);
+#endif
 }
 
 void wg_bench_start_handshake(struct wg_bench_client* client) {
-    client->worker_handshake = true;
-    pthread_kill(client->background, SIGHUP);
+    uint8_t ciphertext[WG_BENCH_MTU + 32];
+    struct wireguard_result result;
+    result = wireguard_force_handshake(client->tunnel, ciphertext, sizeof(ciphertext));
+
+    const struct sockaddr* peer = (const struct sockaddr*)&client->peer;
+    socklen_t peerlen = sizeof(client->peer);
+
+    // Process timeouts and state updates.
+    switch (result.op) {
+        case WIREGUARD_DONE:
+            break;
+
+        case WIREGUARD_ERROR:
+            fprintf(stderr, "worker handshake error: %zu\n", result.size);
+            break;
+
+        case WRITE_TO_NETWORK:
+            if (sendto(client->fd, ciphertext, result.size, MSG_DONTWAIT, peer, sizeof(client->peer)) < 0) {
+                atomic_fetch_add(&client->stats.tx_drops, 1);
+            }
+            break;
+
+        case WRITE_TO_TUNNEL_IPV4:
+        case WRITE_TO_TUNNEL_IPV6:
+            // not expected
+            fprintf(stderr, "worker handshake tunnel");
+            break;
+        
+        default:
+            fprintf(stderr, "worker handshake unknown: %d\n", result.op);
+            return;
+    }
 }
 
-void wg_bench_start_recv(struct wg_bench_client* client) {
+void wg_bench_start_worker(struct wg_bench_client* client) {
     if (client->worker_count >= WG_BENCH_MAX_THREADS) {
         return;
     }
-    if (pthread_create(&client->workers[client->worker_count], NULL, wg_bench_recv_worker, client) != 0) {
+    if (pthread_create(&client->workers[client->worker_count], NULL, wg_bench_worker, client) != 0) {
         return;
     }
     client->worker_count++;
@@ -351,16 +441,15 @@ void wg_bench_fetch_stats(const struct wg_bench_client* client, struct wg_bench_
 void wg_bench_close(struct wg_bench_client* client) {
     // Signal that we are shutting down and terminate workers.
     client->worker_shutdown = 1;
-    pthread_cancel(client->background);
-    pthread_kill(client->background, SIGHUP);
-    pthread_join(client->background, NULL);
     for (int i = 0; i < client->worker_count; i++) {
         pthread_cancel(client->workers[i]);
         pthread_kill(client->workers[i], SIGHUP);
         pthread_join(client->workers[i], NULL);
     }
+    close(client->queue);
 
     shutdown(client->fd, SHUT_RDWR);
+    unlink(client->addr.sun_path);
     close(client->fd);
 
     x25519_key_to_str_free(client->pubkey);
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.h b/boringtun/benches/ffi_benches/wg_bench_client.h
index 73add4497..addcd4ace 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.h
+++ b/boringtun/benches/ffi_benches/wg_bench_client.h
@@ -23,6 +23,7 @@ struct wg_bench_client {
     const char *pubkey;
 
     int                 fd;
+    int                 queue;
     struct sockaddr_un  addr;
     struct sockaddr_un  peer;
 
@@ -41,6 +42,6 @@ struct wg_bench_client* wg_bench_create();
 void wg_bench_connect(struct wg_bench_client* client, const char* pubkey);
 void wg_bench_start_handshake(struct wg_bench_client* client);
 void wg_bench_start_send(struct wg_bench_client* client);
-void wg_bench_start_recv(struct wg_bench_client* client);
+void wg_bench_start_worker(struct wg_bench_client* client);
 void wg_bench_fetch_stats(const struct wg_bench_client* client, struct wg_bench_statistics* stats);
 void wg_bench_close(struct wg_bench_client* client);

From e096aa12969ce0195d2c9cffb077c7d3ea4fd558 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Tue, 5 May 2026 10:23:41 -0700
Subject: [PATCH 10/25] Unify wireguard_read and wireguard_try_read

---
 .../benches/ffi_benches/wg_bench_client.c     |  9 +------
 boringtun/src/ffi/mod.rs                      | 27 +++++++------------
 boringtun/src/noise/mod.rs                    |  8 +++---
 boringtun/src/wireguard_ffi.h                 |  6 -----
 4 files changed, 14 insertions(+), 36 deletions(-)

diff --git a/boringtun/benches/ffi_benches/wg_bench_client.c b/boringtun/benches/ffi_benches/wg_bench_client.c
index ef4c28431..54aff66ec 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.c
+++ b/boringtun/benches/ffi_benches/wg_bench_client.c
@@ -199,14 +199,7 @@ static void wg_bench_recv(struct wg_bench_client *client) {
         return;
     }
 
-    if (ciphertext[0] == 0x04) {
-        // Fast path - decrypt data packets without locking.
-        result = wireguard_try_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
-    } else {
-        // Slow path - handle handshake and state changes while locking.
-        result = wireguard_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
-    }
-
+    result = wireguard_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
     switch (result.op) {
         case WIREGUARD_DONE:
             break;
diff --git a/boringtun/src/ffi/mod.rs b/boringtun/src/ffi/mod.rs
index 9590e8173..6a38681fe 100644
--- a/boringtun/src/ffi/mod.rs
+++ b/boringtun/src/ffi/mod.rs
@@ -361,28 +361,19 @@ pub unsafe extern "C" fn wireguard_read(
     dst: *mut u8,
     dst_size: u32,
 ) -> wireguard_result {
-    let mut tunnel = tunnel.as_ref().unwrap().write();
     // Slices are not owned, and therefore will not be freed by Rust
     let src = slice::from_raw_parts(src, src_size as usize);
     let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
-    wireguard_result::from(tunnel.decapsulate(None, src, dst))
-}
+    let packet_type = src.first_chunk::<4>().map(|x| u32::from_le_bytes(*x));
 
-/// Read a UDP packet from the server.
-/// For more details check noise::network_to_tunnel functions.
-#[no_mangle]
-pub unsafe extern "C" fn wireguard_try_read(
-    tunnel: *const RwLock<Tunn>,
-    src: *const u8,
-    src_size: u32,
-    dst: *mut u8,
-    dst_size: u32,
-) -> wireguard_result {
-    let tunnel = tunnel.as_ref().unwrap().read();
-    // Slices are not owned, and therefore will not be freed by Rust
-    let src = slice::from_raw_parts(src, src_size as usize);
-    let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
-    wireguard_result::from(tunnel.try_decapsulate(src, dst))
+    // Data packets can be handled while holding a read lock.
+    if packet_type == Some(super::noise::DATA) {
+        let rotunnel = tunnel.as_ref().unwrap().read();
+        return wireguard_result::from(rotunnel.try_decapsulate(src, dst));
+    }
+
+    let mut tunnel = tunnel.as_ref().unwrap().write();
+    wireguard_result::from(tunnel.decapsulate(None, src, dst))
 }
 
 /// This is a state keeping function, that need to be called periodically.
diff --git a/boringtun/src/noise/mod.rs b/boringtun/src/noise/mod.rs
index 114bb2977..9e175380c 100644
--- a/boringtun/src/noise/mod.rs
+++ b/boringtun/src/noise/mod.rs
@@ -75,10 +75,10 @@ pub struct Tunn {
 }
 
 type MessageType = u32;
-const HANDSHAKE_INIT: MessageType = 1;
-const HANDSHAKE_RESP: MessageType = 2;
-const COOKIE_REPLY: MessageType = 3;
-const DATA: MessageType = 4;
+pub const HANDSHAKE_INIT: MessageType = 1;
+pub const HANDSHAKE_RESP: MessageType = 2;
+pub const COOKIE_REPLY: MessageType = 3;
+pub const DATA: MessageType = 4;
 
 const HANDSHAKE_INIT_SZ: usize = 148;
 const HANDSHAKE_RESP_SZ: usize = 92;
diff --git a/boringtun/src/wireguard_ffi.h b/boringtun/src/wireguard_ffi.h
index 724d8fce6..71bed40a3 100644
--- a/boringtun/src/wireguard_ffi.h
+++ b/boringtun/src/wireguard_ffi.h
@@ -101,12 +101,6 @@ struct wireguard_result wireguard_read(const struct wireguard_tunnel *tunnel,
                                        uint8_t *dst,
                                        uint32_t dst_size);
 
-struct wireguard_result wireguard_try_read(const struct wireguard_tunnel *tunnel,
-                                           const uint8_t *src,
-                                           uint32_t src_size,
-                                           uint8_t *dst,
-                                           uint32_t dst_size);
-
 struct wireguard_result wireguard_tick(const struct wireguard_tunnel *tunnel,
                                        uint8_t *dst,
                                        uint32_t dst_size);

From 09f3bed76b3e52775452f862d4554a468c521a03 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Tue, 5 May 2026 13:14:56 -0700
Subject: [PATCH 11/25] Also permit cookie verification in try_decapsulate()

---
 boringtun/src/ffi/mod.rs   | 16 +++++++++-------
 boringtun/src/noise/mod.rs | 39 +++++++++++++++++++++-----------------
 2 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/boringtun/src/ffi/mod.rs b/boringtun/src/ffi/mod.rs
index 6a38681fe..d7e4970c0 100644
--- a/boringtun/src/ffi/mod.rs
+++ b/boringtun/src/ffi/mod.rs
@@ -11,7 +11,7 @@ use crate::x25519::{PublicKey, StaticSecret};
 use base64::{decode, encode};
 use hex::encode as encode_hex;
 use libc::{raise, SIGSEGV};
-use parking_lot::RwLock;
+use parking_lot::{RwLock, RwLockUpgradableReadGuard};
 use rand_core::OsRng;
 use tracing;
 use tracing_subscriber::fmt;
@@ -364,15 +364,17 @@ pub unsafe extern "C" fn wireguard_read(
     // Slices are not owned, and therefore will not be freed by Rust
     let src = slice::from_raw_parts(src, src_size as usize);
     let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
-    let packet_type = src.first_chunk::<4>().map(|x| u32::from_le_bytes(*x));
 
-    // Data packets can be handled while holding a read lock.
-    if packet_type == Some(super::noise::DATA) {
-        let rotunnel = tunnel.as_ref().unwrap().read();
-        return wireguard_result::from(rotunnel.try_decapsulate(src, dst));
+    // Try handling the packet with a read lock, this is the common case
+    // where we are processing data packets and doing rate limit checks.
+    let tunnel = tunnel.as_ref().unwrap().upgradable_read();
+    if let Some(result) = tunnel.try_decapsulate(None, src, dst) {
+        return wireguard_result::from(result);
     }
 
-    let mut tunnel = tunnel.as_ref().unwrap().write();
+    // Otherwise, whatever this packet is - we will need a write lock to
+    // process it. This is likely a verified handshake packet of some sort.
+    let mut tunnel = RwLockUpgradableReadGuard::upgrade(tunnel);
     wireguard_result::from(tunnel.decapsulate(None, src, dst))
 }
 
diff --git a/boringtun/src/noise/mod.rs b/boringtun/src/noise/mod.rs
index 9e175380c..5fe47b6d2 100644
--- a/boringtun/src/noise/mod.rs
+++ b/boringtun/src/noise/mod.rs
@@ -75,10 +75,10 @@ pub struct Tunn {
 }
 
 type MessageType = u32;
-pub const HANDSHAKE_INIT: MessageType = 1;
-pub const HANDSHAKE_RESP: MessageType = 2;
-pub const COOKIE_REPLY: MessageType = 3;
-pub const DATA: MessageType = 4;
+const HANDSHAKE_INIT: MessageType = 1;
+const HANDSHAKE_RESP: MessageType = 2;
+const COOKIE_REPLY: MessageType = 3;
+const DATA: MessageType = 4;
 
 const HANDSHAKE_INIT_SZ: usize = 148;
 const HANDSHAKE_RESP_SZ: usize = 92;
@@ -322,27 +322,32 @@ impl Tunn {
     /// Receives a UDP datagram from the network and parses it.
     /// Returns TunnResult.
     ///
-    /// This is a subset of decapsulate that only accepts PacketData, but can be called without needing
-    /// to acquire a write lock on the tunnel state. Callers should verify the packet type before calling
-    /// this method.
+    /// This is a subset of decapsulate that operates on a non-mutable tunnel.
+    /// Will return Some(TunnResult) if the packet was handled successfully, or
+    /// None if processing requires a mutable tunnel.
+    ///
+    /// This method can handle the common case of data packet decryption and
+    /// cookie verification while permitting multithreaded access to the tunnel.
     pub fn try_decapsulate<'a>(
         &self,
+        src_addr: Option<IpAddr>,
         datagram: &[u8],
         dst: &'a mut [u8],
-    ) -> TunnResult<'a> {
-        // Dequeueing not supported, caller should use decapsulate() instead.
+    ) -> Option<TunnResult<'a>> {
+        // Packet dequeue operations require a write lock.
         if datagram.is_empty() {
-            return TunnResult::Done;
+            return if self.packet_queue.is_empty() { Some(TunnResult::Done) } else { None };
         }
 
-        // Handle the packet if, and only if, it's a data packet.
-        if let Ok(packet) = Tunn::parse_incoming_packet(datagram) {
-            match packet {
-                Packet::PacketData(p) => self.handle_data(p, dst).unwrap_or_else(TunnResult::from),
-                _ => TunnResult::Done
+        let mut cookie = [0u8; COOKIE_REPLY_SZ];
+        match self.rate_limiter.verify_packet(src_addr, datagram, &mut cookie) {
+            Ok(Packet::PacketData(p)) => Some(self.handle_data(p, dst).unwrap_or_else(TunnResult::from)),
+            Err(TunnResult::WriteToNetwork(cookie)) => {
+                dst[..cookie.len()].copy_from_slice(cookie);
+                return Some(TunnResult::WriteToNetwork(&mut dst[..cookie.len()]));
             }
-        } else {
-            TunnResult::Done
+            Err(TunnResult::Err(e)) => return Some(TunnResult::Err(e)),
+            _ => None
         }
     }
 

From 0ef2c7f57f01fe8ced94b27af1805cc6762a198e Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Thu, 7 May 2026 07:57:34 -0700
Subject: [PATCH 12/25] upgradable_read() generates too much lock contention,
 relock instead

---
 boringtun/src/ffi/mod.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/boringtun/src/ffi/mod.rs b/boringtun/src/ffi/mod.rs
index d7e4970c0..399d9f9cc 100644
--- a/boringtun/src/ffi/mod.rs
+++ b/boringtun/src/ffi/mod.rs
@@ -367,14 +367,16 @@ pub unsafe extern "C" fn wireguard_read(
 
     // Try handling the packet with a read lock, this is the common case
     // where we are processing data packets and doing rate limit checks.
-    let tunnel = tunnel.as_ref().unwrap().upgradable_read();
-    if let Some(result) = tunnel.try_decapsulate(None, src, dst) {
-        return wireguard_result::from(result);
+    {
+        let rotunnel = tunnel.as_ref().unwrap().read();
+        if let Some(result) = rotunnel.try_decapsulate(None, src, dst) {
+            return wireguard_result::from(result);
+        }
     }
 
     // Otherwise, whatever this packet is - we will need a write lock to
     // process it. This is likely a verified handshake packet of some sort.
-    let mut tunnel = RwLockUpgradableReadGuard::upgrade(tunnel);
+    let mut tunnel = tunnel.as_ref().unwrap().write();
     wireguard_result::from(tunnel.decapsulate(None, src, dst))
 }
 

From 51eb11453c53f695c88e0b3df588f9f38488424e Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Thu, 7 May 2026 08:54:05 -0700
Subject: [PATCH 13/25] Abandon the AF_UNIX thing and directly handle packets

---
 boringtun/benches/ffi_benches/main.c          |   9 +-
 .../benches/ffi_benches/wg_bench_client.c     | 288 ++++--------------
 .../benches/ffi_benches/wg_bench_client.h     |   8 +-
 3 files changed, 57 insertions(+), 248 deletions(-)

diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
index 7ad7da42f..ec556bff3 100644
--- a/boringtun/benches/ffi_benches/main.c
+++ b/boringtun/benches/ffi_benches/main.c
@@ -1,7 +1,6 @@
 
 
 #include <errno.h>
-#include <fcntl.h>
 #include <getopt.h>
 #include <limits.h>
 #include <signal.h>
@@ -10,7 +9,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/ioctl.h>
-#include <sys/socket.h>
 #include <sys/time.h>
 #include <time.h> 
 #include <pthread.h>
@@ -222,8 +220,8 @@ int main(int argc, char* argv[]) {
     struct wg_bench_client* b = wg_bench_create();
 
     // Connect the two clients.
-    wg_bench_connect(a, b->pubkey);
-    wg_bench_connect(b, a->pubkey);
+    wg_bench_connect(a, b);
+    wg_bench_connect(b, a);
 
     struct timespec start;
     struct timespec cpustart;
@@ -238,8 +236,6 @@ int main(int argc, char* argv[]) {
     for (int i = 0; i < num_workers; i++) {
         wg_bench_start_worker(a);
         wg_bench_start_worker(b);
-        wg_bench_start_send(a);
-        wg_bench_start_send(b);
     }
 
     struct timespec now;
@@ -273,6 +269,7 @@ int main(int argc, char* argv[]) {
     wg_bench_fetch_stats(b, &stats);
     print_errors(&stats);
     print_stats(&stats, timespec_elapsed(&now, &start), timespec_elapsed(&cpu, &cpustart));
+    printf("\n");
 
     wg_bench_close(a);
     wg_bench_close(b);
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.c b/boringtun/benches/ffi_benches/wg_bench_client.c
index 54aff66ec..e176e5cd9 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.c
+++ b/boringtun/benches/ffi_benches/wg_bench_client.c
@@ -2,22 +2,12 @@
 
 #include <arpa/inet.h>
 #include <errno.h>
-#include <fcntl.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/socket.h>
-#include <sys/un.h>
 #include <unistd.h>
 
-#ifdef __linux
-#include <sys/epoll.h>
-#include <sys/timerfd.h>
-#else
-#include <sys/event.h>
-#endif
-
 #define WG_BENCH_MTU 2048
 
 static void wg_worker_sigmask() {
@@ -53,7 +43,36 @@ struct wg_bench_udphdr {
     uint8_t  dgram[];
 };
 
-static void* wg_bench_send_worker(void *arg) {
+static void wg_bench_input_packet(struct wg_bench_client *client, const void *data, size_t len) {
+    uint8_t plaintext[WG_BENCH_MTU];
+    struct wireguard_result result;
+    result = wireguard_read(client->tunnel, data, len, plaintext, sizeof(plaintext));
+    switch (result.op) {
+        case WIREGUARD_DONE:
+            break;
+        
+        case WIREGUARD_ERROR:
+            if (result.size < WG_BENCH_MAX_ERRORS) {
+                atomic_fetch_add(&client->stats.errors[result.size], 1);
+            }
+            break;
+
+        case WRITE_TO_NETWORK:
+            // This is not expected, but I guess it's possible
+            return wg_bench_input_packet(client->peer, plaintext, result.size);
+
+        case WRITE_TO_TUNNEL_IPV4:
+        case WRITE_TO_TUNNEL_IPV6:
+            atomic_fetch_add(&client->stats.rx_packets, 1);
+            atomic_fetch_add(&client->stats.rx_bytes, result.size);
+            break;
+
+        default:
+            break;
+    }
+}
+
+static void* wg_bench_worker(void *arg) {
     struct wg_bench_client *client = (struct wg_bench_client *)arg;
     struct wireguard_result result;
     uint8_t ciphertext[WG_BENCH_MTU + 32];
@@ -75,19 +94,8 @@ static void* wg_bench_send_worker(void *arg) {
     udp->dport = 0x5678;
     udp->cksum = 0;
 
-    // Create a separate socket for the send workflows.
-    int fd = socket(AF_UNIX, SOCK_DGRAM, 0);
-    if (fd < 0) {
-        fprintf(stderr, "worker socket error: %s\n", strerror(errno));
-        return NULL;
-    }
-    if (connect(fd, (const struct sockaddr*)&client->peer, sizeof(client->peer)) < 0) {
-        fprintf(stderr, "worker connect error: %s\n", strerror(errno));
-        close(fd);
-        return NULL;
-    }
-
-    while (!client->worker_shutdown) {
+    wg_worker_sigmask();
+    while (!atomic_load(&client->worker_shutdown)) {
         // Pick a random datagram size between 512-1024 and fill with a random byte.
         int dsize = 512 + (rand() % 512);
         int pktlen = sizeof(struct wg_bench_iphdr) + sizeof(struct wg_bench_udphdr) + dsize;
@@ -114,16 +122,7 @@ static void* wg_bench_send_worker(void *arg) {
             case WRITE_TO_NETWORK:
                 atomic_fetch_add(&client->stats.tx_packets, 1);
                 atomic_fetch_add(&client->stats.tx_bytes, pktlen);
-                if (send(fd, ciphertext, result.size, 0) < 0) {
-                    atomic_fetch_add(&client->stats.tx_drops, 1);
-                    if (errno == EAGAIN) continue;
-                    if (errno == ENOBUFS) continue;
-                    if (!client->worker_shutdown) {
-                        fprintf(stderr, "worker tx error: %s\n", strerror(errno));
-                    }
-                    close(fd);
-                    return NULL;
-                }
+                wg_bench_input_packet(client->peer, ciphertext, result.size);
                 break;
 
             case WRITE_TO_TUNNEL_IPV4:
@@ -141,29 +140,28 @@ static void* wg_bench_send_worker(void *arg) {
     return NULL;
 }
 
-static void wg_bench_tick(struct wg_bench_client *client) {
-    const struct sockaddr* peer = (const struct sockaddr*)&client->peer;
-    socklen_t peerlen = sizeof(client->peer);
+static void *wg_bench_background(void* arg) {
+    struct wg_bench_client *client = (struct wg_bench_client *)arg;
+    uint8_t ciphertext[WG_BENCH_MTU + 32];
 
-    while (true) {
-        uint8_t ciphertext[WG_BENCH_MTU + 32];
+    wg_worker_sigmask();
+    while (!atomic_load(&client->worker_shutdown)) {
         struct wireguard_result result;
         result = wireguard_tick(client->tunnel, ciphertext, sizeof(ciphertext));
 
         // Process timeouts and state updates.
         switch (result.op) {
             case WIREGUARD_DONE:
-                return;
+                usleep(100000);
+                break;
 
             case WIREGUARD_ERROR:
                 fprintf(stderr, "worker tick error: %zu\n", result.size);
-                return;
+                break;
 
             case WRITE_TO_NETWORK:
-                if (sendto(client->fd, ciphertext, result.size, MSG_DONTWAIT, peer, sizeof(client->peer)) < 0) {
-                    atomic_fetch_add(&client->stats.tx_drops, 1);
-                }
-                continue;
+                wg_bench_input_packet(client->peer, ciphertext, result.size);
+                break;
 
             case WRITE_TO_TUNNEL_IPV4:
             case WRITE_TO_TUNNEL_IPV6:
@@ -173,132 +171,13 @@ static void wg_bench_tick(struct wg_bench_client *client) {
             
             default:
                 fprintf(stderr, "worker tick unknown: %d\n", result.op);
-                return;
-        }
-    }
-}
-
-static void wg_bench_recv(struct wg_bench_client *client) {
-    uint8_t ciphertext[WG_BENCH_MTU + 32];
-    uint8_t plaintext[WG_BENCH_MTU];
-    struct wireguard_result result;
-
-    const struct sockaddr* peer = (const struct sockaddr*)&client->peer;
-    socklen_t peerlen = sizeof(client->peer);
-
-    // Read a packet.
-    ssize_t rx = recv(client->fd, ciphertext, sizeof(ciphertext), 0);
-    if (rx == 0) {
-        fprintf(stderr, "worker shutdown\n");
-        return;
-    }
-    if (rx < 0) {
-        if (!client->worker_shutdown) {
-            //fprintf(stderr, "worker rx error: %s\n", strerror(errno));
-        }
-        return;
-    }
-
-    result = wireguard_read(client->tunnel, ciphertext, rx, plaintext, sizeof(plaintext));
-    switch (result.op) {
-        case WIREGUARD_DONE:
-            break;
-        
-        case WIREGUARD_ERROR:
-            if (result.size < WG_BENCH_MAX_ERRORS) {
-                atomic_fetch_add(&client->stats.errors[result.size], 1);
-            }
-            break;
-
-        case WRITE_TO_NETWORK:
-            // This is not expected, but I guess it's possible
-            if (sendto(client->fd, plaintext, result.size, MSG_DONTWAIT, peer, peerlen) < 0) {
-                atomic_fetch_add(&client->stats.tx_drops, 1);
-            }
-            break;
-
-        case WRITE_TO_TUNNEL_IPV4:
-        case WRITE_TO_TUNNEL_IPV6:
-            atomic_fetch_add(&client->stats.rx_packets, 1);
-            atomic_fetch_add(&client->stats.rx_bytes, result.size);
-            break;
-
-        default:
-            break;
-    }
-}
-
-static void* wg_bench_worker(void *arg) {
-    struct wg_bench_client *client = (struct wg_bench_client *)arg;
-
-    wg_worker_sigmask();
-    while (!client->worker_shutdown) {
-#ifdef __linux
-        // Wait for an event to handle.
-        struct epoll_event ev[16];
-        int maxev = sizeof(ev)/sizeof(struct epoll_event);
-        int nev = epoll_wait(client->queue, ev, maxev, -1);
-        if (nfds == -1) {
-            if (errno == EINTR) continue;
-            fprintf(stderr, "worker epoll error: %s\n", strerror(errno));
-            continue;
-        }
-    
-        // Handle events.
-        for (int i = 0; i < nev; i++) {
-            if (ev[i].data.fd < 0) {
-                wg_bench_tick(client);
-            } else if (ev[i].data.fd == client->fd) {
-                wg_bench_recv(client);
-            }
-        }
-#else
-        // Wait for an event to handle.
-        struct kevent kev[16];
-        int maxev = sizeof(kev)/sizeof(struct kevent);
-        int nev = kevent(client->queue, NULL, 0, kev, maxev, NULL);
-        if (nev < 0) {
-            if (errno == EINTR) continue;
-            fprintf(stderr, "worker kevent error: %s\n", strerror(errno));
-            continue;
-        }
-
-        // Handle events.
-        for (int i = 0; i < nev; i++) {
-            if (kev[i].filter == EVFILT_TIMER) {
-                wg_bench_tick(client);
-            } else if (kev[i].filter == EVFILT_READ) {
-                wg_bench_recv(client);
-            }
+                return NULL;
         }
-#endif
     }
 
     return NULL;
 }
 
-// Fill a sockaddr_un with the named pipe for a given public key.
-static struct sockaddr* wg_bench_sockaddr(const char* pubkey, struct sockaddr_un *addr) {
-    memset(addr, 0, sizeof(struct sockaddr_un));
-    addr->sun_family = AF_UNIX;
-#ifdef SUN_LEN
-    addr->sun_len = sizeof(struct sockaddr_un);
-#endif
-
-    // Set the name to /tmp/ffi-bench-<base64key>.sock, using URL-safe encoding.
-    strcpy(addr->sun_path, "/tmp/ffi-bench-");
-    char *urlsafe = strchr(addr->sun_path, '\0');
-    while (*pubkey != '\0') {
-        char c = *pubkey++;
-        if (c == '+') *urlsafe++ = '-';
-        else if (c == '/') *urlsafe++ = '_';
-        else if (c != '=') *urlsafe++ = c;
-    }
-    strcat(urlsafe, ".sock");
-
-    return (struct sockaddr*)addr;
-}
-
 struct wg_bench_client* wg_bench_create() {
     struct wg_bench_client* client = calloc(sizeof(struct wg_bench_client), 1);
     if (!client) {
@@ -306,63 +185,16 @@ struct wg_bench_client* wg_bench_create() {
     }
     client->secret = x25519_secret_key();
     client->pubkey = x25519_key_to_base64(x25519_public_key(client->secret));
-
-    // Create a named datagram pipe for handling packets.
-    client->fd = socket(AF_UNIX, SOCK_DGRAM, 0);
-    if (client->fd < 0) {
-        x25519_key_to_str_free(client->pubkey);
-        free(client);
-        return NULL;
-    }
-
-    struct sockaddr* sa = wg_bench_sockaddr(client->pubkey, &client->addr);
-    if (bind(client->fd, sa, sizeof(client->addr)) < 0) {
-        x25519_key_to_str_free(client->pubkey);
-        close(client->fd);
-        free(client);
-        return NULL;
-    }
-
-    fcntl(client->fd, F_SETFL, fcntl(client->fd, F_GETFL) | O_NONBLOCK);
-#ifdef __linux
-    client->queue = epoll_create1(0);
-#else
-    client->queue = kqueue();
-#endif
     return client;
 }
 
-void wg_bench_connect(struct wg_bench_client* client, const char* pubkey) {
+void wg_bench_connect(struct wg_bench_client* client, struct wg_bench_client* peer) {
     const char* statickey = x25519_key_to_base64(client->secret);
-    client->tunnel = new_tunnel(statickey, pubkey, NULL, 5, rand() & 0xffffff);
+    client->peer = peer;
+    client->tunnel = new_tunnel(statickey, peer->pubkey, NULL, 5, rand() & 0xffffff);
     x25519_key_to_str_free(statickey);
 
-    wg_bench_sockaddr(pubkey, &client->peer);
-
-    // Begin packet processing.
-#ifdef __linux
-    struct epoll_event ev;
-    ev.events = EPOLLIN;
-    ev.data.fd = client->fd;
-    epoll_ctl(client->queue, EPOLL_CTL_ADD, client->fd, &ev);
-
-    int timer = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK);
-    ev.events = EPOLLIN;
-    ev.data.fd = -1;
-    epoll_ctl(client->queue, EPOLL_CTL_ADD, timer, &ev);
-
-    struct itimerspec itspec;
-    itspec.it_value.tv_sec = 0;
-    itspec.it_value.tv_nsec = 100000000;
-    itspec.it_interval.tv_sec = 0;
-    itspec.it_interval.tv_nsec = 100000000;
-    timerfd_settime(timer, 0, &itspec, NULL);
-#else
-    struct kevent kev[2];
-    EV_SET(&kev[0], client->fd, EVFILT_READ, EV_ADD, 0, 0, NULL);
-    EV_SET(&kev[1], 1, EVFILT_TIMER, EV_ADD, 0, 100, NULL);
-    kevent(client->queue, kev, 2, NULL, 0, NULL);
-#endif
+    pthread_create(&client->background, NULL, wg_bench_background, client);
 }
 
 void wg_bench_start_handshake(struct wg_bench_client* client) {
@@ -383,9 +215,7 @@ void wg_bench_start_handshake(struct wg_bench_client* client) {
             break;
 
         case WRITE_TO_NETWORK:
-            if (sendto(client->fd, ciphertext, result.size, MSG_DONTWAIT, peer, sizeof(client->peer)) < 0) {
-                atomic_fetch_add(&client->stats.tx_drops, 1);
-            }
+            wg_bench_input_packet(client->peer, ciphertext, result.size);
             break;
 
         case WRITE_TO_TUNNEL_IPV4:
@@ -410,16 +240,6 @@ void wg_bench_start_worker(struct wg_bench_client* client) {
     client->worker_count++;
 }
 
-void wg_bench_start_send(struct wg_bench_client* client) {
-    if (client->worker_count >= WG_BENCH_MAX_THREADS) {
-        return;
-    }
-    if (pthread_create(&client->workers[client->worker_count], NULL, wg_bench_send_worker, client) != 0) {
-        return;
-    }
-    client->worker_count++;
-}
-
 void wg_bench_fetch_stats(const struct wg_bench_client* client, struct wg_bench_statistics* st) {
     atomic_fetch_add(&st->tx_packets, atomic_load(&client->stats.tx_packets));
     atomic_fetch_add(&st->tx_drops, atomic_load(&client->stats.tx_drops));
@@ -433,17 +253,13 @@ void wg_bench_fetch_stats(const struct wg_bench_client* client, struct wg_bench_
 
 void wg_bench_close(struct wg_bench_client* client) {
     // Signal that we are shutting down and terminate workers.
+    atomic_store(&client->worker_shutdown, true);
     client->worker_shutdown = 1;
+    pthread_kill(client->background, SIGHUP);
+    pthread_join(client->background, NULL);
     for (int i = 0; i < client->worker_count; i++) {
-        pthread_cancel(client->workers[i]);
-        pthread_kill(client->workers[i], SIGHUP);
         pthread_join(client->workers[i], NULL);
     }
-    close(client->queue);
-
-    shutdown(client->fd, SHUT_RDWR);
-    unlink(client->addr.sun_path);
-    close(client->fd);
 
     x25519_key_to_str_free(client->pubkey);
     tunnel_free(client->tunnel);
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.h b/boringtun/benches/ffi_benches/wg_bench_client.h
index addcd4ace..2bdb3b968 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.h
+++ b/boringtun/benches/ffi_benches/wg_bench_client.h
@@ -2,7 +2,6 @@
 
 #include <pthread.h>
 #include <stdatomic.h>
-#include <sys/un.h>
 #include "wireguard_ffi.h"
 
 #define WG_BENCH_MAX_THREADS 256
@@ -22,10 +21,7 @@ struct wg_bench_client {
     struct x25519_key secret;
     const char *pubkey;
 
-    int                 fd;
-    int                 queue;
-    struct sockaddr_un  addr;
-    struct sockaddr_un  peer;
+    struct wg_bench_client* peer;
 
     // The packet send and receive worker pool.
     int       worker_handshake;
@@ -39,7 +35,7 @@ struct wg_bench_client {
 };
 
 struct wg_bench_client* wg_bench_create();
-void wg_bench_connect(struct wg_bench_client* client, const char* pubkey);
+void wg_bench_connect(struct wg_bench_client* client, struct wg_bench_client* peer);
 void wg_bench_start_handshake(struct wg_bench_client* client);
 void wg_bench_start_send(struct wg_bench_client* client);
 void wg_bench_start_worker(struct wg_bench_client* client);

From c22ca29e1593c6d0b1cab0185dc344c3a6d061b8 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Thu, 7 May 2026 09:28:58 -0700
Subject: [PATCH 14/25] Add duration argument and re-negotiate handshake every
 second

---
 boringtun/benches/ffi_benches/main.c | 35 +++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
index ec556bff3..1c438eca8 100644
--- a/boringtun/benches/ffi_benches/main.c
+++ b/boringtun/benches/ffi_benches/main.c
@@ -65,6 +65,13 @@ static void handle_signal(int sig) {
     }
 }
 
+static long timespec_cmp(const struct timespec *a, const struct timespec* b) {
+    if (long i = (a->tv_sec - b->tv_sec)) {
+        return i;
+    }
+    return (a->tv_nsec - b->tv_nsec);
+}
+
 static double timespec_elapsed(const struct timespec *a, const struct timespec* b) {
     double result = (a->tv_sec - b->tv_sec) * 1000000000.0;
     return (double)(result + a->tv_nsec - b->tv_nsec) / 1000000000.0;
@@ -164,18 +171,21 @@ static void print_usage(FILE* fp, const char* name) {
     fprintf(fp, "Run FFI benchmarks for the boringtun library.\n");
     fprintf(fp, "\n");
     fprintf(fp, "Options:\n");
+    fprintf(fp, "\t--time, -t DUR   run benchmark for DUR seconds\n");
     fprintf(fp, "\t--jobs, -j NUM   create NUM parallel threads\n");
     fprintf(fp, "\t--help, -h       display this message and exit\n");
 }
 
 int main(int argc, char* argv[]) {
-    const char* shortopts = "hj:";
+    const char* shortopts = "t:j:h";
     const struct option longopts[] = {
-        {"help", no_argument,       0, 'h'},
+        {"time", required_argument, 0, 't'},
         {"jobs", required_argument, 0, 'j'},
+        {"help", no_argument,       0, 'h'},
         {NULL, 0, 0, 0}
     };
     unsigned int num_workers = 1;
+    unsigned int duration = 10;
 
     // Parse options
     while (true) {
@@ -187,6 +197,14 @@ int main(int argc, char* argv[]) {
 
         char* endp;
         switch (opt) {
+            case 't':
+                duration = strtoul(optarg, &endp, 10);
+                if (*endp != '\0' || (duration == 0)) {
+                    fprintf(stderr, "Invalid duration: %s\n", optarg);
+                    return 1;
+                }
+                break;
+
             case 'j':
                 num_workers = strtoul(optarg, &endp, 10);
                 if (*endp != '\0' || (num_workers == 0)) {
@@ -226,10 +244,13 @@ int main(int argc, char* argv[]) {
     struct timespec start;
     struct timespec cpustart;
     struct timespec end;
+    struct timespec renegotiate;
     clock_gettime(CLOCK_MONOTONIC, &start);
     clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &cpustart);
-    end.tv_sec = start.tv_sec + 10;
+    end.tv_sec = start.tv_sec + duration;
     end.tv_nsec = start.tv_nsec;
+    renegotiate.tv_sec = start.tv_sec + 1;
+    renegotiate.tv_nsec = start.tv_nsec;
 
     // Launch workers.
     wg_bench_start_handshake(a);
@@ -252,11 +273,13 @@ int main(int argc, char* argv[]) {
         print_stats(&stats, timespec_elapsed(&now, &start), timespec_elapsed(&cpu, &cpustart));
 
         // Check for the end condition.
-        if (end.tv_sec < now.tv_sec) {
-            break;
-        } else if ((end.tv_sec == now.tv_sec) && (end.tv_nsec < now.tv_nsec)) {
+        if (timespec_cmp(&end, &now) < 0) {
             break;
         }
+        if (timespec_cmp(&renegotiate, &now) < 0) {
+            renegotiate.tv_sec++;
+            wg_bench_start_handshake(a);
+        }
 
         // Sleep for more data.
         usleep(100000);

From 2caf9aca545cbce9a33193d518e915052ad6bf1c Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Fri, 8 May 2026 09:10:20 -0700
Subject: [PATCH 15/25] Switch to portable_atomic to support 32-bit systems

---
 boringtun/src/noise/mod.rs    | 2 +-
 boringtun/src/noise/timers.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/boringtun/src/noise/mod.rs b/boringtun/src/noise/mod.rs
index 5fe47b6d2..8c8308737 100644
--- a/boringtun/src/noise/mod.rs
+++ b/boringtun/src/noise/mod.rs
@@ -18,7 +18,7 @@ use std::collections::VecDeque;
 use std::convert::{TryFrom, TryInto};
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 use std::sync::Arc;
-use std::sync::atomic::{AtomicUsize, Ordering};
+use portable_atomic::{AtomicUsize, Ordering};
 use std::time::Duration;
 
 /// The default value to use for rate limiting, when no other rate limiter is defined
diff --git a/boringtun/src/noise/timers.rs b/boringtun/src/noise/timers.rs
index d657f0966..d3089f233 100644
--- a/boringtun/src/noise/timers.rs
+++ b/boringtun/src/noise/timers.rs
@@ -3,7 +3,7 @@
 
 use super::errors::WireGuardError;
 use crate::noise::{Tunn, TunnResult};
-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use portable_atomic::{AtomicBool, AtomicU64, Ordering};
 
 use std::time::Duration;
 

From 7e352eb89d97e3d8579f35c6fa57760aa5951e2c Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Fri, 8 May 2026 11:56:35 -0700
Subject: [PATCH 16/25] Lets make RateLimiter lock-free while we're at it

---
 boringtun/src/noise/rate_limiter.rs | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/boringtun/src/noise/rate_limiter.rs b/boringtun/src/noise/rate_limiter.rs
index 421d5680f..a04b1b283 100644
--- a/boringtun/src/noise/rate_limiter.rs
+++ b/boringtun/src/noise/rate_limiter.rs
@@ -2,6 +2,8 @@ use super::handshake::{b2s_hash, b2s_keyed_mac_16, b2s_keyed_mac_16_2, b2s_mac_2
 use crate::noise::handshake::{LABEL_COOKIE, LABEL_MAC1};
 use crate::noise::{HandshakeInit, HandshakeResponse, Packet, Tunn, TunnResult, WireGuardError};
 
+use std::time::Duration;
+
 #[cfg(feature = "mock-instant")]
 use mock_instant::Instant;
 use portable_atomic::{AtomicU64, Ordering};
@@ -13,7 +15,6 @@ use crate::sleepyinstant::Instant;
 use aead::generic_array::GenericArray;
 use aead::{AeadInPlace, KeyInit};
 use chacha20poly1305::{Key, XChaCha20Poly1305};
-use parking_lot::Mutex;
 use rand_core::{OsRng, RngCore};
 use ring::constant_time::verify_slices_are_equal;
 
@@ -22,7 +23,7 @@ const COOKIE_SIZE: usize = 16;
 const COOKIE_NONCE_SIZE: usize = 24;
 
 /// How often should reset count in seconds
-const RESET_PERIOD: u64 = 1;
+const RESET_PERIOD: Duration = Duration::from_secs(1);
 
 type Cookie = [u8; COOKIE_SIZE];
 
@@ -47,8 +48,8 @@ pub struct RateLimiter {
     limit: u64,
     /// The counter since last reset
     count: AtomicU64,
-    /// The time last reset was performed on this rate limiter
-    last_reset: Mutex<Instant>,
+    /// The time last reset was performed on this rate limiter, in milliseconds from start_time
+    last_reset: AtomicU64,
 }
 
 impl RateLimiter {
@@ -64,7 +65,7 @@ impl RateLimiter {
             cookie_key: b2s_hash(LABEL_COOKIE, public_key.as_bytes()).into(),
             limit,
             count: AtomicU64::new(0),
-            last_reset: Mutex::new(Instant::now()),
+            last_reset: AtomicU64::new(0),
         }
     }
 
@@ -77,11 +78,13 @@ impl RateLimiter {
     /// Reset packet count (ideally should be called with a period of 1 second)
     pub fn reset_count(&self) {
         // The rate limiter is not very accurate, but at the scale we care about it doesn't matter much
-        let current_time = Instant::now();
-        let mut last_reset_time = self.last_reset.lock();
-        if current_time.duration_since(*last_reset_time).as_secs() >= RESET_PERIOD {
-            self.count.store(0, Ordering::SeqCst);
-            *last_reset_time = current_time;
+        let now = Instant::now().duration_since(self.start_time);
+        let last_msec = self.last_reset.load(Ordering::Acquire);
+        let last_reset = Duration::from_millis(last_msec);
+        if now - last_reset >= RESET_PERIOD {
+            if self.last_reset.compare_exchange(last_msec, now.as_millis() as u64, Ordering::SeqCst, Ordering::Relaxed).is_ok() {
+                self.count.store(0, Ordering::SeqCst);
+            }
         }
     }
 

From f7e1906db6bbbb76d67fd8328af715933fbf29f1 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Fri, 8 May 2026 12:53:15 -0700
Subject: [PATCH 17/25] timer_tick should probably use a CAS loop

---
 boringtun/src/noise/timers.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/boringtun/src/noise/timers.rs b/boringtun/src/noise/timers.rs
index d3089f233..a54504e9e 100644
--- a/boringtun/src/noise/timers.rs
+++ b/boringtun/src/noise/timers.rs
@@ -110,8 +110,14 @@ impl Tunn {
             _ => {}
         }
 
-        let msecs = self.timers.current.as_millis() as u64;
-        self.timers.timers[timer_name as usize].store(msecs, Ordering::Release);
+        let index = timer_name as usize;
+        let current = self.timers.current.as_millis() as u64;
+        let mut prev = self.timers.timers[index].load(Ordering::Acquire);
+        while prev < current {
+            prev = self.timers.timers[index]
+                .compare_exchange_weak(prev, current, Ordering::SeqCst, Ordering::Acquire)
+                .unwrap_or_else(|x| x);
+        }
     }
 
     pub(super) fn timer_fetch(&self, timer_name: TimerName) -> Duration {

From ccce7ee8de088e144ef7b8928bbb97c12c6f544b Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Fri, 8 May 2026 14:23:28 -0700
Subject: [PATCH 18/25] Unify wireguard_write and wireguard_try_write

---
 boringtun/benches/ffi_benches/Makefile        |  2 +-
 boringtun/benches/ffi_benches/main.c          |  4 +-
 .../benches/ffi_benches/wg_bench_client.c     |  5 +--
 boringtun/src/ffi/mod.rs                      | 43 +++++++++----------
 boringtun/src/noise/mod.rs                    | 15 +++++--
 boringtun/src/wireguard_ffi.h                 |  6 ---
 6 files changed, 35 insertions(+), 40 deletions(-)

diff --git a/boringtun/benches/ffi_benches/Makefile b/boringtun/benches/ffi_benches/Makefile
index 923db3432..a29ecc15a 100644
--- a/boringtun/benches/ffi_benches/Makefile
+++ b/boringtun/benches/ffi_benches/Makefile
@@ -18,7 +18,7 @@ BENCH_CFLAGS := -I ${PKGDIR}/src
 BENCH_SRCS := main.c wg_bench_client.c
 BENCH_OBJS := $(patsubst %.c,%.o,${BENCH_SRCS})
 
-# Profiling support
+# Build with profiling support
 BENCH_CFLAGS += -g -fno-omit-frame-pointer
 BENCH_LDFLAGS := -g -fno-omit-frame-pointer
 release/libboringtun.a: export RUSTFLAGS=-C force-frame-pointers=y
diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
index 1c438eca8..e93dfc1e0 100644
--- a/boringtun/benches/ffi_benches/main.c
+++ b/boringtun/benches/ffi_benches/main.c
@@ -1,5 +1,3 @@
-
-
 #include <errno.h>
 #include <getopt.h>
 #include <limits.h>
@@ -225,7 +223,7 @@ int main(int argc, char* argv[]) {
     srand(time(0));
     set_logging_function(wg_print_msg);
 
-    // This thread should handle signals.
+    // The main thread should handle signals.
     struct sigaction action = {
         .sa_handler = handle_signal,
     };
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.c b/boringtun/benches/ffi_benches/wg_bench_client.c
index e176e5cd9..9262700f8 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.c
+++ b/boringtun/benches/ffi_benches/wg_bench_client.c
@@ -58,7 +58,6 @@ static void wg_bench_input_packet(struct wg_bench_client *client, const void *da
             break;
 
         case WRITE_TO_NETWORK:
-            // This is not expected, but I guess it's possible
             return wg_bench_input_packet(client->peer, plaintext, result.size);
 
         case WRITE_TO_TUNNEL_IPV4:
@@ -106,8 +105,8 @@ static void* wg_bench_worker(void *arg) {
         ip->cksum = 0;
 
         // Encrypt the packet.
-        result = wireguard_try_write(client->tunnel, plaintext, pktlen,
-                                     ciphertext, sizeof(ciphertext));
+        result = wireguard_write(client->tunnel, plaintext, pktlen,
+                                 ciphertext, sizeof(ciphertext));
         switch (result.op) {
             case WIREGUARD_DONE:
                 break;
diff --git a/boringtun/src/ffi/mod.rs b/boringtun/src/ffi/mod.rs
index 399d9f9cc..2a2c26a16 100644
--- a/boringtun/src/ffi/mod.rs
+++ b/boringtun/src/ffi/mod.rs
@@ -11,7 +11,7 @@ use crate::x25519::{PublicKey, StaticSecret};
 use base64::{decode, encode};
 use hex::encode as encode_hex;
 use libc::{raise, SIGSEGV};
-use parking_lot::{RwLock, RwLockUpgradableReadGuard};
+use parking_lot::RwLock;
 use rand_core::OsRng;
 use tracing;
 use tracing_subscriber::fmt;
@@ -327,28 +327,25 @@ pub unsafe extern "C" fn wireguard_write(
     dst: *mut u8,
     dst_size: u32,
 ) -> wireguard_result {
-    let mut tunnel = tunnel.as_ref().unwrap().write();
     // Slices are not owned, and therefore will not be freed by Rust
     let src = slice::from_raw_parts(src, src_size as usize);
     let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
-    wireguard_result::from(tunnel.encapsulate(src, dst))
-}
 
-/// Write an IP packet from the tunnel interface.
-/// For more details check noise::tunnel_to_network functions.
-#[no_mangle]
-pub unsafe extern "C" fn wireguard_try_write(
-    tunnel: *const RwLock<Tunn>,
-    src: *const u8,
-    src_size: u32,
-    dst: *mut u8,
-    dst_size: u32,
-) -> wireguard_result {
-    let tunnel = tunnel.as_ref().unwrap().read();
-    // Slices are not owned, and therefore will not be freed by Rust
-    let src = slice::from_raw_parts(src, src_size as usize);
-    let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
-    wireguard_result::from(tunnel.try_encapsulate(src, dst))
+    // Try handling the packet with only a read lock, this covers the common
+    // case where we are encrypting data packets and a valid session exists.
+    {
+        let rotunnel = tunnel.as_ref().unwrap().read();
+        let result = rotunnel.try_encapsulate(src, dst);
+        if !matches!(result, TunnResult::Done) {
+            return wireguard_result::from(result);
+        }
+    }
+
+    // Otherwise, acquire a write lock to queue the packet and start a new
+    // handshake if there isn't already one in progress.
+    let mut tunnel = tunnel.as_ref().unwrap().write();
+    tunnel.queue_packet(src);
+    wireguard_result::from(tunnel.format_handshake_initiation(dst, false))
 }
 
 /// Read a UDP packet from the server.
@@ -365,8 +362,8 @@ pub unsafe extern "C" fn wireguard_read(
     let src = slice::from_raw_parts(src, src_size as usize);
     let dst = slice::from_raw_parts_mut(dst, dst_size as usize);
 
-    // Try handling the packet with a read lock, this is the common case
-    // where we are processing data packets and doing rate limit checks.
+    // Try handling the packet with only a read lock, this covers the common
+    // case where we are processing data packets and doing validity checks.
     {
         let rotunnel = tunnel.as_ref().unwrap().read();
         if let Some(result) = rotunnel.try_decapsulate(None, src, dst) {
@@ -374,8 +371,8 @@ pub unsafe extern "C" fn wireguard_read(
         }
     }
 
-    // Otherwise, whatever this packet is - we will need a write lock to
-    // process it. This is likely a verified handshake packet of some sort.
+    // Otherwise, we must acquire a write lock to continue processing this
+    // packet. This is likely a verified handshake packet of some sort.
     let mut tunnel = tunnel.as_ref().unwrap().write();
     wireguard_result::from(tunnel.decapsulate(None, src, dst))
 }
diff --git a/boringtun/src/noise/mod.rs b/boringtun/src/noise/mod.rs
index 8c8308737..ee9078a21 100644
--- a/boringtun/src/noise/mod.rs
+++ b/boringtun/src/noise/mod.rs
@@ -268,6 +268,13 @@ impl Tunn {
         return self.format_handshake_initiation(dst, false);
     }
 
+    /// Encapsulate a single packet from the tunnel interface.
+    /// Returns TunnResult::WriteToNetwork if there is a valid session, or
+    /// TunnResult::Done if the packet could not be handled yet.
+    ///
+    /// # Panics
+    /// Panics if dst buffer is too small.
+    /// Size of dst should be at least src.len() + 32, and no less than 148 bytes.
     pub fn try_encapsulate<'a>(&self, src: &[u8], dst: &'a mut [u8]) -> TunnResult<'a> {
         let current = self.current.load(Ordering::Relaxed);
         if let Some(ref session) = self.sessions[current % N_SESSIONS] {
@@ -322,12 +329,12 @@ impl Tunn {
     /// Receives a UDP datagram from the network and parses it.
     /// Returns TunnResult.
     ///
-    /// This is a subset of decapsulate that operates on a non-mutable tunnel.
+    /// This is a subset of decapsulate that operates on a read only tunnel.
     /// Will return Some(TunnResult) if the packet was handled successfully, or
-    /// None if processing requires a mutable tunnel.
+    /// None if processing requires a mutable reference to the tunnel.
     ///
     /// This method can handle the common case of data packet decryption and
-    /// cookie verification while permitting multithreaded access to the tunnel.
+    /// cookie verification without needing to hold a write lock on the tunnel.
     pub fn try_decapsulate<'a>(
         &self,
         src_addr: Option<IpAddr>,
@@ -573,7 +580,7 @@ impl Tunn {
     }
 
     /// Push packet to the back of the queue
-    fn queue_packet(&mut self, packet: &[u8]) {
+    pub fn queue_packet(&mut self, packet: &[u8]) {
         if self.packet_queue.len() < MAX_QUEUE_DEPTH {
             // Drop if too many are already in queue
             self.packet_queue.push_back(packet.to_vec());
diff --git a/boringtun/src/wireguard_ffi.h b/boringtun/src/wireguard_ffi.h
index 71bed40a3..5cdd90125 100644
--- a/boringtun/src/wireguard_ffi.h
+++ b/boringtun/src/wireguard_ffi.h
@@ -89,12 +89,6 @@ struct wireguard_result wireguard_write(const struct wireguard_tunnel *tunnel,
                                         uint8_t *dst,
                                         uint32_t dst_size);
 
-struct wireguard_result wireguard_try_write(const struct wireguard_tunnel *tunnel,
-                                            const uint8_t *src,
-                                            uint32_t src_size,
-                                            uint8_t *dst,
-                                            uint32_t dst_size);
-
 struct wireguard_result wireguard_read(const struct wireguard_tunnel *tunnel,
                                        const uint8_t *src,
                                        uint32_t src_size,

From 417f334ba37dd121d4fe870267a0315823f124b8 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Tue, 12 May 2026 12:51:09 -0700
Subject: [PATCH 19/25] Fixup some compile fails on macOS

---
 boringtun/benches/ffi_benches/main.c            |  6 ++----
 boringtun/benches/ffi_benches/wg_bench_client.c |  5 +++--
 boringtun/benches/ffi_benches/wg_bench_client.h | 10 +++++-----
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/boringtun/benches/ffi_benches/main.c b/boringtun/benches/ffi_benches/main.c
index e93dfc1e0..a00e9632f 100644
--- a/boringtun/benches/ffi_benches/main.c
+++ b/boringtun/benches/ffi_benches/main.c
@@ -64,10 +64,8 @@ static void handle_signal(int sig) {
 }
 
 static long timespec_cmp(const struct timespec *a, const struct timespec* b) {
-    if (long i = (a->tv_sec - b->tv_sec)) {
-        return i;
-    }
-    return (a->tv_nsec - b->tv_nsec);
+    long i = (a->tv_sec - b->tv_sec);
+    return i ? i : (a->tv_nsec - b->tv_nsec);
 }
 
 static double timespec_elapsed(const struct timespec *a, const struct timespec* b) {
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.c b/boringtun/benches/ffi_benches/wg_bench_client.c
index 9262700f8..9a96529d0 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.c
+++ b/boringtun/benches/ffi_benches/wg_bench_client.c
@@ -20,6 +20,8 @@ static void wg_worker_sigmask() {
     sigemptyset(&sigset);
     sigaddset(&sigset, SIGHUP);
     pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
+
+    pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
 }
 
 struct wg_bench_iphdr {
@@ -253,10 +255,9 @@ void wg_bench_fetch_stats(const struct wg_bench_client* client, struct wg_bench_
 void wg_bench_close(struct wg_bench_client* client) {
     // Signal that we are shutting down and terminate workers.
     atomic_store(&client->worker_shutdown, true);
-    client->worker_shutdown = 1;
-    pthread_kill(client->background, SIGHUP);
     pthread_join(client->background, NULL);
     for (int i = 0; i < client->worker_count; i++) {
+        pthread_cancel(client->workers[i]);
         pthread_join(client->workers[i], NULL);
     }
 
diff --git a/boringtun/benches/ffi_benches/wg_bench_client.h b/boringtun/benches/ffi_benches/wg_bench_client.h
index 2bdb3b968..4f1d2feb2 100644
--- a/boringtun/benches/ffi_benches/wg_bench_client.h
+++ b/boringtun/benches/ffi_benches/wg_bench_client.h
@@ -24,11 +24,11 @@ struct wg_bench_client {
     struct wg_bench_client* peer;
 
     // The packet send and receive worker pool.
-    int       worker_handshake;
-    int       worker_shutdown;
-    int       worker_count;
-    pthread_t background;
-    pthread_t workers[WG_BENCH_MAX_THREADS];
+    int         worker_handshake;
+    atomic_bool worker_shutdown;
+    int         worker_count;
+    pthread_t   background;
+    pthread_t   workers[WG_BENCH_MAX_THREADS];
 
     // Statistics.
     struct wg_bench_statistics stats;

From c7188b29d802e94d59850efbd7eaabf4fe0eb524 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Thu, 21 May 2026 12:43:22 -0700
Subject: [PATCH 20/25] Implement ReceivingKeyCounterValidator with atomics

---
 boringtun/src/noise/session.rs | 189 ++++++++++++++++++---------------
 1 file changed, 101 insertions(+), 88 deletions(-)

diff --git a/boringtun/src/noise/session.rs b/boringtun/src/noise/session.rs
index d547b9819..49d2e17df 100644
--- a/boringtun/src/noise/session.rs
+++ b/boringtun/src/noise/session.rs
@@ -3,7 +3,6 @@
 
 use super::PacketData;
 use crate::noise::errors::WireGuardError;
-use parking_lot::Mutex;
 use portable_atomic::{AtomicU64, Ordering};
 use ring::aead::{Aad, LessSafeKey, Nonce, UnboundKey, CHACHA20_POLY1305};
 
@@ -13,7 +12,7 @@ pub struct Session {
     receiver: LessSafeKey,
     sender: LessSafeKey,
     sending_key_counter: AtomicU64,
-    receiving_key_counter: Mutex<ReceivingKeyCounterValidator>,
+    receiving_key_counter: ReceivingKeyCounterValidator,
 }
 
 impl std::fmt::Debug for Session {
@@ -33,61 +32,97 @@ const AEAD_SIZE: usize = 16;
 
 // Receiving buffer constants
 const WORD_SIZE: u64 = 64;
-const N_WORDS: u64 = 16; // Suffice to reorder 64*16 = 1024 packets; can be increased at will
-const N_BITS: u64 = WORD_SIZE * N_WORDS;
-
-#[derive(Debug, Clone, Default)]
+const N_WORDS: usize = 16; // Suffice to reorder 64*16 = 1024 packets; can be increased at will
+const N_BITS: u64 = WORD_SIZE * N_WORDS as u64;
+// The most significant bit used as a flag to indicate that the bitmap is being updated,
+// and acts effectively as a spinlock.
+const COUNTER_LOCK: u64 = 1u64 << (WORD_SIZE - 1);
+const COUNTER_MASK: u64 = !COUNTER_LOCK;
+
+#[derive(Debug, Default)]
 struct ReceivingKeyCounterValidator {
     /// In order to avoid replays while allowing for some reordering of the packets, we keep a
     /// bitmap of received packets, and the value of the highest counter
-    next: u64,
+    next: AtomicU64,
     /// Used to estimate packet loss
-    receive_cnt: u64,
-    bitmap: [u64; N_WORDS as usize],
+    receive_cnt: AtomicU64,
+    bitmap: [AtomicU64; N_WORDS as usize],
 }
 
 impl ReceivingKeyCounterValidator {
+    /// Returns true if bit is set, false otherwise
     #[inline(always)]
-    fn set_bit(&mut self, idx: u64) {
+    fn check_bit(&self, idx: u64) -> bool {
         let bit_idx = idx % N_BITS;
         let word = (bit_idx / WORD_SIZE) as usize;
         let bit = (bit_idx % WORD_SIZE) as usize;
-        self.bitmap[word] |= 1 << bit;
+        ((self.bitmap[word].load(Ordering::Acquire) >> bit) & 1) == 1
     }
 
+    /// Mark the packet as received, release the spinlock and return the verdict.
     #[inline(always)]
-    fn clear_bit(&mut self, idx: u64) {
+    fn mark_and_unlock(&self, idx: u64) -> Result<(), WireGuardError> {
         let bit_idx = idx % N_BITS;
         let word = (bit_idx / WORD_SIZE) as usize;
         let bit = (bit_idx % WORD_SIZE) as usize;
-        self.bitmap[word] &= !(1u64 << bit);
+        let previous = self.bitmap[word].fetch_or(1 << bit, Ordering::SeqCst);
+        self.next.fetch_and(COUNTER_MASK, Ordering::SeqCst);
+        if (previous >> bit) & 1 == 1 {
+            Err(WireGuardError::DuplicateCounter)
+        } else {
+            Ok(())
+        } 
     }
 
-    /// Clear the word that contains idx
+    /// Clear all the packets between prev and next.
     #[inline(always)]
-    fn clear_word(&mut self, idx: u64) {
-        let bit_idx = idx % N_BITS;
-        let word = (bit_idx / WORD_SIZE) as usize;
-        self.bitmap[word] = 0;
-    }
+    fn clear_range(&self, prev: u64, next: u64) {
+        if next <= prev {
+            return;
+        }
 
-    /// Returns true if bit is set, false otherwise
-    #[inline(always)]
-    fn check_bit(&self, idx: u64) -> bool {
-        let bit_idx = idx % N_BITS;
-        let word = (bit_idx / WORD_SIZE) as usize;
-        let bit = (bit_idx % WORD_SIZE) as usize;
-        ((self.bitmap[word] >> bit) & 1) == 1
+        if next - prev >= N_BITS {
+            // Too far ahead, clear all the bits
+            for i in 0..N_WORDS {
+                self.bitmap[i].store(0, Ordering::SeqCst);
+            }
+            return;
+        }
+
+        let prev_idx = (prev / WORD_SIZE) as usize;
+        let prev_bit = 1u64 << (prev % WORD_SIZE);
+        let next_idx = (next / WORD_SIZE) as usize;
+        let next_bit = 1u64 << (next % WORD_SIZE);
+        if next_idx == prev_idx {
+            // The bits to clear all fit within a single word.
+            let mask = !(next_bit - prev_bit);
+            self.bitmap[prev_idx % N_WORDS].fetch_and(mask, Ordering::SeqCst);
+        } else {
+            // The bits to clear span multiple words.
+            let mut mask: u64 = prev_bit - 1;
+            for i in prev_idx..next_idx-1 {
+                self.bitmap[i % N_WORDS].fetch_and(mask, Ordering::SeqCst);
+                mask = 0;
+            }
+            self.bitmap[next_idx % N_WORDS].fetch_and(!(next_bit - 1), Ordering::SeqCst);
+        }
     }
 
     /// Returns true if the counter was not yet received, and is not too far back
+    /// This check is lock-free, but it can return a false positive in case a
+    /// race condition occurs.
     #[inline(always)]
     fn will_accept(&self, counter: u64) -> Result<(), WireGuardError> {
-        if counter >= self.next {
+        if counter >= COUNTER_MASK {
+            // Too many packets, counter would overflow.
+            return Err(WireGuardError::InvalidCounter);
+        }
+        let next = self.next.load(Ordering::Acquire) & COUNTER_MASK;
+        if counter >= next {
             // As long as the counter is growing no replay took place for sure
             return Ok(());
         }
-        if counter + N_BITS < self.next {
+        if counter + N_BITS < next {
             // Drop if too far back
             return Err(WireGuardError::InvalidCounter);
         }
@@ -101,52 +136,45 @@ impl ReceivingKeyCounterValidator {
     /// Marks the counter as received, and returns true if it is still good (in case during
     /// decryption something changed)
     #[inline(always)]
-    fn mark_did_receive(&mut self, counter: u64) -> Result<(), WireGuardError> {
-        if counter + N_BITS < self.next {
-            // Drop if too far back
+    fn mark_did_receive(&self, counter: u64) -> Result<(), WireGuardError> {
+        if counter >= COUNTER_MASK {
+            // Too many packets, counter would overflow.
             return Err(WireGuardError::InvalidCounter);
         }
-        if counter == self.next {
-            // Usually the packets arrive in order, in that case we simply mark the bit and
-            // increment the counter
-            self.set_bit(counter);
-            self.next += 1;
-            return Ok(());
-        }
-        if counter < self.next {
-            // A packet arrived out of order, check if it is valid, and mark
-            if self.check_bit(counter) {
+
+        let mut prev = self.next.load(Ordering::Acquire);
+        loop {
+            if (counter + N_BITS) < (prev & COUNTER_MASK) {
+                // Drop if too far back that the packet would fall outside the bitmask.
                 return Err(WireGuardError::InvalidCounter);
             }
-            self.set_bit(counter);
-            return Ok(());
-        }
-        // Packets where dropped, or maybe reordered, skip them and mark unused
-        if counter - self.next >= N_BITS {
-            // Too far ahead, clear all the bits
-            for c in self.bitmap.iter_mut() {
-                *c = 0;
-            }
-        } else {
-            let mut i = self.next;
-            while i % WORD_SIZE != 0 && i < counter {
-                // Clear until i aligned to word size
-                self.clear_bit(i);
-                i += 1;
+            if prev & COUNTER_LOCK == COUNTER_LOCK {
+                // Someone else has the spinlock. Try again.
+                prev = self.next.load(Ordering::Acquire);
+                continue;
             }
-            while i + WORD_SIZE < counter {
-                // Clear whole word at a time
-                self.clear_word(i);
-                i = (i + WORD_SIZE) & 0u64.wrapping_sub(WORD_SIZE);
-            }
-            while i < counter {
-                // Clear any remaining bits
-                self.clear_bit(i);
-                i += 1;
+            if counter < prev {
+                // This packet arrived out of order, just acquire the spinlock.
+                match self.next.compare_exchange_weak(prev, prev | COUNTER_LOCK, Ordering::SeqCst, Ordering::Relaxed) {
+                    Ok(_) => break,
+                    Err(x) => prev = x,
+                }
+            } else {
+                // This packet arrived in-order.
+                // Acquire the spinlock, update the next packet counter, and clear bits that wrapped over.
+                match self.next.compare_exchange_weak(prev, (counter+1) | COUNTER_LOCK, Ordering::SeqCst, Ordering::Relaxed) {
+                    Ok(_) => {
+                        self.clear_range(prev, counter);
+                        break;
+                    },
+                    Err(x) => prev = x,
+                }
             }
         }
-        self.set_bit(counter);
-        self.next = counter + 1;
+
+        // And finally try to mark the packet, release the spinlock, and return the verdict.
+        self.mark_and_unlock(counter)?;
+        self.receive_cnt.fetch_add(1, Ordering::Relaxed);
         Ok(())
     }
 }
@@ -166,7 +194,7 @@ impl Session {
             ),
             sender: LessSafeKey::new(UnboundKey::new(&CHACHA20_POLY1305, &sending_key).unwrap()),
             sending_key_counter: AtomicU64::new(0),
-            receiving_key_counter: Mutex::new(Default::default()),
+            receiving_key_counter: Default::default(),
         }
     }
 
@@ -174,22 +202,6 @@ impl Session {
         self.receiving_index as usize
     }
 
-    /// Returns true if receiving counter is good to use
-    fn receiving_counter_quick_check(&self, counter: u64) -> Result<(), WireGuardError> {
-        let counter_validator = self.receiving_key_counter.lock();
-        counter_validator.will_accept(counter)
-    }
-
-    /// Returns true if receiving counter is good to use, and marks it as used {
-    fn receiving_counter_mark(&self, counter: u64) -> Result<(), WireGuardError> {
-        let mut counter_validator = self.receiving_key_counter.lock();
-        let ret = counter_validator.mark_did_receive(counter);
-        if ret.is_ok() {
-            counter_validator.receive_cnt += 1;
-        }
-        ret
-    }
-
     /// src - an IP packet from the interface
     /// dst - pre-allocated space to hold the encapsulating UDP packet to send over the network
     /// returns the size of the formatted packet
@@ -247,7 +259,7 @@ impl Session {
             return Err(WireGuardError::WrongIndex);
         }
         // Don't reuse counters, in case this is a replay attack we want to quickly check the counter without running expensive decryption
-        self.receiving_counter_quick_check(packet.counter)?;
+        self.receiving_key_counter.will_accept(packet.counter)?;
 
         let ret = {
             let mut nonce = [0u8; 12];
@@ -263,14 +275,15 @@ impl Session {
         };
 
         // After decryption is done, check counter again, and mark as received
-        self.receiving_counter_mark(packet.counter)?;
+        self.receiving_key_counter.mark_did_receive(packet.counter)?;
         Ok(ret)
     }
 
     /// Returns the estimated downstream packet loss for this session
     pub(super) fn current_packet_cnt(&self) -> (u64, u64) {
-        let counter_validator = self.receiving_key_counter.lock();
-        (counter_validator.next, counter_validator.receive_cnt)
+        let next = self.receiving_key_counter.next.load(Ordering::Relaxed) & COUNTER_MASK;
+        let rx = self.receiving_key_counter.receive_cnt.load(Ordering::Relaxed);
+        (next, rx)
     }
 }
 

From 2a2ecf4c5d70ec741c12b9cc2a883b2cbbd7d1c2 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Thu, 21 May 2026 20:15:21 -0700
Subject: [PATCH 21/25] Lets racecheck this thing with some unit tests

---
 boringtun/src/noise/session.rs | 169 +++++++++++++++++++++++++++++----
 1 file changed, 149 insertions(+), 20 deletions(-)

diff --git a/boringtun/src/noise/session.rs b/boringtun/src/noise/session.rs
index 49d2e17df..a5ded0647 100644
--- a/boringtun/src/noise/session.rs
+++ b/boringtun/src/noise/session.rs
@@ -50,6 +50,14 @@ struct ReceivingKeyCounterValidator {
 }
 
 impl ReceivingKeyCounterValidator {
+    pub const fn new() -> ReceivingKeyCounterValidator {
+        ReceivingKeyCounterValidator {
+            next: AtomicU64::new(0),
+            receive_cnt: AtomicU64::new(0),
+            bitmap: [ const { AtomicU64::new(0) }; N_WORDS as usize],
+        }
+    }
+
     /// Returns true if bit is set, false otherwise
     #[inline(always)]
     fn check_bit(&self, idx: u64) -> bool {
@@ -77,7 +85,7 @@ impl ReceivingKeyCounterValidator {
     /// Clear all the packets between prev and next.
     #[inline(always)]
     fn clear_range(&self, prev: u64, next: u64) {
-        if next <= prev {
+        if next < prev {
             return;
         }
 
@@ -90,21 +98,21 @@ impl ReceivingKeyCounterValidator {
         }
 
         let prev_idx = (prev / WORD_SIZE) as usize;
-        let prev_bit = 1u64 << (prev % WORD_SIZE);
+        let prev_mask = (1u64 << (prev % WORD_SIZE)) - 1;
+
         let next_idx = (next / WORD_SIZE) as usize;
-        let next_bit = 1u64 << (next % WORD_SIZE);
+        let next_mask = (u64::MAX - 1) << (next % WORD_SIZE);
+
         if next_idx == prev_idx {
             // The bits to clear all fit within a single word.
-            let mask = !(next_bit - prev_bit);
-            self.bitmap[prev_idx % N_WORDS].fetch_and(mask, Ordering::SeqCst);
+            self.bitmap[prev_idx % N_WORDS].fetch_and(next_mask | prev_mask, Ordering::SeqCst);
         } else {
             // The bits to clear span multiple words.
-            let mut mask: u64 = prev_bit - 1;
-            for i in prev_idx..next_idx-1 {
-                self.bitmap[i % N_WORDS].fetch_and(mask, Ordering::SeqCst);
-                mask = 0;
+            self.bitmap[prev_idx % N_WORDS].fetch_and(prev_mask, Ordering::SeqCst);
+            for i in prev_idx+1..next_idx {
+                self.bitmap[i % N_WORDS].store(0, Ordering::SeqCst);
             }
-            self.bitmap[next_idx % N_WORDS].fetch_and(!(next_bit - 1), Ordering::SeqCst);
+            self.bitmap[next_idx % N_WORDS].fetch_and(next_mask, Ordering::SeqCst);
         }
     }
 
@@ -117,7 +125,10 @@ impl ReceivingKeyCounterValidator {
             // Too many packets, counter would overflow.
             return Err(WireGuardError::InvalidCounter);
         }
-        let next = self.next.load(Ordering::Acquire) & COUNTER_MASK;
+        let mut next = self.next.load(Ordering::SeqCst);
+        while (next & COUNTER_LOCK) != 0 {
+            next = self.next.load(Ordering::SeqCst);
+        }
         if counter >= next {
             // As long as the counter is growing no replay took place for sure
             return Ok(());
@@ -145,24 +156,24 @@ impl ReceivingKeyCounterValidator {
         let mut prev = self.next.load(Ordering::Acquire);
         loop {
             if (counter + N_BITS) < (prev & COUNTER_MASK) {
-                // Drop if too far back that the packet would fall outside the bitmask.
+                // Drop if too far back that the packet would fall outside the bitmap.
                 return Err(WireGuardError::InvalidCounter);
             }
             if prev & COUNTER_LOCK == COUNTER_LOCK {
                 // Someone else has the spinlock. Try again.
-                prev = self.next.load(Ordering::Acquire);
+                prev = self.next.load(Ordering::SeqCst);
                 continue;
             }
             if counter < prev {
-                // This packet arrived out of order, just acquire the spinlock.
-                match self.next.compare_exchange_weak(prev, prev | COUNTER_LOCK, Ordering::SeqCst, Ordering::Relaxed) {
+                // This packet arrived out of order, acquire the spinlock to mark the packet.
+                match self.next.compare_exchange_weak(prev, prev | COUNTER_LOCK, Ordering::SeqCst, Ordering::Acquire) {
                     Ok(_) => break,
                     Err(x) => prev = x,
                 }
             } else {
                 // This packet arrived in-order.
-                // Acquire the spinlock, update the next packet counter, and clear bits that wrapped over.
-                match self.next.compare_exchange_weak(prev, (counter+1) | COUNTER_LOCK, Ordering::SeqCst, Ordering::Relaxed) {
+                // Acquire the spinlock, update the next packet counter, and clear bits that will wrap over.
+                match self.next.compare_exchange_weak(prev, (counter+1) | COUNTER_LOCK, Ordering::SeqCst, Ordering::Acquire) {
                     Ok(_) => {
                         self.clear_range(prev, counter);
                         break;
@@ -287,12 +298,66 @@ impl Session {
     }
 }
 
+#[cfg(test)]
+use std::thread;
+
 #[cfg(test)]
 mod tests {
     use super::*;
+
+    #[cfg(test)]
+    fn check_replay_clear_range(start: u64, end: u64) {
+        // Setup the replay bitmap with all bits set.
+        let c: ReceivingKeyCounterValidator = Default::default();
+        for i in 0..N_WORDS {
+            c.bitmap[i].store(!0, Ordering::Release);
+        }
+        c.next.store(N_BITS, Ordering::Release);
+        for i in 0..N_BITS {
+            assert!(c.check_bit(i));
+        }
+
+        // Clear a range, and recheck the bitmap.
+        c.clear_range(start, end);
+        let check_start = (start / WORD_SIZE) * WORD_SIZE;
+        for i in check_start..check_start+N_BITS {
+            if i < start || i > end {
+                assert!(c.check_bit(i), "expected bit {} to be set", i);
+            } else {
+                assert!(!c.check_bit(i), "expected bit {} to be clear", i);
+            }
+        }
+    }
+
+    #[test]
+    fn test_replay_clear_range() {
+        // Clear a single bit and check edge cases.
+        check_replay_clear_range(0, 0);
+        check_replay_clear_range(42, 42);
+        check_replay_clear_range(WORD_SIZE-1, WORD_SIZE-1);
+        check_replay_clear_range(WORD_SIZE, WORD_SIZE);
+        check_replay_clear_range(N_BITS-1, N_BITS-1);
+        check_replay_clear_range(N_BITS, N_BITS);
+
+        // Clear some bits in the middle of a single word.
+        check_replay_clear_range(13, 19);
+
+        // Clear precisely one word.
+        check_replay_clear_range(WORD_SIZE, WORD_SIZE * 2 - 1);
+
+        // Clear some bits spanning two words.
+        check_replay_clear_range(WORD_SIZE + 7, WORD_SIZE * 2 + 13);
+
+        // Clear some bits that span many words.
+        check_replay_clear_range(WORD_SIZE + 7, N_BITS - 7);
+
+        // Clear some bits that wrap around to the start of the index.
+        check_replay_clear_range(N_BITS - 7, N_BITS + 7);
+    }
+
     #[test]
     fn test_replay_counter() {
-        let mut c: ReceivingKeyCounterValidator = Default::default();
+        let c: ReceivingKeyCounterValidator = Default::default();
 
         assert!(c.mark_did_receive(0).is_ok());
         assert!(c.mark_did_receive(0).is_err());
@@ -304,8 +369,8 @@ mod tests {
         assert!(c.mark_did_receive(15).is_err());
 
         for i in 64..N_BITS + 128 {
-            assert!(c.mark_did_receive(i).is_ok());
-            assert!(c.mark_did_receive(i).is_err());
+            assert!(c.mark_did_receive(i).is_ok(), "unexpected mark failed for bit {}", i);
+            assert!(c.mark_did_receive(i).is_err(), "duplicate packet not caught for bit {}", i);
         }
 
         assert!(c.mark_did_receive(N_BITS * 3).is_ok());
@@ -339,4 +404,68 @@ mod tests {
         assert!(c.mark_did_receive(N_BITS * 3 + 71).is_err());
         assert!(c.mark_did_receive(N_BITS * 3 + 72).is_err());
     }
+
+    const RACE_MAX_PACKETS: u64 = 1024 * 1024;
+
+    // Race check worker to try and send valid packets, they should be accepted.
+    #[cfg(test)]
+    fn racecheck_counter_worker(counter: &AtomicU64, validator: &ReceivingKeyCounterValidator) {
+        loop {
+            let value = counter.fetch_add(1, Ordering::Relaxed);
+            if value > RACE_MAX_PACKETS {
+                break;
+            }
+
+            // If we are spinning the CPU hard enough, it is possible to get
+            // invalid counters when threads stall long enough for their counter
+            // to grow too old for the bitmap.
+            match validator.will_accept(value) {
+                Ok(_) => {},
+                Err(WireGuardError::InvalidCounter) => continue,
+                Err(WireGuardError::DuplicateCounter) => panic!("duplicate while marking {}", value),
+                _ => panic!("error while marking packet {}", value),
+            };
+
+            match validator.mark_did_receive(value) {
+                Ok(_) => {},
+                Err(WireGuardError::InvalidCounter) => continue,
+                Err(WireGuardError::DuplicateCounter) => panic!("duplicate while marking {}", value),
+                _ => panic!("error while marking {}", value),
+            };
+
+            // Resend it as a duplicate, it must be rejected.
+            assert!(validator.mark_did_receive(value).is_err(),
+                    "race encountered while checking duplicate {}", value);
+            
+            thread::yield_now();
+        }
+    }
+
+    // Race check worker to try and send duplicate packets, they must all be rejected.
+    #[cfg(test)]
+    fn racecheck_dup_worker(counter: &AtomicU64, validator: &ReceivingKeyCounterValidator) {
+        while counter.load(Ordering::Relaxed) < RACE_MAX_PACKETS {
+            let value = validator.next.load(Ordering::Relaxed) & COUNTER_MASK;
+            if value > 0 {
+                assert!(validator.mark_did_receive(value - 1).is_err());
+            }
+        }
+    }
+
+    #[test]
+    fn test_replay_racecheck() {
+        static COUNTER: AtomicU64 = AtomicU64::new(0);
+        static VALIDATOR: ReceivingKeyCounterValidator = ReceivingKeyCounterValidator::new();
+        let mut threads = Vec::new();
+        let num_threads = thread::available_parallelism().map_or(8, |x| x.get());
+
+        for _ in 0..num_threads-1 {
+            threads.push(thread::spawn(|| { racecheck_counter_worker(&COUNTER, &VALIDATOR) }));
+        }
+        threads.push(thread::spawn(|| { racecheck_dup_worker(&COUNTER, &VALIDATOR) }));
+
+        for handle in threads {
+            handle.join().unwrap();
+        }
+    }
 }

From 9acf9db6b3652691ebd89b13a16e3b894911b68f Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Fri, 22 May 2026 09:56:51 -0700
Subject: [PATCH 22/25] A bit more tweaking, add fallback to AtomicU32 too

---
 boringtun/src/noise/session.rs | 70 ++++++++++++++++++++++------------
 1 file changed, 46 insertions(+), 24 deletions(-)

diff --git a/boringtun/src/noise/session.rs b/boringtun/src/noise/session.rs
index a5ded0647..f54912140 100644
--- a/boringtun/src/noise/session.rs
+++ b/boringtun/src/noise/session.rs
@@ -5,6 +5,7 @@ use super::PacketData;
 use crate::noise::errors::WireGuardError;
 use portable_atomic::{AtomicU64, Ordering};
 use ring::aead::{Aad, LessSafeKey, Nonce, UnboundKey, CHACHA20_POLY1305};
+use std::hint;
 
 pub struct Session {
     pub(crate) receiving_index: u32,
@@ -30,13 +31,19 @@ const DATA_OFFSET: usize = 16;
 /// The overhead of the AEAD
 const AEAD_SIZE: usize = 16;
 
+#[cfg(target_has_atomic="64")]
+type CounterBitmap = AtomicU64;
+#[cfg(not(target_has_atomic="64"))]
+use portable_atomic::AtomicU32 as CounterBitmap;
+
 // Receiving buffer constants
-const WORD_SIZE: u64 = 64;
-const N_WORDS: usize = 16; // Suffice to reorder 64*16 = 1024 packets; can be increased at will
-const N_BITS: u64 = WORD_SIZE * N_WORDS as u64;
+const WORD_SIZE: u64 = if cfg!(target_has_atomic="64") { 64 } else { 32 };
+const N_BITS: u64 = 1024;
+const N_WORDS: usize = (N_BITS / WORD_SIZE) as usize;
+
 // The most significant bit used as a flag to indicate that the bitmap is being updated,
 // and acts effectively as a spinlock.
-const COUNTER_LOCK: u64 = 1u64 << (WORD_SIZE - 1);
+const COUNTER_LOCK: u64 = 1u64 << 63;
 const COUNTER_MASK: u64 = !COUNTER_LOCK;
 
 #[derive(Debug, Default)]
@@ -46,7 +53,7 @@ struct ReceivingKeyCounterValidator {
     next: AtomicU64,
     /// Used to estimate packet loss
     receive_cnt: AtomicU64,
-    bitmap: [AtomicU64; N_WORDS as usize],
+    bitmap: [ CounterBitmap; N_WORDS as usize],
 }
 
 impl ReceivingKeyCounterValidator {
@@ -54,16 +61,21 @@ impl ReceivingKeyCounterValidator {
         ReceivingKeyCounterValidator {
             next: AtomicU64::new(0),
             receive_cnt: AtomicU64::new(0),
-            bitmap: [ const { AtomicU64::new(0) }; N_WORDS as usize],
+            bitmap: [ const { CounterBitmap::new(0) }; N_WORDS],
         }
     }
 
+    #[inline(always)]
+    fn get_next(&self) -> u64 {
+        self.next.load(Ordering::Acquire) & COUNTER_MASK
+    }
+
     /// Returns true if bit is set, false otherwise
     #[inline(always)]
     fn check_bit(&self, idx: u64) -> bool {
         let bit_idx = idx % N_BITS;
         let word = (bit_idx / WORD_SIZE) as usize;
-        let bit = (bit_idx % WORD_SIZE) as usize;
+        let bit = bit_idx % WORD_SIZE;
         ((self.bitmap[word].load(Ordering::Acquire) >> bit) & 1) == 1
     }
 
@@ -72,7 +84,7 @@ impl ReceivingKeyCounterValidator {
     fn mark_and_unlock(&self, idx: u64) -> Result<(), WireGuardError> {
         let bit_idx = idx % N_BITS;
         let word = (bit_idx / WORD_SIZE) as usize;
-        let bit = (bit_idx % WORD_SIZE) as usize;
+        let bit = bit_idx % WORD_SIZE;
         let previous = self.bitmap[word].fetch_or(1 << bit, Ordering::SeqCst);
         self.next.fetch_and(COUNTER_MASK, Ordering::SeqCst);
         if (previous >> bit) & 1 == 1 {
@@ -97,11 +109,15 @@ impl ReceivingKeyCounterValidator {
             return;
         }
 
-        let prev_idx = (prev / WORD_SIZE) as usize;
-        let prev_mask = (1u64 << (prev % WORD_SIZE)) - 1;
+        #[cfg(target_has_atomic="64")]
+        const ONE: u64 = 1;
+        #[cfg(not(target_has_atomic="32"))]
+        const ONE: u32 = 1;
 
+        let prev_idx = (prev / WORD_SIZE) as usize;
+        let prev_mask = (ONE << (prev % WORD_SIZE)) - 1;
         let next_idx = (next / WORD_SIZE) as usize;
-        let next_mask = (u64::MAX - 1) << (next % WORD_SIZE);
+        let next_mask = (!ONE) << (next % WORD_SIZE);
 
         if next_idx == prev_idx {
             // The bits to clear all fit within a single word.
@@ -128,6 +144,7 @@ impl ReceivingKeyCounterValidator {
         let mut next = self.next.load(Ordering::SeqCst);
         while (next & COUNTER_LOCK) != 0 {
             next = self.next.load(Ordering::SeqCst);
+            hint::spin_loop();
         }
         if counter >= next {
             // As long as the counter is growing no replay took place for sure
@@ -162,9 +179,7 @@ impl ReceivingKeyCounterValidator {
             if prev & COUNTER_LOCK == COUNTER_LOCK {
                 // Someone else has the spinlock. Try again.
                 prev = self.next.load(Ordering::SeqCst);
-                continue;
-            }
-            if counter < prev {
+            } else if counter < prev {
                 // This packet arrived out of order, acquire the spinlock to mark the packet.
                 match self.next.compare_exchange_weak(prev, prev | COUNTER_LOCK, Ordering::SeqCst, Ordering::Acquire) {
                     Ok(_) => break,
@@ -181,6 +196,7 @@ impl ReceivingKeyCounterValidator {
                     Err(x) => prev = x,
                 }
             }
+            hint::spin_loop();
         }
 
         // And finally try to mark the packet, release the spinlock, and return the verdict.
@@ -292,9 +308,8 @@ impl Session {
 
     /// Returns the estimated downstream packet loss for this session
     pub(super) fn current_packet_cnt(&self) -> (u64, u64) {
-        let next = self.receiving_key_counter.next.load(Ordering::Relaxed) & COUNTER_MASK;
         let rx = self.receiving_key_counter.receive_cnt.load(Ordering::Relaxed);
-        (next, rx)
+        (self.receiving_key_counter.get_next(), rx)
     }
 }
 
@@ -416,19 +431,26 @@ mod tests {
                 break;
             }
 
-            // If we are spinning the CPU hard enough, it is possible to get
-            // invalid counters when threads stall long enough for their counter
-            // to grow too old for the bitmap.
             match validator.will_accept(value) {
                 Ok(_) => {},
-                Err(WireGuardError::InvalidCounter) => continue,
-                Err(WireGuardError::DuplicateCounter) => panic!("duplicate while marking {}", value),
-                _ => panic!("error while marking packet {}", value),
+                Err(WireGuardError::InvalidCounter) => {
+                    // This error is allowed if, and only if, the thread hit an
+                    // unlucky interrupt and the counter is too old now.
+                    assert!(validator.get_next() >= value + N_BITS);
+                    continue;
+                },
+                Err(WireGuardError::DuplicateCounter) => panic!("duplicate while checking {}", value),
+                _ => panic!("error while checking packet {}", value),
             };
 
             match validator.mark_did_receive(value) {
                 Ok(_) => {},
-                Err(WireGuardError::InvalidCounter) => continue,
+                Err(WireGuardError::InvalidCounter) => {
+                    // This error is allowed if, and only if, the thread hit an
+                    // unlucky interrupt and the counter is too old now.
+                    assert!(validator.get_next() >= value + N_BITS);
+                    continue;
+                },
                 Err(WireGuardError::DuplicateCounter) => panic!("duplicate while marking {}", value),
                 _ => panic!("error while marking {}", value),
             };
@@ -445,7 +467,7 @@ mod tests {
     #[cfg(test)]
     fn racecheck_dup_worker(counter: &AtomicU64, validator: &ReceivingKeyCounterValidator) {
         while counter.load(Ordering::Relaxed) < RACE_MAX_PACKETS {
-            let value = validator.next.load(Ordering::Relaxed) & COUNTER_MASK;
+            let value = validator.get_next();
             if value > 0 {
                 assert!(validator.mark_did_receive(value - 1).is_err());
             }

From 9fe32a0394eb360d37d03cc00a32ab37d393da38 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Tue, 26 May 2026 09:10:55 -0700
Subject: [PATCH 23/25] Fix race in will_accept()

I think there's another race hiding in will_accept that goes something
like this:

A: Receives packet N
A: Checks counter is good
A: Interrupted
B: Receives packet N+N_BITS
B: Checks counter is good
B: Marks bitmap as received
A: Resumes and find bitmap marked by N+N_BITS

This is a pretty minor race as the packet would be rejected anyways
but we return WireGuardError::DuplicateCounter instead of
WireGuardError::InvalidCounter and that causes the unit test to fail.
---
 boringtun/src/noise/session.rs | 59 ++++++++++++++++++++++------------
 1 file changed, 39 insertions(+), 20 deletions(-)

diff --git a/boringtun/src/noise/session.rs b/boringtun/src/noise/session.rs
index f54912140..c6410b746 100644
--- a/boringtun/src/noise/session.rs
+++ b/boringtun/src/noise/session.rs
@@ -53,7 +53,7 @@ struct ReceivingKeyCounterValidator {
     next: AtomicU64,
     /// Used to estimate packet loss
     receive_cnt: AtomicU64,
-    bitmap: [ CounterBitmap; N_WORDS as usize],
+    bitmap: [ CounterBitmap; N_WORDS],
 }
 
 impl ReceivingKeyCounterValidator {
@@ -67,7 +67,7 @@ impl ReceivingKeyCounterValidator {
 
     #[inline(always)]
     fn get_next(&self) -> u64 {
-        self.next.load(Ordering::Acquire) & COUNTER_MASK
+        self.next.load(Ordering::SeqCst) & COUNTER_MASK
     }
 
     /// Returns true if bit is set, false otherwise
@@ -76,7 +76,7 @@ impl ReceivingKeyCounterValidator {
         let bit_idx = idx % N_BITS;
         let word = (bit_idx / WORD_SIZE) as usize;
         let bit = bit_idx % WORD_SIZE;
-        ((self.bitmap[word].load(Ordering::Acquire) >> bit) & 1) == 1
+        ((self.bitmap[word].load(Ordering::SeqCst) >> bit) & 1) == 1
     }
 
     /// Mark the packet as received, release the spinlock and return the verdict.
@@ -111,7 +111,7 @@ impl ReceivingKeyCounterValidator {
 
         #[cfg(target_has_atomic="64")]
         const ONE: u64 = 1;
-        #[cfg(not(target_has_atomic="32"))]
+        #[cfg(not(target_has_atomic="64"))]
         const ONE: u32 = 1;
 
         let prev_idx = (prev / WORD_SIZE) as usize;
@@ -132,32 +132,43 @@ impl ReceivingKeyCounterValidator {
         }
     }
 
-    /// Returns true if the counter was not yet received, and is not too far back
-    /// This check is lock-free, but it can return a false positive in case a
-    /// race condition occurs.
+    /// Returns true if the counter was not yet received, and is not too far back.
     #[inline(always)]
     fn will_accept(&self, counter: u64) -> Result<(), WireGuardError> {
         if counter >= COUNTER_MASK {
             // Too many packets, counter would overflow.
             return Err(WireGuardError::InvalidCounter);
         }
+
+        // Spin while checking the counter until the bitmap is updated, as
+        // indicated by the COUNTER_LOCK bit being cleared.
         let mut next = self.next.load(Ordering::SeqCst);
-        while (next & COUNTER_LOCK) != 0 {
+        loop {
+            if counter >= (next & COUNTER_MASK) {
+                // As long as the counter is growing no replay took place for sure
+                return Ok(());
+            }
+            if counter + N_BITS < (next & COUNTER_MASK) {
+                // Drop if too far back
+                return Err(WireGuardError::InvalidCounter);
+            }
+            if (next & COUNTER_LOCK) == 0 {
+                break;
+            }
             next = self.next.load(Ordering::SeqCst);
             hint::spin_loop();
         }
-        if counter >= next {
-            // As long as the counter is growing no replay took place for sure
-            return Ok(());
-        }
-        if counter + N_BITS < next {
-            // Drop if too far back
-            return Err(WireGuardError::InvalidCounter);
-        }
-        if !self.check_bit(counter) {
-            Ok(())
-        } else {
+
+        // Check the bitmap for duplicates, then re-check the counter
+        // one last time in case a race conditioned occurred.
+        let duplicate = self.check_bit(counter);
+        let next = self.next.load(Ordering::SeqCst) & COUNTER_MASK;
+        return if counter + N_BITS < next {
+            Err(WireGuardError::InvalidCounter)
+        } else if duplicate {
             Err(WireGuardError::DuplicateCounter)
+        } else {
+            Ok(())
         }
     }
 
@@ -426,11 +437,19 @@ mod tests {
     #[cfg(test)]
     fn racecheck_counter_worker(counter: &AtomicU64, validator: &ReceivingKeyCounterValidator) {
         loop {
-            let value = counter.fetch_add(1, Ordering::Relaxed);
+            let mut value = counter.fetch_add(1, Ordering::Relaxed);
             if value > RACE_MAX_PACKETS {
                 break;
             }
 
+            // To drive the out-of-order packet handling a little harder, simulate some packet
+            // reordering. If the packet is a multiple of 29, increment the value being sent by
+            // 29. This should still never generate a duplicate but forces the validator to
+            // interact with the bitmap to figure it out.
+            if value % 29 == 0 {
+                value += 29;
+            }
+
             match validator.will_accept(value) {
                 Ok(_) => {},
                 Err(WireGuardError::InvalidCounter) => {

From f2daa33a193656ca07855541e957d81a7cfc29e2 Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Tue, 26 May 2026 21:19:04 -0700
Subject: [PATCH 24/25] Run cargo fmt

---
 boringtun/src/noise/mod.rs          | 30 +++++++---
 boringtun/src/noise/rate_limiter.rs | 11 +++-
 boringtun/src/noise/session.rs      | 93 +++++++++++++++++++----------
 3 files changed, 92 insertions(+), 42 deletions(-)

diff --git a/boringtun/src/noise/mod.rs b/boringtun/src/noise/mod.rs
index ee9078a21..1c0767c66 100644
--- a/boringtun/src/noise/mod.rs
+++ b/boringtun/src/noise/mod.rs
@@ -14,11 +14,11 @@ use crate::noise::rate_limiter::RateLimiter;
 use crate::noise::timers::{TimerName, Timers};
 use crate::x25519;
 
+use portable_atomic::{AtomicUsize, Ordering};
 use std::collections::VecDeque;
 use std::convert::{TryFrom, TryInto};
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 use std::sync::Arc;
-use portable_atomic::{AtomicUsize, Ordering};
 use std::time::Duration;
 
 /// The default value to use for rate limiting, when no other rate limiter is defined
@@ -265,7 +265,7 @@ impl Tunn {
         // If there is no session, queue the packet for future retry
         self.queue_packet(src);
         // Initiate a new handshake if none is in progress
-        return self.format_handshake_initiation(dst, false);
+        self.format_handshake_initiation(dst, false)
     }
 
     /// Encapsulate a single packet from the tunnel interface.
@@ -343,18 +343,27 @@ impl Tunn {
     ) -> Option<TunnResult<'a>> {
         // Packet dequeue operations require a write lock.
         if datagram.is_empty() {
-            return if self.packet_queue.is_empty() { Some(TunnResult::Done) } else { None };
+            return if self.packet_queue.is_empty() {
+                Some(TunnResult::Done)
+            } else {
+                None
+            };
         }
 
         let mut cookie = [0u8; COOKIE_REPLY_SZ];
-        match self.rate_limiter.verify_packet(src_addr, datagram, &mut cookie) {
-            Ok(Packet::PacketData(p)) => Some(self.handle_data(p, dst).unwrap_or_else(TunnResult::from)),
+        match self
+            .rate_limiter
+            .verify_packet(src_addr, datagram, &mut cookie)
+        {
+            Ok(Packet::PacketData(p)) => {
+                Some(self.handle_data(p, dst).unwrap_or_else(TunnResult::from))
+            }
             Err(TunnResult::WriteToNetwork(cookie)) => {
                 dst[..cookie.len()].copy_from_slice(cookie);
                 return Some(TunnResult::WriteToNetwork(&mut dst[..cookie.len()]));
             }
             Err(TunnResult::Err(e)) => return Some(TunnResult::Err(e)),
-            _ => None
+            _ => None,
         }
     }
 
@@ -450,12 +459,17 @@ impl Tunn {
             // There is nothing to do, already using this session, this is the common case
             return;
         }
-        
+
         if self.sessions[cur_idx % N_SESSIONS].is_none()
             || self.timers.session_timers[new_idx % N_SESSIONS]
                 >= self.timers.session_timers[cur_idx % N_SESSIONS]
         {
-            if let Ok(idx) = self.current.compare_exchange(cur_idx, new_idx, Ordering::Acquire, Ordering::Relaxed) {
+            if let Ok(idx) = self.current.compare_exchange(
+                cur_idx,
+                new_idx,
+                Ordering::Acquire,
+                Ordering::Relaxed,
+            ) {
                 tracing::debug!(message = "New session", session = idx);
             }
         }
diff --git a/boringtun/src/noise/rate_limiter.rs b/boringtun/src/noise/rate_limiter.rs
index a04b1b283..fcecc7a9c 100644
--- a/boringtun/src/noise/rate_limiter.rs
+++ b/boringtun/src/noise/rate_limiter.rs
@@ -82,7 +82,16 @@ impl RateLimiter {
         let last_msec = self.last_reset.load(Ordering::Acquire);
         let last_reset = Duration::from_millis(last_msec);
         if now - last_reset >= RESET_PERIOD {
-            if self.last_reset.compare_exchange(last_msec, now.as_millis() as u64, Ordering::SeqCst, Ordering::Relaxed).is_ok() {
+            if self
+                .last_reset
+                .compare_exchange(
+                    last_msec,
+                    now.as_millis() as u64,
+                    Ordering::SeqCst,
+                    Ordering::Relaxed,
+                )
+                .is_ok()
+            {
                 self.count.store(0, Ordering::SeqCst);
             }
         }
diff --git a/boringtun/src/noise/session.rs b/boringtun/src/noise/session.rs
index c6410b746..18299811f 100644
--- a/boringtun/src/noise/session.rs
+++ b/boringtun/src/noise/session.rs
@@ -31,13 +31,17 @@ const DATA_OFFSET: usize = 16;
 /// The overhead of the AEAD
 const AEAD_SIZE: usize = 16;
 
-#[cfg(target_has_atomic="64")]
+#[cfg(target_has_atomic = "64")]
 type CounterBitmap = AtomicU64;
-#[cfg(not(target_has_atomic="64"))]
+#[cfg(not(target_has_atomic = "64"))]
 use portable_atomic::AtomicU32 as CounterBitmap;
 
 // Receiving buffer constants
-const WORD_SIZE: u64 = if cfg!(target_has_atomic="64") { 64 } else { 32 };
+const WORD_SIZE: u64 = if cfg!(target_has_atomic = "64") {
+    64
+} else {
+    32
+};
 const N_BITS: u64 = 1024;
 const N_WORDS: usize = (N_BITS / WORD_SIZE) as usize;
 
@@ -53,7 +57,7 @@ struct ReceivingKeyCounterValidator {
     next: AtomicU64,
     /// Used to estimate packet loss
     receive_cnt: AtomicU64,
-    bitmap: [ CounterBitmap; N_WORDS],
+    bitmap: [CounterBitmap; N_WORDS],
 }
 
 impl ReceivingKeyCounterValidator {
@@ -61,7 +65,7 @@ impl ReceivingKeyCounterValidator {
         ReceivingKeyCounterValidator {
             next: AtomicU64::new(0),
             receive_cnt: AtomicU64::new(0),
-            bitmap: [ const { CounterBitmap::new(0) }; N_WORDS],
+            bitmap: [const { CounterBitmap::new(0) }; N_WORDS],
         }
     }
 
@@ -91,7 +95,7 @@ impl ReceivingKeyCounterValidator {
             Err(WireGuardError::DuplicateCounter)
         } else {
             Ok(())
-        } 
+        }
     }
 
     /// Clear all the packets between prev and next.
@@ -109,9 +113,9 @@ impl ReceivingKeyCounterValidator {
             return;
         }
 
-        #[cfg(target_has_atomic="64")]
+        #[cfg(target_has_atomic = "64")]
         const ONE: u64 = 1;
-        #[cfg(not(target_has_atomic="64"))]
+        #[cfg(not(target_has_atomic = "64"))]
         const ONE: u32 = 1;
 
         let prev_idx = (prev / WORD_SIZE) as usize;
@@ -125,7 +129,7 @@ impl ReceivingKeyCounterValidator {
         } else {
             // The bits to clear span multiple words.
             self.bitmap[prev_idx % N_WORDS].fetch_and(prev_mask, Ordering::SeqCst);
-            for i in prev_idx+1..next_idx {
+            for i in prev_idx + 1..next_idx {
                 self.bitmap[i % N_WORDS].store(0, Ordering::SeqCst);
             }
             self.bitmap[next_idx % N_WORDS].fetch_and(next_mask, Ordering::SeqCst);
@@ -169,7 +173,7 @@ impl ReceivingKeyCounterValidator {
             Err(WireGuardError::DuplicateCounter)
         } else {
             Ok(())
-        }
+        };
     }
 
     /// Marks the counter as received, and returns true if it is still good (in case during
@@ -192,18 +196,28 @@ impl ReceivingKeyCounterValidator {
                 prev = self.next.load(Ordering::SeqCst);
             } else if counter < prev {
                 // This packet arrived out of order, acquire the spinlock to mark the packet.
-                match self.next.compare_exchange_weak(prev, prev | COUNTER_LOCK, Ordering::SeqCst, Ordering::Acquire) {
+                match self.next.compare_exchange_weak(
+                    prev,
+                    prev | COUNTER_LOCK,
+                    Ordering::SeqCst,
+                    Ordering::Acquire,
+                ) {
                     Ok(_) => break,
                     Err(x) => prev = x,
                 }
             } else {
                 // This packet arrived in-order.
                 // Acquire the spinlock, update the next packet counter, and clear bits that will wrap over.
-                match self.next.compare_exchange_weak(prev, (counter+1) | COUNTER_LOCK, Ordering::SeqCst, Ordering::Acquire) {
+                match self.next.compare_exchange_weak(
+                    prev,
+                    (counter + 1) | COUNTER_LOCK,
+                    Ordering::SeqCst,
+                    Ordering::Acquire,
+                ) {
                     Ok(_) => {
                         self.clear_range(prev, counter);
                         break;
-                    },
+                    }
                     Err(x) => prev = x,
                 }
             }
@@ -232,7 +246,7 @@ impl Session {
             ),
             sender: LessSafeKey::new(UnboundKey::new(&CHACHA20_POLY1305, &sending_key).unwrap()),
             sending_key_counter: AtomicU64::new(0),
-            receiving_key_counter: Default::default(),
+            receiving_key_counter: ReceivingKeyCounterValidator::new(),
         }
     }
 
@@ -313,13 +327,17 @@ impl Session {
         };
 
         // After decryption is done, check counter again, and mark as received
-        self.receiving_key_counter.mark_did_receive(packet.counter)?;
+        self.receiving_key_counter
+            .mark_did_receive(packet.counter)?;
         Ok(ret)
     }
 
     /// Returns the estimated downstream packet loss for this session
     pub(super) fn current_packet_cnt(&self) -> (u64, u64) {
-        let rx = self.receiving_key_counter.receive_cnt.load(Ordering::Relaxed);
+        let rx = self
+            .receiving_key_counter
+            .receive_cnt
+            .load(Ordering::Relaxed);
         (self.receiving_key_counter.get_next(), rx)
     }
 }
@@ -346,7 +364,7 @@ mod tests {
         // Clear a range, and recheck the bitmap.
         c.clear_range(start, end);
         let check_start = (start / WORD_SIZE) * WORD_SIZE;
-        for i in check_start..check_start+N_BITS {
+        for i in check_start..check_start + N_BITS {
             if i < start || i > end {
                 assert!(c.check_bit(i), "expected bit {} to be set", i);
             } else {
@@ -360,9 +378,9 @@ mod tests {
         // Clear a single bit and check edge cases.
         check_replay_clear_range(0, 0);
         check_replay_clear_range(42, 42);
-        check_replay_clear_range(WORD_SIZE-1, WORD_SIZE-1);
+        check_replay_clear_range(WORD_SIZE - 1, WORD_SIZE - 1);
         check_replay_clear_range(WORD_SIZE, WORD_SIZE);
-        check_replay_clear_range(N_BITS-1, N_BITS-1);
+        check_replay_clear_range(N_BITS - 1, N_BITS - 1);
         check_replay_clear_range(N_BITS, N_BITS);
 
         // Clear some bits in the middle of a single word.
@@ -395,8 +413,8 @@ mod tests {
         assert!(c.mark_did_receive(15).is_err());
 
         for i in 64..N_BITS + 128 {
-            assert!(c.mark_did_receive(i).is_ok(), "unexpected mark failed for bit {}", i);
-            assert!(c.mark_did_receive(i).is_err(), "duplicate packet not caught for bit {}", i);
+            assert!(c.mark_did_receive(i).is_ok());
+            assert!(c.mark_did_receive(i).is_err());
         }
 
         assert!(c.mark_did_receive(N_BITS * 3).is_ok());
@@ -451,33 +469,40 @@ mod tests {
             }
 
             match validator.will_accept(value) {
-                Ok(_) => {},
+                Ok(_) => {}
                 Err(WireGuardError::InvalidCounter) => {
                     // This error is allowed if, and only if, the thread hit an
                     // unlucky interrupt and the counter is too old now.
                     assert!(validator.get_next() >= value + N_BITS);
                     continue;
-                },
-                Err(WireGuardError::DuplicateCounter) => panic!("duplicate while checking {}", value),
+                }
+                Err(WireGuardError::DuplicateCounter) => {
+                    panic!("duplicate while checking {}", value)
+                }
                 _ => panic!("error while checking packet {}", value),
             };
 
             match validator.mark_did_receive(value) {
-                Ok(_) => {},
+                Ok(_) => {}
                 Err(WireGuardError::InvalidCounter) => {
                     // This error is allowed if, and only if, the thread hit an
                     // unlucky interrupt and the counter is too old now.
                     assert!(validator.get_next() >= value + N_BITS);
                     continue;
-                },
-                Err(WireGuardError::DuplicateCounter) => panic!("duplicate while marking {}", value),
+                }
+                Err(WireGuardError::DuplicateCounter) => {
+                    panic!("duplicate while marking {}", value)
+                }
                 _ => panic!("error while marking {}", value),
             };
 
             // Resend it as a duplicate, it must be rejected.
-            assert!(validator.mark_did_receive(value).is_err(),
-                    "race encountered while checking duplicate {}", value);
-            
+            assert!(
+                validator.mark_did_receive(value).is_err(),
+                "race encountered while checking duplicate {}",
+                value
+            );
+
             thread::yield_now();
         }
     }
@@ -500,10 +525,12 @@ mod tests {
         let mut threads = Vec::new();
         let num_threads = thread::available_parallelism().map_or(8, |x| x.get());
 
-        for _ in 0..num_threads-1 {
-            threads.push(thread::spawn(|| { racecheck_counter_worker(&COUNTER, &VALIDATOR) }));
+        for _ in 0..num_threads - 1 {
+            threads.push(thread::spawn(|| {
+                racecheck_counter_worker(&COUNTER, &VALIDATOR)
+            }));
         }
-        threads.push(thread::spawn(|| { racecheck_dup_worker(&COUNTER, &VALIDATOR) }));
+        threads.push(thread::spawn(|| racecheck_dup_worker(&COUNTER, &VALIDATOR)));
 
         for handle in threads {
             handle.join().unwrap();

From 9a60fd21f4455b657cd27a63a491613e1bae36af Mon Sep 17 00:00:00 2001
From: Naomi Kirby <oskirby@gmail.com>
Date: Fri, 29 May 2026 10:06:50 -0700
Subject: [PATCH 25/25] Use RwLock in JNI bindings too

---
 boringtun/src/jni.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/boringtun/src/jni.rs b/boringtun/src/jni.rs
index 7bc2bdcd7..5ed9d57b5 100644
--- a/boringtun/src/jni.rs
+++ b/boringtun/src/jni.rs
@@ -12,7 +12,7 @@ use jni::objects::{JByteBuffer, JClass, JString};
 use jni::strings::JNIStr;
 use jni::sys::{jbyteArray, jint, jlong, jshort, jstring};
 use jni::JNIEnv;
-use parking_lot::Mutex;
+use parking_lot::RwLock;
 
 use crate::ffi::new_tunnel;
 use crate::ffi::wireguard_read;
@@ -194,7 +194,7 @@ pub unsafe extern "C" fn encrypt_raw_packet(
     };
 
     let output: wireguard_result = wireguard_write(
-        tunnel as *const Mutex<Tunn>,
+        tunnel as *const RwLock<Tunn>,
         env.convert_byte_array(src).unwrap().as_mut_ptr(),
         src_size as u32,
         dst_ptr,
@@ -229,7 +229,7 @@ pub unsafe extern "C" fn decrypt_to_raw_packet(
     };
 
     let output: wireguard_result = wireguard_read(
-        tunnel as *const Mutex<Tunn>,
+        tunnel as *const RwLock<Tunn>,
         env.convert_byte_array(src).unwrap().as_mut_ptr(),
         src_size as u32,
         dst_ptr,
@@ -263,7 +263,7 @@ pub unsafe extern "C" fn run_periodic_task(
     };
 
     let output: wireguard_result =
-        wireguard_tick(tunnel as *const Mutex<Tunn>, dst_ptr, dst_size as u32);
+        wireguard_tick(tunnel as *const RwLock<Tunn>, dst_ptr, dst_size as u32);
 
     *op_ptr = output.op as u8;