diff --git a/Makefile.am b/Makefile.am index 6652b3c..28cca77 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,11 @@ EXTRA_DIST = README.html +EXTRA_DIST += \ + benchmarks/README.md \ + benchmarks/benchmark.c \ + benchmarks/run.sh \ + benchmarks/runner.py \ + benchmarks/selftest.py SUBDIRS = . tests diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 0000000..d778e48 --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1,3 @@ +artifacts/ +raw/ +__pycache__/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..e81902c --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,64 @@ +# libfastjson performance benchmarks + +This suite isolates the libfastjson operations that dominate rsyslog's `$!` +variable path. It measures case-insensitive object lookup by width and key +position, replacement writes, string creation around inline-storage +boundaries, and narrow-object construction/destruction both with normal +duplicate checking and with the existing `FJSON_OBJECT_ADD_KEY_IS_NEW` flag. + +Every workload uses a correctness checksum and structural oracle. The runner +performs one calibration, then 11 measured trials, records resumable raw JSON, +and reports medians, median absolute deviation, and outliers. Paired mode runs +baseline and candidate builds in alternating order. + +Build libfastjson first: + +```sh +CFLAGS='-O2 -g -fno-omit-frame-pointer' ./autogen.sh --configure +make -j"$(nproc)" +``` + +Benchmark build directories must be clean Git checkouts. The runner rejects +uncommitted library sources so recorded measurements can be reproduced. + +Record a characterization session: + +```sh +benchmarks/run.sh \ + --build-dir "$PWD" \ + --label current-main-a \ + --output benchmarks/raw/current-main-a.json \ + --summary-json benchmarks/results/current-main-a.json \ + --summary-markdown benchmarks/results/current-main-a.md +``` + +Use `--smoke` for a reduced matrix. Filters `--operation`, `--width`, +`--position`, and `--value-bytes` are repeatable. + +For candidate acceptance, use paired mode with separate worktrees/builds: + +```sh +benchmarks/run.sh \ + --build-dir /path/to/baseline \ + --label baseline \ + --output benchmarks/raw/baseline.json \ + --pair-build-dir /path/to/candidate \ + --pair-label candidate \ + --pair-output benchmarks/raw/candidate.json \ + --comparison-json benchmarks/results/comparison.json \ + --comparison-markdown benchmarks/results/comparison.md +``` + +A runtime candidate should be retained only when two independent paired +sessions each improve its targeted workload by at least 10%, no core workload +regresses by more than 5%, and relative MAD is at most 0.05 after one rerun. + +Run deterministic harness tests with: + +```sh +python3 benchmarks/selftest.py +``` + +Tracked current-main characterization and direct-lookup candidate results are +under `benchmarks/results/`. Raw trial data remain ignored under +`benchmarks/raw/`. diff --git a/benchmarks/benchmark.c b/benchmarks/benchmark.c new file mode 100644 index 0000000..0fb9635 --- /dev/null +++ b/benchmarks/benchmark.c @@ -0,0 +1,269 @@ +/* + * Microbenchmark for libfastjson operations used heavily by rsyslog. + * + * Setup is outside the timed interval for lookup and replacement workloads. + * Every workload verifies its final value, member count, and accumulated + * result so an optimizing compiler or a broken library cannot silently turn + * the timed loop into a no-op. + */ +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include +#include +#include +#include +#include + +#include "json.h" + +static volatile uint64_t result_sink; + +static void +fail(const char *const message) +{ + fprintf(stderr, "benchmark error: %s\n", message); + exit(1); +} + +static uint64_t +parse_u64(const char *const text, const char *const name) +{ + char *end = NULL; + errno = 0; + const unsigned long long value = strtoull(text, &end, 10); + if (errno != 0 || end == text || *end != '\0') { + fprintf(stderr, "benchmark error: invalid %s: %s\n", name, text); + exit(2); + } + return (uint64_t) value; +} + +static uint64_t +now_ns(void) +{ + struct timespec now; + if (clock_gettime(CLOCK_MONOTONIC_RAW, &now) != 0) + fail("clock_gettime failed"); + return (uint64_t) now.tv_sec * UINT64_C(1000000000) + (uint64_t) now.tv_nsec; +} + +static char ** +make_keys(const size_t width) +{ + char **const keys = calloc(width == 0 ? 1 : width, sizeof(*keys)); + if (keys == NULL) + fail("key table allocation failed"); + for (size_t i = 0 ; i < width ; ++i) { + keys[i] = malloc(32); + if (keys[i] == NULL) + fail("key allocation failed"); + snprintf(keys[i], 32, "key-%03zu", i); + } + return keys; +} + +static void +free_keys(char **const keys, const size_t width) +{ + for (size_t i = 0 ; i < width ; ++i) + free(keys[i]); + free(keys); +} + +static size_t +position_index(const char *const position, const size_t width) +{ + if (!strcmp(position, "first")) + return 0; + if (!strcmp(position, "middle")) + return width / 2; + if (!strcmp(position, "last")) + return width - 1; + if (!strcmp(position, "miss")) + return width; + fail("unknown position"); + return 0; +} + +static struct fjson_object * +make_object(char **const keys, const size_t width, const int assume_new) +{ + struct fjson_object *const object = fjson_object_new_object(); + if (object == NULL) + fail("object allocation failed"); + for (size_t i = 0 ; i < width ; ++i) { + struct fjson_object *const value = fjson_object_new_int64((int64_t) i); + if (value == NULL) + fail("integer allocation failed"); + if (assume_new) + fjson_object_object_add_ex(object, keys[i], value, FJSON_OBJECT_ADD_KEY_IS_NEW); + else + fjson_object_object_add(object, keys[i], value); + } + if ((size_t) fjson_object_object_length(object) != width) + fail("object setup member count mismatch"); + return object; +} + +static uint64_t +run_lookup(const size_t width, const char *const position, const uint64_t iterations) +{ + if (width == 0) + fail("lookup width must be positive"); + char **const keys = make_keys(width + 1); + struct fjson_object *const object = make_object(keys, width, 0); + const size_t index = position_index(position, width); + uint64_t checksum = 0; + const uint64_t start = now_ns(); + for (uint64_t i = 0 ; i < iterations ; ++i) { + struct fjson_object *value = NULL; + const fjson_bool found = fjson_object_object_get_ex(object, keys[index], &value); + if (index == width) { + checksum += found ? UINT64_C(1000003) : 1; + } else { + if (!found || value == NULL) + fail("lookup unexpectedly missed"); + checksum += (uint64_t) fjson_object_get_int64(value) + 1; + } + } + const uint64_t elapsed = now_ns() - start; + const uint64_t expected = iterations * (index == width ? 1 : index + 1); + if (checksum != expected) + fail("lookup checksum mismatch"); + result_sink = checksum; + fjson_object_put(object); + free_keys(keys, width + 1); + return elapsed; +} + +static uint64_t +run_replace(const size_t width, const char *const position, const uint64_t iterations) +{ + if (width == 0 || !strcmp(position, "miss")) + fail("replacement requires a present key"); + char **const keys = make_keys(width); + struct fjson_object *const object = make_object(keys, width, 0); + const size_t index = position_index(position, width); + uint64_t checksum = 0; + const uint64_t start = now_ns(); + for (uint64_t i = 0 ; i < iterations ; ++i) { + struct fjson_object *const value = fjson_object_new_int64((int64_t) i); + if (value == NULL) + fail("replacement value allocation failed"); + fjson_object_object_add(object, keys[index], value); + checksum += i & 1; + } + const uint64_t elapsed = now_ns() - start; + struct fjson_object *final_value = NULL; + if (!fjson_object_object_get_ex(object, keys[index], &final_value) || final_value == NULL) + fail("replacement final lookup failed"); + if (fjson_object_get_int64(final_value) != (int64_t) (iterations - 1)) + fail("replacement final value mismatch"); + if ((size_t) fjson_object_object_length(object) != width) + fail("replacement changed member count"); + if (checksum != iterations / 2) + fail("replacement checksum mismatch"); + result_sink = checksum; + fjson_object_put(object); + free_keys(keys, width); + return elapsed; +} + +static uint64_t +run_string(const size_t value_bytes, const uint64_t iterations) +{ + if (value_bytes == SIZE_MAX) + fail("string value bytes too large"); + char *const value = malloc(value_bytes + 1); + if (value == NULL) + fail("string source allocation failed"); + for (size_t i = 0 ; i < value_bytes ; ++i) + value[i] = (char) ('a' + (i % 23)); + value[value_bytes] = '\0'; + uint64_t checksum = 0; + const uint64_t start = now_ns(); + for (uint64_t i = 0 ; i < iterations ; ++i) { + struct fjson_object *const object = fjson_object_new_string(value); + if (object == NULL) + fail("string object allocation failed"); + if ((size_t) fjson_object_get_string_len(object) != value_bytes) + fail("string length mismatch"); + const char *const copy = fjson_object_get_string(object); + if (copy == NULL || (value_bytes > 0 && copy[value_bytes - 1] != value[value_bytes - 1])) + fail("string content mismatch"); + checksum += value_bytes + (value_bytes == 0 ? 0 : (unsigned char) copy[0]); + fjson_object_put(object); + } + const uint64_t elapsed = now_ns() - start; + const uint64_t expected = iterations * (value_bytes + (value_bytes == 0 ? 0 : (unsigned char) value[0])); + if (checksum != expected) + fail("string checksum mismatch"); + result_sink = checksum; + free(value); + return elapsed; +} + +static uint64_t +run_object(const size_t width, const uint64_t iterations, const int assume_new) +{ + char **const keys = make_keys(width); + uint64_t checksum = 0; + const uint64_t start = now_ns(); + for (uint64_t i = 0 ; i < iterations ; ++i) { + struct fjson_object *const object = make_object(keys, width, assume_new); + checksum += (uint64_t) fjson_object_object_length(object) + 1; + fjson_object_put(object); + } + const uint64_t elapsed = now_ns() - start; + if (checksum != iterations * (width + 1)) + fail("object lifecycle checksum mismatch"); + result_sink = checksum; + free_keys(keys, width); + return elapsed; +} + +int +main(const int argc, char **const argv) +{ + if (argc != 7) { + fprintf(stderr, + "usage: %s OPERATION WIDTH POSITION VALUE_BYTES ITERATIONS CASE_SENSITIVE\n", + argv[0]); + return 2; + } + const char *const operation = argv[1]; + const size_t width = (size_t) parse_u64(argv[2], "width"); + const char *const position = argv[3]; + const size_t value_bytes = (size_t) parse_u64(argv[4], "value bytes"); + const uint64_t iterations = parse_u64(argv[5], "iterations"); + const uint64_t case_mode = parse_u64(argv[6], "case mode"); + if (iterations == 0 || case_mode > 1) + fail("iterations and case mode are out of range"); + const int case_sensitive = (int) case_mode; + fjson_global_do_case_sensitive_comparison(case_sensitive); + + uint64_t elapsed; + if (!strcmp(operation, "lookup")) + elapsed = run_lookup(width, position, iterations); + else if (!strcmp(operation, "replace")) + elapsed = run_replace(width, position, iterations); + else if (!strcmp(operation, "string")) + elapsed = run_string(value_bytes, iterations); + else if (!strcmp(operation, "object")) + elapsed = run_object(width, iterations, 0); + else if (!strcmp(operation, "object-new")) + elapsed = run_object(width, iterations, 1); + else + fail("unknown operation"); + + printf("{\"operation\":\"%s\",\"width\":%zu,\"position\":\"%s\"," + "\"value_bytes\":%zu,\"iterations\":%" PRIu64 ",\"case_sensitive\":%d," + "\"elapsed_ns\":%" PRIu64 ",\"ns_per_operation\":%.9f," + "\"operations_per_second\":%.6f,\"oracle\":true}\n", + operation, width, position, value_bytes, iterations, case_sensitive, elapsed, + (double) elapsed / (double) iterations, + (double) iterations * 1000000000.0 / (double) elapsed); + return result_sink == UINT64_MAX ? 3 : 0; +} diff --git a/benchmarks/results/current-main-a.json b/benchmarks/results/current-main-a.json new file mode 100644 index 0000000..e68f838 --- /dev/null +++ b/benchmarks/results/current-main-a.json @@ -0,0 +1,1101 @@ +{ + "config": { + "calibrations": 1, + "target_ms": 200, + "trials": 11, + "workloads": [ + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "middle", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "last", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "middle", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "last", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "middle", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "last", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "middle", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "last", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "first", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "first", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "last", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "first", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "last", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 8, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 16, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 31, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 32, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 33, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 64, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 127, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 128, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 129, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 208, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 209, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 256, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 4096, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 32 + } + ] + }, + "kind": "characterization", + "metadata": { + "architecture": "x86_64", + "cflags": "-O2 -g -fno-omit-frame-pointer", + "clock": "CLOCK_MONOTONIC_RAW", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "config_status_sha256": "bb25b3f70ff347a5f92b5e404fb5af0c4b33ae94766ddee0b8d4930bfa15ad8e", + "configure_args": "'CFLAGS=-O2 -g -fno-omit-frame-pointer'", + "cpu": "Intel(R) Core(TM) i7-14700K", + "harness_sha256": "2ec960faa7b4c2b859502af7da5f041ddbe69616521f70c3b95e27b9a2c97133", + "host_exclusive": false, + "kernel": "5.15.167.4-microsoft-standard-WSL2", + "label": "current-main-a", + "library_sha256": "e143fd02577ad7d926db5d5effacb1c4b787eec2e3bdea281e56ef4c78721f2f", + "library_source_dirty": false, + "library_source_fingerprint": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "python": "3.12.3", + "revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018" + }, + "schema_version": 1, + "workloads": [ + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.1759327000000006, + "median_ns_per_operation": 14.812725637, + "median_operations_per_second": 67509520.15894683, + "operation": "lookup", + "outlier_trials": [], + "position": "first", + "relative_mad": 0.011877132157267985, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.11837615200000151, + "median_ns_per_operation": 14.777685545, + "median_operations_per_second": 67669595.27964431, + "operation": "lookup", + "outlier_trials": [ + 2 + ], + "position": "miss", + "relative_mad": 0.00801046629660176, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.11256165800000062, + "median_ns_per_operation": 14.880281632, + "median_operations_per_second": 67203029.1314852, + "operation": "lookup", + "outlier_trials": [], + "position": "first", + "relative_mad": 0.007564484381662314, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.5825649999999989, + "median_ns_per_operation": 31.327197314, + "median_operations_per_second": 31921144.74770151, + "operation": "lookup", + "outlier_trials": [], + "position": "last", + "relative_mad": 0.018596141689944695, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.06919514700000207, + "median_ns_per_operation": 19.711185739, + "median_operations_per_second": 50732615.13747638, + "operation": "lookup", + "outlier_trials": [], + "position": "middle", + "relative_mad": 0.003510450762132208, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.05020599699999906, + "median_ns_per_operation": 22.664028353, + "median_operations_per_second": 44122782.782683544, + "operation": "lookup", + "outlier_trials": [ + 4, + 5, + 7 + ], + "position": "miss", + "relative_mad": 0.00221522830001902, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.08255120000000105, + "median_ns_per_operation": 14.782551943, + "median_operations_per_second": 67647318.53173235, + "operation": "lookup", + "outlier_trials": [], + "position": "first", + "relative_mad": 0.0055843673215768145, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.6352754799999971, + "median_ns_per_operation": 36.570931924, + "median_operations_per_second": 27344121.338722054, + "operation": "lookup", + "outlier_trials": [], + "position": "last", + "relative_mad": 0.01737104980863482, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.645506349999998, + "median_ns_per_operation": 31.173266651, + "median_operations_per_second": 32078768.362504005, + "operation": "lookup", + "outlier_trials": [], + "position": "middle", + "relative_mad": 0.020707048678175376, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.328051815000002, + "median_ns_per_operation": 43.037421053, + "median_operations_per_second": 23235593.014937244, + "operation": "lookup", + "outlier_trials": [ + 4, + 6, + 9 + ], + "position": "miss", + "relative_mad": 0.007622478461151532, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.20236342300000132, + "median_ns_per_operation": 14.789998818, + "median_operations_per_second": 67613257.60100543, + "operation": "lookup", + "outlier_trials": [ + 8 + ], + "position": "first", + "relative_mad": 0.013682450248320318, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 1.175693975999998, + "median_ns_per_operation": 59.684533756, + "median_operations_per_second": 16754759.35002125, + "operation": "lookup", + "outlier_trials": [], + "position": "last", + "relative_mad": 0.019698469637149628, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.08705463799999791, + "median_ns_per_operation": 66.499125974, + "median_operations_per_second": 15037791.630388986, + "operation": "lookup", + "outlier_trials": [ + 10 + ], + "position": "middle", + "relative_mad": 0.0013091095067029115, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.18440566400000336, + "median_ns_per_operation": 61.679601712, + "median_operations_per_second": 16212815.456709512, + "operation": "lookup", + "outlier_trials": [ + 11 + ], + "position": "miss", + "relative_mad": 0.002989734999604035, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.06134993899999941, + "median_ns_per_operation": 14.766967028, + "median_operations_per_second": 67718712.86120407, + "operation": "lookup", + "outlier_trials": [ + 2 + ], + "position": "first", + "relative_mad": 0.004154538903193345, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 1.941241395999981, + "median_ns_per_operation": 137.794022567, + "median_operations_per_second": 7257208.849634728, + "operation": "lookup", + "outlier_trials": [], + "position": "last", + "relative_mad": 0.014087994238328337, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.24832689800000196, + "median_ns_per_operation": 62.969113852, + "median_operations_per_second": 15880801.536295375, + "operation": "lookup", + "outlier_trials": [], + "position": "middle", + "relative_mad": 0.003943630183261896, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.5328674699999993, + "median_ns_per_operation": 117.778086839, + "median_operations_per_second": 8490543.757659925, + "operation": "lookup", + "outlier_trials": [ + 4 + ], + "position": "miss", + "relative_mad": 0.0045243345710685315, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.17236799299999817, + "median_ns_per_operation": 26.150814717, + "median_operations_per_second": 38239726.40324374, + "operation": "object", + "outlier_trials": [ + 10 + ], + "position": "none", + "relative_mad": 0.006591304892996163, + "trials": 11, + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.3110038539999991, + "median_ns_per_operation": 60.400775293, + "median_operations_per_second": 16556078.877283758, + "operation": "object", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.005149004337963226, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.2827091069999881, + "median_ns_per_operation": 176.324410677, + "median_operations_per_second": 5671364.481868882, + "operation": "object", + "outlier_trials": [ + 2 + ], + "position": "none", + "relative_mad": 0.0016033463881406015, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 1.2356566160000284, + "median_ns_per_operation": 427.785113083, + "median_operations_per_second": 2337622.2533624666, + "operation": "object", + "outlier_trials": [ + 11 + ], + "position": "none", + "relative_mad": 0.0028884984030762262, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 3.525021156999969, + "median_ns_per_operation": 557.389322924, + "median_operations_per_second": 1794078.1404891566, + "operation": "object", + "outlier_trials": [ + 10 + ], + "position": "none", + "relative_mad": 0.006324163402535441, + "trials": 11, + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 3.58854675900011, + "median_ns_per_operation": 1047.242270357, + "median_operations_per_second": 954888.8813083381, + "operation": "object", + "outlier_trials": [ + 3, + 8 + ], + "position": "none", + "relative_mad": 0.0034266634002242777, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 55.761020923999695, + "median_ns_per_operation": 3646.320692978, + "median_operations_per_second": 274249.05382726673, + "operation": "object", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.015292407228849338, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.46330584200000047, + "median_ns_per_operation": 26.222619205, + "median_operations_per_second": 38135015.88770831, + "operation": "object-new", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.017668175645538093, + "trials": 11, + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.28738365099999896, + "median_ns_per_operation": 54.947984342, + "median_operations_per_second": 18199029.718286514, + "operation": "object-new", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.005230103605098661, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.18110990099998503, + "median_ns_per_operation": 130.113278972, + "median_operations_per_second": 7685610.630220126, + "operation": "object-new", + "outlier_trials": [ + 1, + 7, + 8, + 10 + ], + "position": "none", + "relative_mad": 0.0013919401803635996, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 1.1813003149999872, + "median_ns_per_operation": 248.354003801, + "median_operations_per_second": 4026510.4838063153, + "operation": "object-new", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.004756518102871151, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 1.1646863230000122, + "median_ns_per_operation": 356.025562252, + "median_operations_per_second": 2808787.081676415, + "operation": "object-new", + "outlier_trials": [ + 3 + ], + "position": "none", + "relative_mad": 0.003271355898247639, + "trials": 11, + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 1.2442294599999855, + "median_ns_per_operation": 488.741356521, + "median_operations_per_second": 2046071.9901386788, + "operation": "object-new", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.0025457830474113438, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 4.881220201999895, + "median_ns_per_operation": 1327.268728668, + "median_operations_per_second": 753426.9273439182, + "operation": "object-new", + "outlier_trials": [ + 10 + ], + "position": "none", + "relative_mad": 0.003677642738481841, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.13634460000000104, + "median_ns_per_operation": 31.357131883, + "median_operations_per_second": 31890671.75311851, + "operation": "replace", + "outlier_trials": [ + 5 + ], + "position": "first", + "relative_mad": 0.004348120883910275, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.06471257599999802, + "median_ns_per_operation": 31.400104232, + "median_operations_per_second": 31847028.042056467, + "operation": "replace", + "outlier_trials": [ + 11 + ], + "position": "first", + "relative_mad": 0.0020609032225456475, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.10137016000000187, + "median_ns_per_operation": 57.27294228, + "median_operations_per_second": 17460251.9128689, + "operation": "replace", + "outlier_trials": [ + 2, + 5, + 8, + 9 + ], + "position": "last", + "relative_mad": 0.0017699485300478588, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.05641114400000191, + "median_ns_per_operation": 31.241667974, + "median_operations_per_second": 32008534.270072326, + "operation": "replace", + "outlier_trials": [ + 1, + 3, + 11 + ], + "position": "first", + "relative_mad": 0.001805638035938046, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.42430595500002255, + "median_ns_per_operation": 136.568728501, + "median_operations_per_second": 7322320.497350736, + "operation": "replace", + "outlier_trials": [], + "position": "last", + "relative_mad": 0.003106904191444644, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.32586338600000175, + "median_ns_per_operation": 21.695666522, + "median_operations_per_second": 46092153.88639813, + "operation": "string", + "outlier_trials": [ + 9 + ], + "position": "none", + "relative_mad": 0.015019745333454834, + "trials": 11, + "value_bytes": 8, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.35592439800000264, + "median_ns_per_operation": 21.756158418, + "median_operations_per_second": 45963996.988211304, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.016359707957703043, + "trials": 11, + "value_bytes": 16, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.21567376399999816, + "median_ns_per_operation": 21.738756459, + "median_operations_per_second": 46000791.34636944, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.00992116381665004, + "trials": 11, + "value_bytes": 31, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.3865870539999996, + "median_ns_per_operation": 29.416492143, + "median_operations_per_second": 33994535.96094263, + "operation": "string", + "outlier_trials": [ + 10 + ], + "position": "none", + "relative_mad": 0.01314184750923786, + "trials": 11, + "value_bytes": 32, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.3212571290000028, + "median_ns_per_operation": 29.513456857, + "median_operations_per_second": 33882848.92702496, + "operation": "string", + "outlier_trials": [ + 1, + 2 + ], + "position": "none", + "relative_mad": 0.010885106768636865, + "trials": 11, + "value_bytes": 33, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.2255460840000012, + "median_ns_per_operation": 29.582812723, + "median_operations_per_second": 33803411.777086414, + "operation": "string", + "outlier_trials": [ + 2 + ], + "position": "none", + "relative_mad": 0.007624227152161362, + "trials": 11, + "value_bytes": 64, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.4078788229999972, + "median_ns_per_operation": 29.450274785, + "median_operations_per_second": 33955540.56118122, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.01384974591842326, + "trials": 11, + "value_bytes": 127, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.371953714, + "median_ns_per_operation": 29.616253996, + "median_operations_per_second": 33765242.56359568, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.012559107375640295, + "trials": 11, + "value_bytes": 128, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.26420439600000023, + "median_ns_per_operation": 29.523526268, + "median_operations_per_second": 33871292.708143786, + "operation": "string", + "outlier_trials": [ + 10 + ], + "position": "none", + "relative_mad": 0.008948944431694342, + "trials": 11, + "value_bytes": 129, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.19901763600000066, + "median_ns_per_operation": 29.96031589, + "median_operations_per_second": 33377485.193130918, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.006642708198761941, + "trials": 11, + "value_bytes": 208, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.11842336000000131, + "median_ns_per_operation": 30.005267989, + "median_operations_per_second": 33327481.039882805, + "operation": "string", + "outlier_trials": [ + 2 + ], + "position": "none", + "relative_mad": 0.003946752285079259, + "trials": 11, + "value_bytes": 209, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.3205579589999985, + "median_ns_per_operation": 32.566092437, + "median_operations_per_second": 30706785.038288746, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.009843304339325532, + "trials": 11, + "value_bytes": 256, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.2611455520000021, + "median_ns_per_operation": 88.573817662, + "median_operations_per_second": 11290018.048177918, + "operation": "string", + "outlier_trials": [ + 3, + 8, + 11 + ], + "position": "none", + "relative_mad": 0.002948337995281409, + "trials": 11, + "value_bytes": 4096, + "width": 0 + } + ] +} diff --git a/benchmarks/results/current-main-a.md b/benchmarks/results/current-main-a.md new file mode 100644 index 0000000..62559b6 --- /dev/null +++ b/benchmarks/results/current-main-a.md @@ -0,0 +1,56 @@ +# libfastjson characterization: current-main-a + +Revision: `f1983a831a1abd6ab8d8cdbc5137f7bc997b4018` + +| Operation | Width | Position | Bytes | Median ns/op | M operations/s | MAD | Outliers | +|---|---:|---|---:|---:|---:|---:|---| +| lookup | 1 | first | 0 | 14.813 | 67.510 | 0.0119 | - | +| lookup | 1 | miss | 0 | 14.778 | 67.670 | 0.0080 | 2 | +| lookup | 4 | first | 0 | 14.880 | 67.203 | 0.0076 | - | +| lookup | 4 | last | 0 | 31.327 | 31.921 | 0.0186 | - | +| lookup | 4 | middle | 0 | 19.711 | 50.733 | 0.0035 | - | +| lookup | 4 | miss | 0 | 22.664 | 44.123 | 0.0022 | 4,5,7 | +| lookup | 8 | first | 0 | 14.783 | 67.647 | 0.0056 | - | +| lookup | 8 | last | 0 | 36.571 | 27.344 | 0.0174 | - | +| lookup | 8 | middle | 0 | 31.173 | 32.079 | 0.0207 | - | +| lookup | 8 | miss | 0 | 43.037 | 23.236 | 0.0076 | 4,6,9 | +| lookup | 16 | first | 0 | 14.790 | 67.613 | 0.0137 | 8 | +| lookup | 16 | last | 0 | 59.685 | 16.755 | 0.0197 | - | +| lookup | 16 | middle | 0 | 66.499 | 15.038 | 0.0013 | 10 | +| lookup | 16 | miss | 0 | 61.680 | 16.213 | 0.0030 | 11 | +| lookup | 32 | first | 0 | 14.767 | 67.719 | 0.0042 | 2 | +| lookup | 32 | last | 0 | 137.794 | 7.257 | 0.0141 | - | +| lookup | 32 | middle | 0 | 62.969 | 15.881 | 0.0039 | - | +| lookup | 32 | miss | 0 | 117.778 | 8.491 | 0.0045 | 4 | +| object | 0 | none | 0 | 26.151 | 38.240 | 0.0066 | 10 | +| object | 1 | none | 0 | 60.401 | 16.556 | 0.0051 | - | +| object | 4 | none | 0 | 176.324 | 5.671 | 0.0016 | 2 | +| object | 8 | none | 0 | 427.785 | 2.338 | 0.0029 | 11 | +| object | 9 | none | 0 | 557.389 | 1.794 | 0.0063 | 10 | +| object | 16 | none | 0 | 1047.242 | 0.955 | 0.0034 | 3,8 | +| object | 32 | none | 0 | 3646.321 | 0.274 | 0.0153 | - | +| object-new | 0 | none | 0 | 26.223 | 38.135 | 0.0177 | - | +| object-new | 1 | none | 0 | 54.948 | 18.199 | 0.0052 | - | +| object-new | 4 | none | 0 | 130.113 | 7.686 | 0.0014 | 1,7,8,10 | +| object-new | 8 | none | 0 | 248.354 | 4.027 | 0.0048 | - | +| object-new | 9 | none | 0 | 356.026 | 2.809 | 0.0033 | 3 | +| object-new | 16 | none | 0 | 488.741 | 2.046 | 0.0025 | - | +| object-new | 32 | none | 0 | 1327.269 | 0.753 | 0.0037 | 10 | +| replace | 1 | first | 0 | 31.357 | 31.891 | 0.0043 | 5 | +| replace | 8 | first | 0 | 31.400 | 31.847 | 0.0021 | 11 | +| replace | 8 | last | 0 | 57.273 | 17.460 | 0.0018 | 2,5,8,9 | +| replace | 32 | first | 0 | 31.242 | 32.009 | 0.0018 | 1,3,11 | +| replace | 32 | last | 0 | 136.569 | 7.322 | 0.0031 | - | +| string | 0 | none | 8 | 21.696 | 46.092 | 0.0150 | 9 | +| string | 0 | none | 16 | 21.756 | 45.964 | 0.0164 | - | +| string | 0 | none | 31 | 21.739 | 46.001 | 0.0099 | - | +| string | 0 | none | 32 | 29.416 | 33.995 | 0.0131 | 10 | +| string | 0 | none | 33 | 29.513 | 33.883 | 0.0109 | 1,2 | +| string | 0 | none | 64 | 29.583 | 33.803 | 0.0076 | 2 | +| string | 0 | none | 127 | 29.450 | 33.956 | 0.0138 | - | +| string | 0 | none | 128 | 29.616 | 33.765 | 0.0126 | - | +| string | 0 | none | 129 | 29.524 | 33.871 | 0.0089 | 10 | +| string | 0 | none | 208 | 29.960 | 33.377 | 0.0066 | - | +| string | 0 | none | 209 | 30.005 | 33.327 | 0.0039 | 2 | +| string | 0 | none | 256 | 32.566 | 30.707 | 0.0098 | - | +| string | 0 | none | 4096 | 88.574 | 11.290 | 0.0029 | 3,8,11 | diff --git a/benchmarks/results/current-main-b.json b/benchmarks/results/current-main-b.json new file mode 100644 index 0000000..11638cd --- /dev/null +++ b/benchmarks/results/current-main-b.json @@ -0,0 +1,1108 @@ +{ + "config": { + "calibrations": 1, + "target_ms": 200, + "trials": 11, + "workloads": [ + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "middle", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "last", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "middle", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "last", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "middle", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "last", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "first", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "middle", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "last", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "lookup", + "position": "miss", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "first", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "first", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "last", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "first", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "replace", + "position": "last", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 8, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 16, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 31, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 32, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 33, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 64, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 127, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 128, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 129, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 208, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 209, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 256, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "string", + "position": "none", + "value_bytes": 4096, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "object", + "position": "none", + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "operation": "object-new", + "position": "none", + "value_bytes": 0, + "width": 32 + } + ] + }, + "kind": "characterization", + "metadata": { + "architecture": "x86_64", + "cflags": "-O2 -g -fno-omit-frame-pointer", + "clock": "CLOCK_MONOTONIC_RAW", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "config_status_sha256": "bb25b3f70ff347a5f92b5e404fb5af0c4b33ae94766ddee0b8d4930bfa15ad8e", + "configure_args": "'CFLAGS=-O2 -g -fno-omit-frame-pointer'", + "cpu": "Intel(R) Core(TM) i7-14700K", + "harness_sha256": "2ec960faa7b4c2b859502af7da5f041ddbe69616521f70c3b95e27b9a2c97133", + "host_exclusive": false, + "kernel": "5.15.167.4-microsoft-standard-WSL2", + "label": "current-main-b", + "library_sha256": "e143fd02577ad7d926db5d5effacb1c4b787eec2e3bdea281e56ef4c78721f2f", + "library_source_dirty": false, + "library_source_fingerprint": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "python": "3.12.3", + "revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018" + }, + "schema_version": 1, + "workloads": [ + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.043675417000001104, + "median_ns_per_operation": 14.739515072, + "median_operations_per_second": 67844837.16833097, + "operation": "lookup", + "outlier_trials": [ + 4, + 5, + 7 + ], + "position": "first", + "relative_mad": 0.002963151554624029, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.03257357199999866, + "median_ns_per_operation": 14.76651039, + "median_operations_per_second": 67720806.987493, + "operation": "lookup", + "outlier_trials": [ + 3, + 10, + 11 + ], + "position": "miss", + "relative_mad": 0.002205908582305116, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.06670808500000014, + "median_ns_per_operation": 14.610032366, + "median_operations_per_second": 68446118.04742938, + "operation": "lookup", + "outlier_trials": [], + "position": "first", + "relative_mad": 0.004565909460627963, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.6723772729999986, + "median_ns_per_operation": 30.491033688, + "median_operations_per_second": 32796526.684943393, + "operation": "lookup", + "outlier_trials": [ + 6 + ], + "position": "last", + "relative_mad": 0.022051639176293922, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.0732915279999986, + "median_ns_per_operation": 19.595713395, + "median_operations_per_second": 51031568.98871351, + "operation": "lookup", + "outlier_trials": [ + 5 + ], + "position": "middle", + "relative_mad": 0.003740181667420157, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.07879744699999947, + "median_ns_per_operation": 22.5796705, + "median_operations_per_second": 44287625.89781813, + "operation": "lookup", + "outlier_trials": [], + "position": "miss", + "relative_mad": 0.0034897518544391277, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.10834713799999918, + "median_ns_per_operation": 14.749629481, + "median_operations_per_second": 67798313.25852409, + "operation": "lookup", + "outlier_trials": [], + "position": "first", + "relative_mad": 0.007345753202788483, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.9408228739999984, + "median_ns_per_operation": 37.194341762, + "median_operations_per_second": 26885809.84706821, + "operation": "lookup", + "outlier_trials": [], + "position": "last", + "relative_mad": 0.025294784890136174, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.486178301999999, + "median_ns_per_operation": 29.59296458, + "median_operations_per_second": 33791815.52752698, + "operation": "lookup", + "outlier_trials": [ + 7 + ], + "position": "middle", + "relative_mad": 0.01642884749467027, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.45483872399999825, + "median_ns_per_operation": 43.433152149, + "median_operations_per_second": 23023887.29626256, + "operation": "lookup", + "outlier_trials": [ + 11 + ], + "position": "miss", + "relative_mad": 0.010472155519351833, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.1605484730000004, + "median_ns_per_operation": 14.762725131, + "median_operations_per_second": 67738171.0440518, + "operation": "lookup", + "outlier_trials": [], + "position": "first", + "relative_mad": 0.010875259924935361, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.44439042999999856, + "median_ns_per_operation": 59.140815714, + "median_operations_per_second": 16908796.199834574, + "operation": "lookup", + "outlier_trials": [ + 4, + 6, + 8, + 9 + ], + "position": "last", + "relative_mad": 0.007514107214026827, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.29522098500000027, + "median_ns_per_operation": 66.622533284, + "median_operations_per_second": 15009936.589129357, + "operation": "lookup", + "outlier_trials": [], + "position": "middle", + "relative_mad": 0.004431248264630313, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.4823204059999995, + "median_ns_per_operation": 61.656348541, + "median_operations_per_second": 16218929.98309856, + "operation": "lookup", + "outlier_trials": [], + "position": "miss", + "relative_mad": 0.007822720894333661, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.05126841699999929, + "median_ns_per_operation": 14.803477483, + "median_operations_per_second": 67551695.27892205, + "operation": "lookup", + "outlier_trials": [], + "position": "first", + "relative_mad": 0.003463268482616659, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.7897805100000141, + "median_ns_per_operation": 139.116260754, + "median_operations_per_second": 7188232.307137015, + "operation": "lookup", + "outlier_trials": [ + 6 + ], + "position": "last", + "relative_mad": 0.00567712577752925, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.44572291400000097, + "median_ns_per_operation": 62.071871588, + "median_operations_per_second": 16110356.823739214, + "operation": "lookup", + "outlier_trials": [ + 3 + ], + "position": "middle", + "relative_mad": 0.007180755189056842, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.7867476809999943, + "median_ns_per_operation": 117.625287181, + "median_operations_per_second": 8501573.292324593, + "operation": "lookup", + "outlier_trials": [ + 3 + ], + "position": "miss", + "relative_mad": 0.00668859307258786, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.24258823300000287, + "median_ns_per_operation": 26.040184231, + "median_operations_per_second": 38402186.064779535, + "operation": "object", + "outlier_trials": [ + 4 + ], + "position": "none", + "relative_mad": 0.0093159184607922, + "trials": 11, + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.35530285899999825, + "median_ns_per_operation": 60.418653242, + "median_operations_per_second": 16551179.914497837, + "operation": "object", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.005880681543444428, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.6437584390000097, + "median_ns_per_operation": 176.268771708, + "median_operations_per_second": 5673154.639419404, + "operation": "object", + "outlier_trials": [ + 3, + 9 + ], + "position": "none", + "relative_mad": 0.0036521411748782987, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 3.2781385540000088, + "median_ns_per_operation": 427.119237978, + "median_operations_per_second": 2341266.585729177, + "operation": "object", + "outlier_trials": [ + 2, + 3 + ], + "position": "none", + "relative_mad": 0.007674996259870783, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 2.2218076930000734, + "median_ns_per_operation": 551.022701641, + "median_operations_per_second": 1814807.2611562123, + "operation": "object", + "outlier_trials": [ + 10 + ], + "position": "none", + "relative_mad": 0.0040321527341492655, + "trials": 11, + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 4.468098419000171, + "median_ns_per_operation": 1041.454417223, + "median_operations_per_second": 960195.6489526093, + "operation": "object", + "outlier_trials": [ + 7, + 11 + ], + "position": "none", + "relative_mad": 0.004290248661015996, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 26.832779941000354, + "median_ns_per_operation": 3633.926701007, + "median_operations_per_second": 275184.4168246127, + "operation": "object", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.0073839628998473485, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.5419845790000011, + "median_ns_per_operation": 26.024940156, + "median_operations_per_second": 38424680.09554489, + "operation": "object-new", + "outlier_trials": [ + 10 + ], + "position": "none", + "relative_mad": 0.020825584064793616, + "trials": 11, + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.37600082800000223, + "median_ns_per_operation": 54.626059032, + "median_operations_per_second": 18306281.24599285, + "operation": "object-new", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.006883176906094224, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.6085177429999931, + "median_ns_per_operation": 129.593470107, + "median_operations_per_second": 7716438.175274889, + "operation": "object-new", + "outlier_trials": [ + 9 + ], + "position": "none", + "relative_mad": 0.00469558954241726, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.433406768999987, + "median_ns_per_operation": 248.301978324, + "median_operations_per_second": 4027354.1384963808, + "operation": "object-new", + "outlier_trials": [ + 8 + ], + "position": "none", + "relative_mad": 0.0017454825447844424, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 1.9792149329999802, + "median_ns_per_operation": 356.865801498, + "median_operations_per_second": 2802173.802595664, + "operation": "object-new", + "outlier_trials": [ + 6, + 10 + ], + "position": "none", + "relative_mad": 0.005546104234958676, + "trials": 11, + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 2.0055915780000078, + "median_ns_per_operation": 490.718643803, + "median_operations_per_second": 2037827.6077919959, + "operation": "object-new", + "outlier_trials": [ + 11 + ], + "position": "none", + "relative_mad": 0.0040870498876035295, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 8.673606425000116, + "median_ns_per_operation": 1337.652840075, + "median_operations_per_second": 747578.1234419026, + "operation": "object-new", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.006484198414675216, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.1385427469999989, + "median_ns_per_operation": 31.35919419, + "median_operations_per_second": 31888574.494011894, + "operation": "replace", + "outlier_trials": [ + 1, + 4 + ], + "position": "first", + "relative_mad": 0.004417930708314508, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.12333883000000156, + "median_ns_per_operation": 31.520071623, + "median_operations_per_second": 31725816.23419619, + "operation": "replace", + "outlier_trials": [], + "position": "first", + "relative_mad": 0.003913025055120814, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.22153471600000074, + "median_ns_per_operation": 57.257526376, + "median_operations_per_second": 17464952.876817934, + "operation": "replace", + "outlier_trials": [], + "position": "last", + "relative_mad": 0.0038690933755192573, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.1631242300000011, + "median_ns_per_operation": 31.28989211, + "median_operations_per_second": 31959202.559231836, + "operation": "replace", + "outlier_trials": [ + 4, + 6 + ], + "position": "first", + "relative_mad": 0.005213320308888757, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.4337072670000168, + "median_ns_per_operation": 137.34761092, + "median_operations_per_second": 7280796.464544722, + "operation": "replace", + "outlier_trials": [ + 7, + 8 + ], + "position": "last", + "relative_mad": 0.0031577343362210763, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.12441915199999798, + "median_ns_per_operation": 22.854522605, + "median_operations_per_second": 43755015.90137021, + "operation": "string", + "outlier_trials": [ + 9 + ], + "position": "none", + "relative_mad": 0.005443961974194909, + "trials": 11, + "value_bytes": 8, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.2701618589999981, + "median_ns_per_operation": 22.903590611, + "median_operations_per_second": 43661276.39042439, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.011795611595949782, + "trials": 11, + "value_bytes": 16, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.19179157700000005, + "median_ns_per_operation": 22.274240136, + "median_operations_per_second": 44894909.720569246, + "operation": "string", + "outlier_trials": [ + 10 + ], + "position": "none", + "relative_mad": 0.008610465534580607, + "trials": 11, + "value_bytes": 31, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.3086031630000008, + "median_ns_per_operation": 29.346346691, + "median_operations_per_second": 34075791.80227848, + "operation": "string", + "outlier_trials": [ + 2 + ], + "position": "none", + "relative_mad": 0.010515897131912637, + "trials": 11, + "value_bytes": 32, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.33847543499999944, + "median_ns_per_operation": 29.539930337, + "median_operations_per_second": 33852483.3536069, + "operation": "string", + "outlier_trials": [ + 7 + ], + "position": "none", + "relative_mad": 0.011458234028942336, + "trials": 11, + "value_bytes": 33, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.2515065969999988, + "median_ns_per_operation": 29.706094651, + "median_operations_per_second": 33663125.757472694, + "operation": "string", + "outlier_trials": [ + 3 + ], + "position": "none", + "relative_mad": 0.008466498203644963, + "trials": 11, + "value_bytes": 64, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.5364782489999982, + "median_ns_per_operation": 30.146561721, + "median_operations_per_second": 33171278.676977716, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.01779566950171598, + "trials": 11, + "value_bytes": 127, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.24104181399999902, + "median_ns_per_operation": 30.278472213, + "median_operations_per_second": 33026765.451218903, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.0079608314549143, + "trials": 11, + "value_bytes": 128, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.33529365999999783, + "median_ns_per_operation": 30.335419257, + "median_operations_per_second": 32964766.08838187, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.01105287707281737, + "trials": 11, + "value_bytes": 129, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.14654486099999886, + "median_ns_per_operation": 29.867145992, + "median_operations_per_second": 33481605.516236898, + "operation": "string", + "outlier_trials": [ + 4 + ], + "position": "none", + "relative_mad": 0.004906557226433731, + "trials": 11, + "value_bytes": 208, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.06928563900000029, + "median_ns_per_operation": 29.836473566, + "median_operations_per_second": 33516025.202775467, + "operation": "string", + "outlier_trials": [ + 1, + 11 + ], + "position": "none", + "relative_mad": 0.0023221792229144124, + "trials": 11, + "value_bytes": 209, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.4538992689999972, + "median_ns_per_operation": 32.370962155, + "median_operations_per_second": 30891883.757169712, + "operation": "string", + "outlier_trials": [], + "position": "none", + "relative_mad": 0.01402180345541222, + "trials": 11, + "value_bytes": 256, + "width": 0 + }, + { + "case_sensitive": 0, + "mad_ns_per_operation": 0.28100638200000105, + "median_ns_per_operation": 88.935351156, + "median_operations_per_second": 11244122.691390928, + "operation": "string", + "outlier_trials": [ + 7 + ], + "position": "none", + "relative_mad": 0.003159670236271879, + "trials": 11, + "value_bytes": 4096, + "width": 0 + } + ] +} diff --git a/benchmarks/results/current-main-b.md b/benchmarks/results/current-main-b.md new file mode 100644 index 0000000..0ec4652 --- /dev/null +++ b/benchmarks/results/current-main-b.md @@ -0,0 +1,56 @@ +# libfastjson characterization: current-main-b + +Revision: `f1983a831a1abd6ab8d8cdbc5137f7bc997b4018` + +| Operation | Width | Position | Bytes | Median ns/op | M operations/s | MAD | Outliers | +|---|---:|---|---:|---:|---:|---:|---| +| lookup | 1 | first | 0 | 14.740 | 67.845 | 0.0030 | 4,5,7 | +| lookup | 1 | miss | 0 | 14.767 | 67.721 | 0.0022 | 3,10,11 | +| lookup | 4 | first | 0 | 14.610 | 68.446 | 0.0046 | - | +| lookup | 4 | last | 0 | 30.491 | 32.797 | 0.0221 | 6 | +| lookup | 4 | middle | 0 | 19.596 | 51.032 | 0.0037 | 5 | +| lookup | 4 | miss | 0 | 22.580 | 44.288 | 0.0035 | - | +| lookup | 8 | first | 0 | 14.750 | 67.798 | 0.0073 | - | +| lookup | 8 | last | 0 | 37.194 | 26.886 | 0.0253 | - | +| lookup | 8 | middle | 0 | 29.593 | 33.792 | 0.0164 | 7 | +| lookup | 8 | miss | 0 | 43.433 | 23.024 | 0.0105 | 11 | +| lookup | 16 | first | 0 | 14.763 | 67.738 | 0.0109 | - | +| lookup | 16 | last | 0 | 59.141 | 16.909 | 0.0075 | 4,6,8,9 | +| lookup | 16 | middle | 0 | 66.623 | 15.010 | 0.0044 | - | +| lookup | 16 | miss | 0 | 61.656 | 16.219 | 0.0078 | - | +| lookup | 32 | first | 0 | 14.803 | 67.552 | 0.0035 | - | +| lookup | 32 | last | 0 | 139.116 | 7.188 | 0.0057 | 6 | +| lookup | 32 | middle | 0 | 62.072 | 16.110 | 0.0072 | 3 | +| lookup | 32 | miss | 0 | 117.625 | 8.502 | 0.0067 | 3 | +| object | 0 | none | 0 | 26.040 | 38.402 | 0.0093 | 4 | +| object | 1 | none | 0 | 60.419 | 16.551 | 0.0059 | - | +| object | 4 | none | 0 | 176.269 | 5.673 | 0.0037 | 3,9 | +| object | 8 | none | 0 | 427.119 | 2.341 | 0.0077 | 2,3 | +| object | 9 | none | 0 | 551.023 | 1.815 | 0.0040 | 10 | +| object | 16 | none | 0 | 1041.454 | 0.960 | 0.0043 | 7,11 | +| object | 32 | none | 0 | 3633.927 | 0.275 | 0.0074 | - | +| object-new | 0 | none | 0 | 26.025 | 38.425 | 0.0208 | 10 | +| object-new | 1 | none | 0 | 54.626 | 18.306 | 0.0069 | - | +| object-new | 4 | none | 0 | 129.593 | 7.716 | 0.0047 | 9 | +| object-new | 8 | none | 0 | 248.302 | 4.027 | 0.0017 | 8 | +| object-new | 9 | none | 0 | 356.866 | 2.802 | 0.0055 | 6,10 | +| object-new | 16 | none | 0 | 490.719 | 2.038 | 0.0041 | 11 | +| object-new | 32 | none | 0 | 1337.653 | 0.748 | 0.0065 | - | +| replace | 1 | first | 0 | 31.359 | 31.889 | 0.0044 | 1,4 | +| replace | 8 | first | 0 | 31.520 | 31.726 | 0.0039 | - | +| replace | 8 | last | 0 | 57.258 | 17.465 | 0.0039 | - | +| replace | 32 | first | 0 | 31.290 | 31.959 | 0.0052 | 4,6 | +| replace | 32 | last | 0 | 137.348 | 7.281 | 0.0032 | 7,8 | +| string | 0 | none | 8 | 22.855 | 43.755 | 0.0054 | 9 | +| string | 0 | none | 16 | 22.904 | 43.661 | 0.0118 | - | +| string | 0 | none | 31 | 22.274 | 44.895 | 0.0086 | 10 | +| string | 0 | none | 32 | 29.346 | 34.076 | 0.0105 | 2 | +| string | 0 | none | 33 | 29.540 | 33.852 | 0.0115 | 7 | +| string | 0 | none | 64 | 29.706 | 33.663 | 0.0085 | 3 | +| string | 0 | none | 127 | 30.147 | 33.171 | 0.0178 | - | +| string | 0 | none | 128 | 30.278 | 33.027 | 0.0080 | - | +| string | 0 | none | 129 | 30.335 | 32.965 | 0.0111 | - | +| string | 0 | none | 208 | 29.867 | 33.482 | 0.0049 | 4 | +| string | 0 | none | 209 | 29.836 | 33.516 | 0.0023 | 1,11 | +| string | 0 | none | 256 | 32.371 | 30.892 | 0.0140 | - | +| string | 0 | none | 4096 | 88.935 | 11.244 | 0.0032 | 7 | diff --git a/benchmarks/results/current-main-findings.json b/benchmarks/results/current-main-findings.json new file mode 100644 index 0000000..060a0ec --- /dev/null +++ b/benchmarks/results/current-main-findings.json @@ -0,0 +1,40 @@ +{ + "characterization_revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018", + "complete_sessions": 2, + "measured_trials_per_workload_per_session": 11, + "case_sensitive": false, + "all_oracles_passed": true, + "workloads_per_session": 50, + "all_relative_mad_at_most_0_05": true, + "findings": { + "lookup_ns_per_operation": { + "width_1_first": [14.813, 14.740], + "width_8_last": [36.571, 37.194], + "width_16_last": [59.685, 59.141], + "width_32_last": [137.794, 139.116], + "width_32_miss": [117.778, 117.625] + }, + "string_ns_per_operation": { + "bytes_31": [21.739, 22.274], + "bytes_32": [29.416, 29.346], + "bytes_33": [29.513, 29.540], + "bytes_128": [29.616, 30.278], + "bytes_208": [29.960, 29.867], + "bytes_4096": [88.574, 88.935] + }, + "known_new_speedup": { + "width_1": [1.099, 1.106], + "width_4": [1.355, 1.360], + "width_8": [1.722, 1.720], + "width_16": [2.143, 2.122], + "width_32": [2.747, 2.717] + } + }, + "candidate_order": [ + "direct_child_page_lookup", + "larger_inline_string_threshold", + "safe_known_new_rsyslog_call_sites", + "hybrid_index_for_wide_objects_only" + ], + "runtime_candidate_accepted": false +} diff --git a/benchmarks/results/current-main-findings.md b/benchmarks/results/current-main-findings.md new file mode 100644 index 0000000..155f99e --- /dev/null +++ b/benchmarks/results/current-main-findings.md @@ -0,0 +1,85 @@ +# Current-main libfastjson performance findings + +Revision `f1983a831a1abd6ab8d8cdbc5137f7bc997b4018` was characterized in two +independent sessions. Each complete session contains 50 workloads and 11 +measured trials per workload. All correctness checks passed. Workloads whose +relative MAD exceeded 0.05 would have been rerun once; none exceeded 0.05. + +The benchmark uses libfastjson's case-insensitive comparison mode because that +is rsyslog's default for `$!` variables. Lookup objects use equal-length keys +named `key-NNN`, intentionally exercising keys with a shared prefix. + +## Lookup + +| Workload | Session A ns/op | Session B ns/op | Approximate cost relative to first key | +|---|---:|---:|---:| +| Width 1, first | 14.813 | 14.740 | 1.0x | +| Width 8, last | 36.571 | 37.194 | 2.5x | +| Width 16, last | 59.685 | 59.141 | 4.0x | +| Width 32, last | 137.794 | 139.116 | 9.3-9.4x | +| Width 32, miss | 117.778 | 117.625 | 8.0x | + +First-key lookup remained approximately 14.6-14.9 ns as width grew. Late and +missing keys scaled with the number and content of comparisons. This confirms +that direct internal child-page traversal is the strongest low-risk library +candidate. It should retain the current array representation, case behavior, +insertion order, deletion holes, and ownership rules. + +## Inline strings + +| Value length | Session A ns/op | Session B ns/op | +|---:|---:|---:| +| 16 bytes | 21.756 | 22.904 | +| 31 bytes | 21.739 | 22.274 | +| 32 bytes | 29.416 | 29.346 | +| 33 bytes | 29.513 | 29.540 | +| 128 bytes | 29.616 | 30.278 | +| 208 bytes | 29.960 | 29.867 | +| 4096 bytes | 88.574 | 88.935 | + +Crossing the current inline boundary from 31 to 32 bytes increased lifecycle +cost by 35.3% and 31.8% in the two sessions. Values from 32 through 208 bytes +then cost approximately 30-31 ns because they all take the external allocation +path. Enlarging the inline buffer is therefore a credible candidate, but its +threshold must be selected with 32-bit and 64-bit layout checks and measured as +a paired implementation rather than inferred from this baseline alone. + +## Writes and object construction + +Replacing the first key cost about 31 ns regardless of object width. Replacing +the last key cost about 58 ns at width 8 and 137-138 ns at width 32, again +showing lookup as the scaling term. + +Fresh object construction was also measured with and without libfastjson's +existing `FJSON_OBJECT_ADD_KEY_IS_NEW` flag: + +| Width | Normal A ns | Known-new A ns | Speedup A | Normal B ns | Known-new B ns | Speedup B | +|---:|---:|---:|---:|---:|---:|---:| +| 1 | 60.401 | 54.948 | 1.10x | 60.419 | 54.626 | 1.11x | +| 4 | 176.324 | 130.113 | 1.36x | 176.269 | 129.593 | 1.36x | +| 8 | 427.785 | 248.354 | 1.72x | 427.119 | 248.302 | 1.72x | +| 16 | 1047.242 | 488.741 | 2.14x | 1041.454 | 490.719 | 2.12x | +| 32 | 3646.321 | 1327.269 | 2.75x | 3633.927 | 1337.653 | 2.72x | + +This is an existing caller-controlled optimization, not a proposed semantic +change. Rsyslog may use it only where absence was already proven while holding +the applicable message/global lock or while constructing a private fresh +object. It is unsafe for ordinary replacement-capable writes. + +The 8-to-9-member object lifecycle step was approximately 29% in both complete +sessions, reflecting allocation of the second child page plus another value +and key. The embedded first page remains a deliberate allocation-avoidance +tradeoff and should not be removed based on this characterization. + +## Candidate order + +1. Implement direct internal page traversal in `_fjson_find_child()` and run + two paired sessions. +2. Independently test larger inline-string thresholds, including compile-time + object-size assertions on supported word sizes. +3. Audit rsyslog call sites for already-proven-new insertions that can safely + use `FJSON_OBJECT_ADD_KEY_IS_NEW`. +4. Consider hybrid indexing only after measuring wider, representative + objects; do not restore the former hash design by default. + +No runtime candidate has been implemented or accepted by these results. diff --git a/benchmarks/results/direct-lookup-a.json b/benchmarks/results/direct-lookup-a.json new file mode 100644 index 0000000..edbd9c7 --- /dev/null +++ b/benchmarks/results/direct-lookup-a.json @@ -0,0 +1,693 @@ +{ + "baseline_metadata": { + "architecture": "x86_64", + "cflags": "-O2 -g -fno-omit-frame-pointer", + "clock": "CLOCK_MONOTONIC_RAW", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "config_status_sha256": "bb25b3f70ff347a5f92b5e404fb5af0c4b33ae94766ddee0b8d4930bfa15ad8e", + "configure_args": "'CFLAGS=-O2 -g -fno-omit-frame-pointer'", + "cpu": "Intel(R) Core(TM) i7-14700K", + "harness_sha256": "2ec960faa7b4c2b859502af7da5f041ddbe69616521f70c3b95e27b9a2c97133", + "host_exclusive": false, + "kernel": "5.15.167.4-microsoft-standard-WSL2", + "label": "baseline-direct-a", + "library_sha256": "e143fd02577ad7d926db5d5effacb1c4b787eec2e3bdea281e56ef4c78721f2f", + "library_source_dirty": false, + "library_source_fingerprint": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "python": "3.12.3", + "revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018" + }, + "candidate_metadata": { + "architecture": "x86_64", + "cflags": "-O2 -g -fno-omit-frame-pointer", + "clock": "CLOCK_MONOTONIC_RAW", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "config_status_sha256": "cf93892d4c08a63918955ebbba91008effd8c4c6a9cc82400a9ff86f34965d7f", + "configure_args": "'CFLAGS=-O2 -g -fno-omit-frame-pointer'", + "cpu": "Intel(R) Core(TM) i7-14700K", + "harness_sha256": "2ec960faa7b4c2b859502af7da5f041ddbe69616521f70c3b95e27b9a2c97133", + "host_exclusive": false, + "kernel": "5.15.167.4-microsoft-standard-WSL2", + "label": "direct-lookup-a", + "library_sha256": "514bda148d17395acd9ff642029eec05fc9dbb6d61ccba6d14949b6ba96a0b09", + "library_source_dirty": true, + "library_source_fingerprint": "9047b06416fe428dfc5880263bfa341a95344212f5e3d57b1e379757f925fdd1", + "python": "3.12.3", + "revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018" + }, + "kind": "comparison", + "ratio_definition": "baseline ns/op divided by candidate ns/op; above 1 is faster", + "schema_version": 1, + "workloads": [ + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.0426774235622851, + "median_speedup_ratio": 3.119496007147787, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.013680871353736995, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.01683272253852719, + "median_speedup_ratio": 2.0522223668871664, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.008202192320932184, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.025547390481095, + "median_speedup_ratio": 3.079285047428362, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.008296533152210342, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.047002973393747816, + "median_speedup_ratio": 3.170653744089117, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.014824379193525306, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.013795259246164537, + "median_speedup_ratio": 2.2195027148002553, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.006215473022030543, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.013159962290652416, + "median_speedup_ratio": 2.2031963409391717, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.005973122797145999, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.04246427922636231, + "median_speedup_ratio": 3.13521384428743, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.013544300751202372, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.015275961607614796, + "median_speedup_ratio": 2.3125546070693903, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.006605665250419069, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.042332640418834, + "median_speedup_ratio": 2.699438590579472, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.015682016463188633, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.05678741146457922, + "median_speedup_ratio": 2.922594106703381, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.019430481753976474, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.018336951195633056, + "median_speedup_ratio": 3.157231465543376, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.005807921083947885, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.015501151788354761, + "median_speedup_ratio": 1.9042643332296763, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.008140231121203878, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.010730606461495817, + "median_speedup_ratio": 3.5192662708942186, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.00304910331742794, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.02229682782107245, + "median_speedup_ratio": 1.9955621558602492, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.011173206384774675, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.02644274170658978, + "median_speedup_ratio": 2.399758257477295, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.011018918936604581, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.016686439286822896, + "median_speedup_ratio": 2.3397796950987044, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.007131628384406068, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.00526625077531917, + "median_speedup_ratio": 1.8130209653220037, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.0029046827786593475, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.012017186149926307, + "median_speedup_ratio": 1.9419814370029602, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.006188105571427246, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.014596083113690428, + "median_speedup_ratio": 1.02266542271612, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.01427258885405978, + "trials": 11, + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.006024602065415996, + "median_speedup_ratio": 0.9763586902483886, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.006170480301540941, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.0036644899174111956, + "median_speedup_ratio": 1.0828870041317098, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.0033840002728165435, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.00688527927686633, + "median_speedup_ratio": 1.273579373252954, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.005406242768583846, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.0032284551707497666, + "median_speedup_ratio": 1.2006924919729558, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.0026888276493216247, + "trials": 11, + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.006726173298245319, + "median_speedup_ratio": 1.352399038896464, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.004973512332376227, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.01613339399144542, + "median_speedup_ratio": 1.6026086576594096, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.010066957965276468, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.019923669311061376, + "median_speedup_ratio": 1.0045101295364387, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.019834214434707342, + "trials": 11, + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.00867768374242639, + "median_speedup_ratio": 0.9758113585944719, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.008892788207472245, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.005022857034980621, + "median_speedup_ratio": 0.9906668593647384, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.005070177716655941, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.004485962067606097, + "median_speedup_ratio": 0.9889437218706071, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.004536114612387457, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.0065867467666800295, + "median_speedup_ratio": 0.996429965303598, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.0066103459310089515, + "trials": 11, + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.01671491915211032, + "median_speedup_ratio": 1.003124739680788, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.01666285207703012, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.014819238445099137, + "median_speedup_ratio": 1.0002620724963327, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.014815355747834245, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.010492819962606426, + "median_speedup_ratio": 1.604645929138755, + "operation": "replace", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.006539025072177841, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.014821800623090686, + "median_speedup_ratio": 1.6164985426263279, + "operation": "replace", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.009169077628124354, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.020288356281103814, + "median_speedup_ratio": 1.8450586657899162, + "operation": "replace", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.010996049425028909, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.013261811367197174, + "median_speedup_ratio": 1.6030749134426046, + "operation": "replace", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.008272733392550835, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.0019341913721415693, + "median_speedup_ratio": 1.8808714176249157, + "operation": "replace", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.0010283485378197643, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.008689269487332285, + "median_speedup_ratio": 0.9767769771563322, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.008895858205656271, + "trials": 11, + "value_bytes": 8, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.014615684545112861, + "median_speedup_ratio": 1.0041453583371334, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.014555347414358877, + "trials": 11, + "value_bytes": 16, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.021819208137381807, + "median_speedup_ratio": 0.9906502852841835, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.022025136883822353, + "trials": 11, + "value_bytes": 31, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.010124287852724345, + "median_speedup_ratio": 0.9922478529203573, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.010203386001720046, + "trials": 11, + "value_bytes": 32, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.011807615692583373, + "median_speedup_ratio": 0.990048488031976, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.011926300413886414, + "trials": 11, + "value_bytes": 33, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.019393277771989847, + "median_speedup_ratio": 0.9950598739103573, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.019489558649148125, + "trials": 11, + "value_bytes": 64, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.012994020467939515, + "median_speedup_ratio": 0.966697071464365, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.013441667355271913, + "trials": 11, + "value_bytes": 127, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.015113914145740814, + "median_speedup_ratio": 0.9945423107650382, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.015196853851410948, + "trials": 11, + "value_bytes": 128, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.02266745613917065, + "median_speedup_ratio": 0.9927234144545918, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.022833606832598267, + "trials": 11, + "value_bytes": 129, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.00935314262578757, + "median_speedup_ratio": 0.9963864849226325, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.009387062919178218, + "trials": 11, + "value_bytes": 208, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.0119513135692344, + "median_speedup_ratio": 0.992794066244318, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.01203805902511638, + "trials": 11, + "value_bytes": 209, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.007355629239509387, + "median_speedup_ratio": 1.0185093214789211, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.007221955739029157, + "trials": 11, + "value_bytes": 256, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.009651084546545952, + "median_speedup_ratio": 1.0017738828912977, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.009633994967697904, + "trials": 11, + "value_bytes": 4096, + "width": 0 + } + ] +} diff --git a/benchmarks/results/direct-lookup-a.md b/benchmarks/results/direct-lookup-a.md new file mode 100644 index 0000000..6627d6f --- /dev/null +++ b/benchmarks/results/direct-lookup-a.md @@ -0,0 +1,56 @@ +# libfastjson paired comparison + +baseline ns/op divided by candidate ns/op; above 1 is faster + +| Operation | Width | Position | Bytes | Speedup | MAD | >=10% | >5% regression | +|---|---:|---|---:|---:|---:|---|---| +| lookup | 1 | first | 0 | 3.1195 | 0.0427 | True | False | +| lookup | 1 | miss | 0 | 2.0522 | 0.0168 | True | False | +| lookup | 4 | first | 0 | 3.0793 | 0.0255 | True | False | +| lookup | 4 | last | 0 | 3.1707 | 0.0470 | True | False | +| lookup | 4 | middle | 0 | 2.2195 | 0.0138 | True | False | +| lookup | 4 | miss | 0 | 2.2032 | 0.0132 | True | False | +| lookup | 8 | first | 0 | 3.1352 | 0.0425 | True | False | +| lookup | 8 | last | 0 | 2.3126 | 0.0153 | True | False | +| lookup | 8 | middle | 0 | 2.6994 | 0.0423 | True | False | +| lookup | 8 | miss | 0 | 2.9226 | 0.0568 | True | False | +| lookup | 16 | first | 0 | 3.1572 | 0.0183 | True | False | +| lookup | 16 | last | 0 | 1.9043 | 0.0155 | True | False | +| lookup | 16 | middle | 0 | 3.5193 | 0.0107 | True | False | +| lookup | 16 | miss | 0 | 1.9956 | 0.0223 | True | False | +| lookup | 32 | first | 0 | 2.3998 | 0.0264 | True | False | +| lookup | 32 | last | 0 | 2.3398 | 0.0167 | True | False | +| lookup | 32 | middle | 0 | 1.8130 | 0.0053 | True | False | +| lookup | 32 | miss | 0 | 1.9420 | 0.0120 | True | False | +| object | 0 | none | 0 | 1.0227 | 0.0146 | False | False | +| object | 1 | none | 0 | 0.9764 | 0.0060 | False | False | +| object | 4 | none | 0 | 1.0829 | 0.0037 | False | False | +| object | 8 | none | 0 | 1.2736 | 0.0069 | True | False | +| object | 9 | none | 0 | 1.2007 | 0.0032 | True | False | +| object | 16 | none | 0 | 1.3524 | 0.0067 | True | False | +| object | 32 | none | 0 | 1.6026 | 0.0161 | True | False | +| object-new | 0 | none | 0 | 1.0045 | 0.0199 | False | False | +| object-new | 1 | none | 0 | 0.9758 | 0.0087 | False | False | +| object-new | 4 | none | 0 | 0.9907 | 0.0050 | False | False | +| object-new | 8 | none | 0 | 0.9889 | 0.0045 | False | False | +| object-new | 9 | none | 0 | 0.9964 | 0.0066 | False | False | +| object-new | 16 | none | 0 | 1.0031 | 0.0167 | False | False | +| object-new | 32 | none | 0 | 1.0003 | 0.0148 | False | False | +| replace | 1 | first | 0 | 1.6046 | 0.0105 | True | False | +| replace | 8 | first | 0 | 1.6165 | 0.0148 | True | False | +| replace | 8 | last | 0 | 1.8451 | 0.0203 | True | False | +| replace | 32 | first | 0 | 1.6031 | 0.0133 | True | False | +| replace | 32 | last | 0 | 1.8809 | 0.0019 | True | False | +| string | 0 | none | 8 | 0.9768 | 0.0087 | False | False | +| string | 0 | none | 16 | 1.0041 | 0.0146 | False | False | +| string | 0 | none | 31 | 0.9907 | 0.0218 | False | False | +| string | 0 | none | 32 | 0.9922 | 0.0101 | False | False | +| string | 0 | none | 33 | 0.9900 | 0.0118 | False | False | +| string | 0 | none | 64 | 0.9951 | 0.0194 | False | False | +| string | 0 | none | 127 | 0.9667 | 0.0130 | False | False | +| string | 0 | none | 128 | 0.9945 | 0.0151 | False | False | +| string | 0 | none | 129 | 0.9927 | 0.0227 | False | False | +| string | 0 | none | 208 | 0.9964 | 0.0094 | False | False | +| string | 0 | none | 209 | 0.9928 | 0.0120 | False | False | +| string | 0 | none | 256 | 1.0185 | 0.0074 | False | False | +| string | 0 | none | 4096 | 1.0018 | 0.0097 | False | False | diff --git a/benchmarks/results/direct-lookup-b-rerun.json b/benchmarks/results/direct-lookup-b-rerun.json new file mode 100644 index 0000000..e16a28b --- /dev/null +++ b/benchmarks/results/direct-lookup-b-rerun.json @@ -0,0 +1,251 @@ +{ + "baseline_metadata": { + "architecture": "x86_64", + "cflags": "-O2 -g -fno-omit-frame-pointer", + "clock": "CLOCK_MONOTONIC_RAW", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "config_status_sha256": "bb25b3f70ff347a5f92b5e404fb5af0c4b33ae94766ddee0b8d4930bfa15ad8e", + "configure_args": "'CFLAGS=-O2 -g -fno-omit-frame-pointer'", + "cpu": "Intel(R) Core(TM) i7-14700K", + "harness_sha256": "2ec960faa7b4c2b859502af7da5f041ddbe69616521f70c3b95e27b9a2c97133", + "host_exclusive": false, + "kernel": "5.15.167.4-microsoft-standard-WSL2", + "label": "baseline-direct-b-rerun", + "library_sha256": "e143fd02577ad7d926db5d5effacb1c4b787eec2e3bdea281e56ef4c78721f2f", + "library_source_dirty": false, + "library_source_fingerprint": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "python": "3.12.3", + "revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018" + }, + "candidate_metadata": { + "architecture": "x86_64", + "cflags": "-O2 -g -fno-omit-frame-pointer", + "clock": "CLOCK_MONOTONIC_RAW", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "config_status_sha256": "cf93892d4c08a63918955ebbba91008effd8c4c6a9cc82400a9ff86f34965d7f", + "configure_args": "'CFLAGS=-O2 -g -fno-omit-frame-pointer'", + "cpu": "Intel(R) Core(TM) i7-14700K", + "harness_sha256": "2ec960faa7b4c2b859502af7da5f041ddbe69616521f70c3b95e27b9a2c97133", + "host_exclusive": false, + "kernel": "5.15.167.4-microsoft-standard-WSL2", + "label": "direct-lookup-b-rerun", + "library_sha256": "514bda148d17395acd9ff642029eec05fc9dbb6d61ccba6d14949b6ba96a0b09", + "library_source_dirty": true, + "library_source_fingerprint": "9047b06416fe428dfc5880263bfa341a95344212f5e3d57b1e379757f925fdd1", + "python": "3.12.3", + "revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018" + }, + "kind": "comparison", + "ratio_definition": "baseline ns/op divided by candidate ns/op; above 1 is faster", + "schema_version": 1, + "workloads": [ + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.035851146488248276, + "median_speedup_ratio": 3.0995756730247748, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.011566469178428643, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.02505848541175615, + "median_speedup_ratio": 2.3422569772823763, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.010698435592165668, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.04336829132049447, + "median_speedup_ratio": 2.693703063381477, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.016099878234556816, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.03492321883777372, + "median_speedup_ratio": 2.7981829501769973, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.012480677446613943, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.04811688697576644, + "median_speedup_ratio": 3.1010944022814066, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.015516098748999034, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.03099993830876202, + "median_speedup_ratio": 1.9395377226986277, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.01598315822681161, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.006414856942445368, + "median_speedup_ratio": 3.5117402355555933, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.0018266889098163812, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.01671779882599278, + "median_speedup_ratio": 1.9864390925360582, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.00841596346387314, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.02554944404297821, + "median_speedup_ratio": 2.364288731567011, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.010806397586662126, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.24430904906011097, + "median_speedup_ratio": 2.3272932659114427, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.10497561808757767, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.019504751423190303, + "median_speedup_ratio": 1.8170658183925943, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.010734201934657777, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.3216093916152889, + "median_speedup_ratio": 2.0523374665147767, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.15670395189024988, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.1132502412510561, + "median_speedup_ratio": 1.4132388718872808, + "operation": "replace", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.08013524359106991, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.19000627328383057, + "median_speedup_ratio": 1.3654944199802481, + "operation": "replace", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.139148333749016, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.024507216947101673, + "median_speedup_ratio": 1.1913294378939183, + "operation": "replace", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.02057131819929385, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.029740602563780083, + "median_speedup_ratio": 1.5637820826694424, + "operation": "replace", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.019018380433808022, + "trials": 11, + "value_bytes": 0, + "width": 32 + } + ] +} diff --git a/benchmarks/results/direct-lookup-b-rerun.md b/benchmarks/results/direct-lookup-b-rerun.md new file mode 100644 index 0000000..0c962dc --- /dev/null +++ b/benchmarks/results/direct-lookup-b-rerun.md @@ -0,0 +1,22 @@ +# libfastjson paired comparison + +baseline ns/op divided by candidate ns/op; above 1 is faster + +| Operation | Width | Position | Bytes | Speedup | MAD | >=10% | >5% regression | +|---|---:|---|---:|---:|---:|---|---| +| lookup | 8 | first | 0 | 3.0996 | 0.0359 | True | False | +| lookup | 8 | last | 0 | 2.3423 | 0.0251 | True | False | +| lookup | 8 | middle | 0 | 2.6937 | 0.0434 | True | False | +| lookup | 8 | miss | 0 | 2.7982 | 0.0349 | True | False | +| lookup | 16 | first | 0 | 3.1011 | 0.0481 | True | False | +| lookup | 16 | last | 0 | 1.9395 | 0.0310 | True | False | +| lookup | 16 | middle | 0 | 3.5117 | 0.0064 | True | False | +| lookup | 16 | miss | 0 | 1.9864 | 0.0167 | True | False | +| lookup | 32 | first | 0 | 2.3643 | 0.0255 | True | False | +| lookup | 32 | last | 0 | 2.3273 | 0.2443 | True | False | +| lookup | 32 | middle | 0 | 1.8171 | 0.0195 | True | False | +| lookup | 32 | miss | 0 | 2.0523 | 0.3216 | True | False | +| replace | 8 | first | 0 | 1.4132 | 0.1133 | True | False | +| replace | 8 | last | 0 | 1.3655 | 0.1900 | True | False | +| replace | 32 | first | 0 | 1.1913 | 0.0245 | True | False | +| replace | 32 | last | 0 | 1.5638 | 0.0297 | True | False | diff --git a/benchmarks/results/direct-lookup-b.json b/benchmarks/results/direct-lookup-b.json new file mode 100644 index 0000000..7d76054 --- /dev/null +++ b/benchmarks/results/direct-lookup-b.json @@ -0,0 +1,693 @@ +{ + "baseline_metadata": { + "architecture": "x86_64", + "cflags": "-O2 -g -fno-omit-frame-pointer", + "clock": "CLOCK_MONOTONIC_RAW", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "config_status_sha256": "bb25b3f70ff347a5f92b5e404fb5af0c4b33ae94766ddee0b8d4930bfa15ad8e", + "configure_args": "'CFLAGS=-O2 -g -fno-omit-frame-pointer'", + "cpu": "Intel(R) Core(TM) i7-14700K", + "harness_sha256": "2ec960faa7b4c2b859502af7da5f041ddbe69616521f70c3b95e27b9a2c97133", + "host_exclusive": false, + "kernel": "5.15.167.4-microsoft-standard-WSL2", + "label": "baseline-direct-b", + "library_sha256": "e143fd02577ad7d926db5d5effacb1c4b787eec2e3bdea281e56ef4c78721f2f", + "library_source_dirty": false, + "library_source_fingerprint": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "python": "3.12.3", + "revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018" + }, + "candidate_metadata": { + "architecture": "x86_64", + "cflags": "-O2 -g -fno-omit-frame-pointer", + "clock": "CLOCK_MONOTONIC_RAW", + "compiler": "cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0", + "config_status_sha256": "cf93892d4c08a63918955ebbba91008effd8c4c6a9cc82400a9ff86f34965d7f", + "configure_args": "'CFLAGS=-O2 -g -fno-omit-frame-pointer'", + "cpu": "Intel(R) Core(TM) i7-14700K", + "harness_sha256": "2ec960faa7b4c2b859502af7da5f041ddbe69616521f70c3b95e27b9a2c97133", + "host_exclusive": false, + "kernel": "5.15.167.4-microsoft-standard-WSL2", + "label": "direct-lookup-b", + "library_sha256": "514bda148d17395acd9ff642029eec05fc9dbb6d61ccba6d14949b6ba96a0b09", + "library_source_dirty": true, + "library_source_fingerprint": "9047b06416fe428dfc5880263bfa341a95344212f5e3d57b1e379757f925fdd1", + "python": "3.12.3", + "revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018" + }, + "kind": "comparison", + "ratio_definition": "baseline ns/op divided by candidate ns/op; above 1 is faster", + "schema_version": 1, + "workloads": [ + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.024028489118366192, + "median_speedup_ratio": 3.1259396216759177, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.007686805257448875, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.013430861485636658, + "median_speedup_ratio": 2.096661492806964, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.006405832096270209, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.025206342345713395, + "median_speedup_ratio": 3.1394575977308015, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.008028884468429365, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.06817458084810912, + "median_speedup_ratio": 3.0032086491349226, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.022700580882965584, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.010163363730463892, + "median_speedup_ratio": 2.2736618469006715, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.004470041903688546, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.01997090791141254, + "median_speedup_ratio": 2.195308763087008, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.00909708385773023, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.011049301253809762, + "median_speedup_ratio": 3.1295912881201917, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.0035305892164745234, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.02143710719059655, + "median_speedup_ratio": 2.3626780832974186, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.009073223873426859, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.017635247152466782, + "median_speedup_ratio": 2.7174623904269746, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.006489601186236063, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.03808170082069795, + "median_speedup_ratio": 2.8301804133231507, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.01345557358867559, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.5195367674833578, + "median_speedup_ratio": 2.946573990871146, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.17631892804760632, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.1413777533342897, + "median_speedup_ratio": 1.8070224650912956, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.07823796110202035, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.175218667376146, + "median_speedup_ratio": 3.0098596220606577, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.058214896831695104, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.07175526154980205, + "median_speedup_ratio": 1.7987938832883301, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.03989076359245121, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.06808562563178566, + "median_speedup_ratio": 1.9511235869293062, + "operation": "lookup", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.034895598663198656, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.2030746081635426, + "median_speedup_ratio": 2.168434673039039, + "operation": "lookup", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.09365032329008816, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.18958151924690259, + "median_speedup_ratio": 1.9114600283959766, + "operation": "lookup", + "position": "middle", + "regresses_over_5_percent": false, + "relative_mad": 0.09918152429585048, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.09527686377149869, + "median_speedup_ratio": 1.6086765392314395, + "operation": "lookup", + "position": "miss", + "regresses_over_5_percent": false, + "relative_mad": 0.05922686223609509, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.021191191748180405, + "median_speedup_ratio": 1.0347835793883233, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.02047886357136325, + "trials": 11, + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.007174785813529816, + "median_speedup_ratio": 0.9958225024241181, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.007204884199809028, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.012549967865702394, + "median_speedup_ratio": 1.0852115764309698, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.011564535559947279, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.004533614097556482, + "median_speedup_ratio": 1.2663664653548783, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.0035800174922399034, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.05025019585851598, + "median_speedup_ratio": 1.1945621970585656, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.042065784420643584, + "trials": 11, + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.005166617860571376, + "median_speedup_ratio": 1.355758622341532, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.003810868524404518, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.011282871990450305, + "median_speedup_ratio": 1.5876095960100687, + "operation": "object", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.007106830305640675, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.017644409507072245, + "median_speedup_ratio": 0.998533993478825, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.01767031430307176, + "trials": 11, + "value_bytes": 0, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.009581252917793659, + "median_speedup_ratio": 0.9842094231459084, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.009734973769269881, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.011392655336718072, + "median_speedup_ratio": 1.0012818176595835, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.011378070724731121, + "trials": 11, + "value_bytes": 0, + "width": 4 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.0035065692352060918, + "median_speedup_ratio": 0.9910254969015624, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.0035383239343179038, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.004166505486345451, + "median_speedup_ratio": 0.9997630105102628, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.004167493138417808, + "trials": 11, + "value_bytes": 0, + "width": 9 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.006864350515782269, + "median_speedup_ratio": 0.9893319229942091, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.006938369576721369, + "trials": 11, + "value_bytes": 0, + "width": 16 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.0023106151729925806, + "median_speedup_ratio": 1.0023723832357339, + "operation": "object-new", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.0023051464821224822, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.03671235421111119, + "median_speedup_ratio": 1.1725945462705463, + "operation": "replace", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.03130865168005032, + "trials": 11, + "value_bytes": 0, + "width": 1 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.1625703066332631, + "median_speedup_ratio": 1.6212114746457298, + "operation": "replace", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.10027705156033902, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.0654256678500813, + "median_speedup_ratio": 1.8613133297618731, + "operation": "replace", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.035150270942534714, + "trials": 11, + "value_bytes": 0, + "width": 8 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.02069638419969988, + "median_speedup_ratio": 1.6028033025432074, + "operation": "replace", + "position": "first", + "regresses_over_5_percent": false, + "relative_mad": 0.012912616393328127, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": true, + "mad": 0.005697531720920912, + "median_speedup_ratio": 1.8726193555117705, + "operation": "replace", + "position": "last", + "regresses_over_5_percent": false, + "relative_mad": 0.0030425466361602496, + "trials": 11, + "value_bytes": 0, + "width": 32 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.007903858466376756, + "median_speedup_ratio": 0.9883442582581359, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.007997070251924735, + "trials": 11, + "value_bytes": 8, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.012634083432289778, + "median_speedup_ratio": 1.0020271235626848, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.012608524395397183, + "trials": 11, + "value_bytes": 16, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.01351044350682129, + "median_speedup_ratio": 0.9707741928236434, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.013917184456175255, + "trials": 11, + "value_bytes": 31, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.009108397574875915, + "median_speedup_ratio": 0.9851603333700322, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.009245599184569225, + "trials": 11, + "value_bytes": 32, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.01288480218448984, + "median_speedup_ratio": 0.9840733212901697, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.013093335532760117, + "trials": 11, + "value_bytes": 33, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.03719633186247928, + "median_speedup_ratio": 1.0392416888991114, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.03579180113711765, + "trials": 11, + "value_bytes": 64, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.010129990066467531, + "median_speedup_ratio": 1.0002389590108325, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.01012756999236002, + "trials": 11, + "value_bytes": 127, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.03187454998990946, + "median_speedup_ratio": 0.9902553570558552, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.032188212628989174, + "trials": 11, + "value_bytes": 128, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.021299459441245117, + "median_speedup_ratio": 0.9840969397113438, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.021643659869009138, + "trials": 11, + "value_bytes": 129, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.020136136527991733, + "median_speedup_ratio": 0.9857081276588032, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.0204280922140897, + "trials": 11, + "value_bytes": 208, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.028722260488134976, + "median_speedup_ratio": 0.9880932964978797, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.029068368938374445, + "trials": 11, + "value_bytes": 209, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.028955774446755456, + "median_speedup_ratio": 1.01462831376711, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.028538307135593837, + "trials": 11, + "value_bytes": 256, + "width": 0 + }, + { + "case_sensitive": 0, + "improves_at_least_10_percent": false, + "mad": 0.029107941836372997, + "median_speedup_ratio": 1.006903557317024, + "operation": "string", + "position": "none", + "regresses_over_5_percent": false, + "relative_mad": 0.028908371238585615, + "trials": 11, + "value_bytes": 4096, + "width": 0 + } + ] +} diff --git a/benchmarks/results/direct-lookup-b.md b/benchmarks/results/direct-lookup-b.md new file mode 100644 index 0000000..91d57e1 --- /dev/null +++ b/benchmarks/results/direct-lookup-b.md @@ -0,0 +1,56 @@ +# libfastjson paired comparison + +baseline ns/op divided by candidate ns/op; above 1 is faster + +| Operation | Width | Position | Bytes | Speedup | MAD | >=10% | >5% regression | +|---|---:|---|---:|---:|---:|---|---| +| lookup | 1 | first | 0 | 3.1259 | 0.0240 | True | False | +| lookup | 1 | miss | 0 | 2.0967 | 0.0134 | True | False | +| lookup | 4 | first | 0 | 3.1395 | 0.0252 | True | False | +| lookup | 4 | last | 0 | 3.0032 | 0.0682 | True | False | +| lookup | 4 | middle | 0 | 2.2737 | 0.0102 | True | False | +| lookup | 4 | miss | 0 | 2.1953 | 0.0200 | True | False | +| lookup | 8 | first | 0 | 3.1296 | 0.0110 | True | False | +| lookup | 8 | last | 0 | 2.3627 | 0.0214 | True | False | +| lookup | 8 | middle | 0 | 2.7175 | 0.0176 | True | False | +| lookup | 8 | miss | 0 | 2.8302 | 0.0381 | True | False | +| lookup | 16 | first | 0 | 2.9466 | 0.5195 | True | False | +| lookup | 16 | last | 0 | 1.8070 | 0.1414 | True | False | +| lookup | 16 | middle | 0 | 3.0099 | 0.1752 | True | False | +| lookup | 16 | miss | 0 | 1.7988 | 0.0718 | True | False | +| lookup | 32 | first | 0 | 1.9511 | 0.0681 | True | False | +| lookup | 32 | last | 0 | 2.1684 | 0.2031 | True | False | +| lookup | 32 | middle | 0 | 1.9115 | 0.1896 | True | False | +| lookup | 32 | miss | 0 | 1.6087 | 0.0953 | True | False | +| object | 0 | none | 0 | 1.0348 | 0.0212 | False | False | +| object | 1 | none | 0 | 0.9958 | 0.0072 | False | False | +| object | 4 | none | 0 | 1.0852 | 0.0125 | False | False | +| object | 8 | none | 0 | 1.2664 | 0.0045 | True | False | +| object | 9 | none | 0 | 1.1946 | 0.0503 | True | False | +| object | 16 | none | 0 | 1.3558 | 0.0052 | True | False | +| object | 32 | none | 0 | 1.5876 | 0.0113 | True | False | +| object-new | 0 | none | 0 | 0.9985 | 0.0176 | False | False | +| object-new | 1 | none | 0 | 0.9842 | 0.0096 | False | False | +| object-new | 4 | none | 0 | 1.0013 | 0.0114 | False | False | +| object-new | 8 | none | 0 | 0.9910 | 0.0035 | False | False | +| object-new | 9 | none | 0 | 0.9998 | 0.0042 | False | False | +| object-new | 16 | none | 0 | 0.9893 | 0.0069 | False | False | +| object-new | 32 | none | 0 | 1.0024 | 0.0023 | False | False | +| replace | 1 | first | 0 | 1.1726 | 0.0367 | True | False | +| replace | 8 | first | 0 | 1.6212 | 0.1626 | True | False | +| replace | 8 | last | 0 | 1.8613 | 0.0654 | True | False | +| replace | 32 | first | 0 | 1.6028 | 0.0207 | True | False | +| replace | 32 | last | 0 | 1.8726 | 0.0057 | True | False | +| string | 0 | none | 8 | 0.9883 | 0.0079 | False | False | +| string | 0 | none | 16 | 1.0020 | 0.0126 | False | False | +| string | 0 | none | 31 | 0.9708 | 0.0135 | False | False | +| string | 0 | none | 32 | 0.9852 | 0.0091 | False | False | +| string | 0 | none | 33 | 0.9841 | 0.0129 | False | False | +| string | 0 | none | 64 | 1.0392 | 0.0372 | False | False | +| string | 0 | none | 127 | 1.0002 | 0.0101 | False | False | +| string | 0 | none | 128 | 0.9903 | 0.0319 | False | False | +| string | 0 | none | 129 | 0.9841 | 0.0213 | False | False | +| string | 0 | none | 208 | 0.9857 | 0.0201 | False | False | +| string | 0 | none | 209 | 0.9881 | 0.0287 | False | False | +| string | 0 | none | 256 | 1.0146 | 0.0290 | False | False | +| string | 0 | none | 4096 | 1.0069 | 0.0291 | False | False | diff --git a/benchmarks/results/direct-lookup-findings.json b/benchmarks/results/direct-lookup-findings.json new file mode 100644 index 0000000..5d047a5 --- /dev/null +++ b/benchmarks/results/direct-lookup-findings.json @@ -0,0 +1,31 @@ +{ + "baseline_revision": "f1983a831a1abd6ab8d8cdbc5137f7bc997b4018", + "candidate": "direct_child_page_lookup", + "candidate_source_fingerprint": "9047b06416fe428dfc5880263bfa341a95344212f5e3d57b1e379757f925fdd1", + "complete_sessions": 2, + "measured_pairs_per_workload_per_session": 11, + "workloads_per_complete_session": 50, + "ratio_definition": "baseline_ns_per_operation / candidate_ns_per_operation", + "acceptance": { + "minimum_target_improvement_percent": 10, + "maximum_core_regression_percent": 5, + "maximum_relative_mad": 0.05, + "retained": false + }, + "representative_results": { + "lookup_width_1_first": [3.1195, 3.1259], + "lookup_width_8_last": [2.3126, 2.3627], + "lookup_width_16_last": [1.9043, 1.9395], + "replace_width_1_first": [1.6046, 1.1726], + "replace_width_8_last": [1.8451, 1.8613], + "replace_width_32_last": [1.8809, 1.8726] + }, + "minimum_core_guardrail_ratio": [0.9667, 0.9708], + "inconclusive_after_one_rerun": [ + "lookup_width_32_last", + "lookup_width_32_miss", + "replace_width_8_first" + ], + "observable_semantics_or_public_api_changed": false, + "rsyslog_end_to_end_validated": false +} diff --git a/benchmarks/results/direct-lookup-findings.md b/benchmarks/results/direct-lookup-findings.md new file mode 100644 index 0000000..84d370e --- /dev/null +++ b/benchmarks/results/direct-lookup-findings.md @@ -0,0 +1,43 @@ +# Direct child-page lookup results + +The candidate replaces iterator API calls inside the private +`_fjson_find_child()` function with direct traversal of the existing child +pages. It preserves the page representation, insertion order, deleted slots, +case-sensitive and case-insensitive comparisons, and key/value ownership. + +Baseline and candidate were built from revision +`f1983a831a1abd6ab8d8cdbc5137f7bc997b4018` with +`-O2 -g -fno-omit-frame-pointer`. Each complete paired session used one +alternating calibration pair followed by 11 alternating measured pairs for +all 50 workloads. The host was non-exclusive and cache state was uncontrolled. +Ratios are baseline ns/op divided by candidate ns/op; above 1 is faster. + +| Profile | Session A speedup (relative MAD) | Session B speedup (relative MAD) | Result | +|---|---:|---:|---| +| Lookup width 1, first | 3.1195x (0.0137) | 3.1259x (0.0077) | pass | +| Lookup width 8, last | 2.3126x (0.0066) | 2.3627x (0.0091) | pass | +| Lookup width 16, last | 1.9043x (0.0081) | 1.9395x (0.0160), rerun | pass | +| Lookup width 32, last | 2.3398x (0.0071) | 2.3273x (0.1050), rerun | inconclusive dispersion | +| Lookup width 32, miss | 1.9420x (0.0062) | 2.0523x (0.1567), rerun | inconclusive dispersion | +| Replace width 1, first | 1.6046x (0.0065) | 1.1726x (0.0313) | pass | +| Replace width 8, last | 1.8451x (0.0110) | 1.8613x (0.0352) | pass | +| Replace width 32, last | 1.8809x (0.0010) | 1.8726x (0.0030) | pass | + +Session B initially had seven workloads above the relative-MAD threshold of +0.05. A focused rerun made four conclusive. Lookup width 32 last/miss and +replace width 8 first remained noisy and are classified as inconclusive. Their +median ratios still favored the candidate, but they are not used as acceptance +evidence. + +No workload in either complete session regressed by more than 5%. The lowest +core guardrail ratios were 0.9667 and 0.9708, both string workloads independent +of object lookup. Normal object construction improved as duplicate checks grew, +while known-new construction remained effectively neutral as expected. + +The candidate is not retained yet: three workloads remained inconclusive after +one rerun, exceeding the documented relative-MAD limit. Stable targeted read +and replacement profiles beat the required 10% improvement in both independent +sessions, and every core guardrail stayed within the 5% regression limit. A +clean-tree rerun is needed before treating this as retained standalone +libfastjson evidence; an rsyslog end-to-end run remains a separate integration +check before attributing the full library speedup to `$!varname` throughput. diff --git a/benchmarks/run.sh b/benchmarks/run.sh new file mode 100755 index 0000000..7c91af7 --- /dev/null +++ b/benchmarks/run.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Stable entry point for the libfastjson benchmark runner. +set -eu + +SCRIPT_DIR=$(dirname -- "$0") +SCRIPT_DIR=$(CDPATH='' cd -- "$SCRIPT_DIR" && pwd) +exec python3 "$SCRIPT_DIR/runner.py" "$@" diff --git a/benchmarks/runner.py b/benchmarks/runner.py new file mode 100755 index 0000000..19d6d08 --- /dev/null +++ b/benchmarks/runner.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +"""Paired, resumable libfastjson microbenchmark runner and report generator.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import tempfile + + +LOOKUP_WIDTHS = (1, 4, 8, 16, 32) +STRING_LENGTHS = (8, 16, 31, 32, 33, 64, 127, 128, 129, 208, 209, 256, 4096) +OBJECT_WIDTHS = (0, 1, 4, 8, 9, 16, 32) +OPERATIONS = ("lookup", "replace", "string", "object", "object-new") +POSITIONS = ("first", "middle", "last", "miss") +SCHEMA_VERSION = 1 + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--build-dir", required=True, type=Path) + parser.add_argument("--label", default="baseline") + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--pair-build-dir", type=Path) + parser.add_argument("--pair-label", default="candidate") + parser.add_argument("--pair-output", type=Path) + parser.add_argument("--trials", type=int, default=11) + parser.add_argument("--calibrations", type=int, choices=(1,), default=1) + parser.add_argument("--target-ms", type=int, default=200) + parser.add_argument("--operation", action="append", choices=OPERATIONS) + parser.add_argument("--width", action="append", type=int) + parser.add_argument("--position", action="append", choices=POSITIONS) + parser.add_argument("--value-bytes", action="append", type=int) + parser.add_argument("--case-sensitive", action="store_true") + parser.add_argument("--smoke", action="store_true") + parser.add_argument("--summary-json", type=Path) + parser.add_argument("--summary-markdown", type=Path) + parser.add_argument("--comparison-json", type=Path) + parser.add_argument("--comparison-markdown", type=Path) + args = parser.parse_args(argv) + if args.trials < 1 or args.target_ms < 10: + parser.error("trials must be positive and target-ms at least 10") + if bool(args.pair_build_dir) != bool(args.pair_output): + parser.error("--pair-build-dir and --pair-output must be provided together") + if args.pair_build_dir and args.label == args.pair_label: + parser.error("paired labels must differ") + if args.output.resolve() == (args.pair_output.resolve() if args.pair_output else None): + parser.error("paired output paths must differ") + if bool(args.comparison_json) != bool(args.pair_build_dir): + parser.error("--comparison-json requires paired mode") + if bool(args.comparison_markdown) != bool(args.pair_build_dir): + parser.error("--comparison-markdown requires paired mode") + if args.smoke: + args.trials = 1 + args.calibrations = 1 + args.target_ms = 20 + return args + + +def workloads(operations=None, widths=None, positions=None, value_bytes=None, case_sensitive=False, smoke=False): + selected = set(operations or OPERATIONS) + width_filter = set(widths or ()) + position_filter = set(positions or ()) + value_filter = set(value_bytes or ()) + output = [] + + if "lookup" in selected: + lookup_widths = (1, 8) if smoke else LOOKUP_WIDTHS + for width in lookup_widths: + valid_positions = ("first", "miss") if width == 1 else POSITIONS + for position in valid_positions: + if width_filter and width not in width_filter: + continue + if position_filter and position not in position_filter: + continue + output.append(workload("lookup", width, position, 0, case_sensitive)) + if "replace" in selected: + replace_widths = (1, 8) if smoke else (1, 8, 32) + for width in replace_widths: + valid_positions = ("first",) if width == 1 else ("first", "last") + for position in valid_positions: + if width_filter and width not in width_filter: + continue + if position_filter and position not in position_filter: + continue + output.append(workload("replace", width, position, 0, case_sensitive)) + if "string" in selected: + lengths = (31, 32, 33, 128) if smoke else STRING_LENGTHS + for length in lengths: + if value_filter and length not in value_filter: + continue + output.append(workload("string", 0, "none", length, case_sensitive)) + for operation in ("object", "object-new"): + if operation in selected: + object_widths = (0, 1, 8, 9) if smoke else OBJECT_WIDTHS + for width in object_widths: + if width_filter and width not in width_filter: + continue + output.append(workload(operation, width, "none", 0, case_sensitive)) + if not output: + raise SystemExit("filters selected no workloads") + return output + + +def workload(operation, width, position, value_bytes, case_sensitive): + return { + "operation": operation, + "width": width, + "position": position, + "value_bytes": value_bytes, + "case_sensitive": int(case_sensitive), + } + + +def workload_key(record): + return tuple(record[name] for name in + ("operation", "width", "position", "value_bytes", "case_sensitive")) + + +def command_output(command, cwd=None): + return subprocess.run(command, cwd=cwd, check=True, text=True, stdout=subprocess.PIPE).stdout.strip() + + +def cpu_model(): + try: + for line in Path("/proc/cpuinfo").read_text().splitlines(): + if line.startswith("model name"): + return line.split(":", 1)[1].strip() + except OSError: + pass + return platform.processor() or platform.machine() + + +def file_hash(path): + digest = hashlib.sha256() + with open(path, "rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def build_metadata(build_dir, label, harness_hash): + build_dir = build_dir.resolve() + revision = command_output(["git", "rev-parse", "HEAD"], build_dir) + tracked_diff = subprocess.run( + ["git", "diff", "--binary", "HEAD", "--", "*.c", "*.h", "Makefile.am", "configure.ac"], + cwd=build_dir, check=True, stdout=subprocess.PIPE).stdout + + # A fingerprint alone cannot reproduce an uncommitted source tree. Refuse + # to publish benchmark records unless both builds are committed states. + if tracked_diff: + raise SystemExit("build directory has uncommitted library sources; benchmark a clean commit") + config_status = Path(build_dir, "config.status") + library = Path(build_dir, ".libs", "libfastjson.so") + if not library.exists(): + raise SystemExit("build directory lacks .libs/libfastjson.so; run configure and make first") + return { + "label": label, + "revision": revision, + "library_source_fingerprint": hashlib.sha256(tracked_diff).hexdigest(), + "library_source_dirty": bool(tracked_diff), + "library_sha256": file_hash(library.resolve()), + "config_status_sha256": file_hash(config_status) if config_status.exists() else None, + "harness_sha256": harness_hash, + "architecture": platform.machine(), + "cpu": cpu_model(), + "kernel": platform.release(), + "python": platform.python_version(), + "clock": "CLOCK_MONOTONIC_RAW", + "host_exclusive": False, + "compiler": command_output([os.environ.get("CC", "cc"), "--version"]).splitlines()[0], + "configure_args": command_output([str(config_status), "--config"], build_dir) + if config_status.exists() else None, + "cflags": next((line.split("=", 1)[1].strip() for line in Path(build_dir, "Makefile").read_text().splitlines() + if line.startswith("CFLAGS =")), None), + } + + +def compile_benchmark(build_dir, output, source): + output.parent.mkdir(parents=True, exist_ok=True) + compiler = os.environ.get("CC", "cc") + command = [compiler, "-O2", "-std=c99", "-Wall", "-Wextra", "-Werror", + "-I", str(build_dir.resolve()), "-I", str(source.parent.parent.resolve()), + str(source.resolve()), str(Path(build_dir, ".libs", "libfastjson.so").resolve()), + "-Wl,-rpath," + str(Path(build_dir, ".libs").resolve()), "-lm", "-o", str(output)] + subprocess.run(command, check=True) + + +def run_once(binary, spec, iterations): + command = [str(binary), spec["operation"], str(spec["width"]), spec["position"], + str(spec["value_bytes"]), str(iterations), str(spec["case_sensitive"])] + result = subprocess.run(command, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + record = json.loads(result.stdout) + if not record.get("oracle"): + raise RuntimeError("benchmark correctness oracle failed") + return record + + +def calibrated_iterations(binary_specs, spec, target_ms): + seed = 10000 + suggestions = [] + calibration_records = [] + for label, binary in binary_specs: + record = run_once(binary, spec, seed) + record.update({"label": label, "measured": False, "trial": 0}) + calibration_records.append(record) + elapsed = max(record["elapsed_ns"], 1) + suggestion = int(seed * target_ms * 1000000 / elapsed) + suggestions.append(max(1000, min(suggestion, 100000000))) + return max(suggestions), calibration_records + + +def atomic_json(path, document): + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile("w", dir=path.parent, delete=False) as stream: + json.dump(document, stream, indent=2, sort_keys=True) + stream.write("\n") + temporary = Path(stream.name) + temporary.replace(path) + + +def load_or_create(path, metadata, config): + if not path.exists(): + return {"schema_version": SCHEMA_VERSION, "metadata": metadata, "config": config, + "calibrations": [], "records": []} + document = json.loads(path.read_text()) + if document.get("schema_version") != SCHEMA_VERSION: + raise SystemExit(f"cannot resume {path}: schema version differs") + for name in ("revision", "library_source_fingerprint", "library_sha256", "config_status_sha256", + "harness_sha256", "architecture", "cpu", "kernel", "python", "clock", "host_exclusive", + "compiler", "configure_args", "cflags"): + if document["metadata"].get(name) != metadata.get(name): + raise SystemExit(f"cannot resume {path}: metadata field {name} differs") + if document.get("config") != config: + raise SystemExit(f"cannot resume {path}: benchmark configuration differs") + return document + + +def median_mad(values): + median = statistics.median(values) + mad = statistics.median(abs(value - median) for value in values) + return median, mad + + +def outlier_trials(records): + values = [record["ns_per_operation"] for record in records] + median, mad = median_mad(values) + if mad == 0: + return [] + return [record["trial"] for record in records + if abs(record["ns_per_operation"] - median) / mad > 3.5] + + +def trial_order(items, trial): + """Alternate paired execution order without mutating the input list.""" + return list(items) if trial % 2 else list(reversed(items)) + + +def validate_complete(document): + """Reject partial, duplicate, or unexpected measured records before reporting.""" + expected_keys = {workload_key(spec) for spec in document["config"]["workloads"]} + expected_trials = set(range(1, document["config"]["trials"] + 1)) + grouped = {} + for record in document["records"]: + grouped.setdefault(workload_key(record), []).append(record["trial"]) + if set(grouped) != expected_keys: + raise SystemExit("benchmark document does not contain the configured workload set") + for key, trials in grouped.items(): + if len(trials) != len(set(trials)): + raise SystemExit(f"benchmark document contains duplicate trials for {key}") + if set(trials) != expected_trials: + raise SystemExit(f"benchmark document is incomplete for {key}") + + +def summarize(document): + validate_complete(document) + grouped = {} + for record in document["records"]: + grouped.setdefault(workload_key(record), []).append(record) + rows = [] + for key, records in sorted(grouped.items()): + values = [record["ns_per_operation"] for record in records] + median, mad = median_mad(values) + row = dict(zip(("operation", "width", "position", "value_bytes", "case_sensitive"), key)) + row.update({ + "trials": len(records), + "median_ns_per_operation": median, + "median_operations_per_second": 1000000000.0 / median, + "mad_ns_per_operation": mad, + "relative_mad": mad / median if median else 0, + "outlier_trials": outlier_trials(records), + }) + rows.append(row) + return {"schema_version": SCHEMA_VERSION, "kind": "characterization", + "metadata": document["metadata"], "config": document["config"], "workloads": rows} + + +def compare(baseline, candidate): + validate_complete(baseline) + validate_complete(candidate) + base_group = {} + candidate_group = {} + for record in baseline["records"]: + base_group.setdefault(workload_key(record), {})[record["trial"]] = record + for record in candidate["records"]: + candidate_group.setdefault(workload_key(record), {})[record["trial"]] = record + if set(base_group) != set(candidate_group): + raise SystemExit("paired documents contain different workloads") + rows = [] + for key in sorted(base_group): + trials = sorted(base_group[key]) + ratios = [base_group[key][trial]["ns_per_operation"] / + candidate_group[key][trial]["ns_per_operation"] for trial in trials] + median, mad = median_mad(ratios) + row = dict(zip(("operation", "width", "position", "value_bytes", "case_sensitive"), key)) + row.update({"trials": len(trials), "median_speedup_ratio": median, "mad": mad, + "relative_mad": mad / median if median else 0, + "improves_at_least_10_percent": median >= 1.10, + "regresses_over_5_percent": median < 0.95}) + rows.append(row) + return {"schema_version": SCHEMA_VERSION, "kind": "comparison", + "ratio_definition": "baseline ns/op divided by candidate ns/op; above 1 is faster", + "baseline_metadata": baseline["metadata"], "candidate_metadata": candidate["metadata"], + "workloads": rows} + + +def markdown(report): + if report["kind"] == "characterization": + lines = [f"# libfastjson characterization: {report['metadata']['label']}", "", + f"Revision: `{report['metadata']['revision']}`", "", + "| Operation | Width | Position | Bytes | Median ns/op | M operations/s | MAD | Outliers |", + "|---|---:|---|---:|---:|---:|---:|---|"] + for row in report["workloads"]: + lines.append("| {operation} | {width} | {position} | {value_bytes} | {median_ns_per_operation:.3f} | " + "{mops:.3f} | {relative_mad:.4f} | {outliers} |".format( + mops=row["median_operations_per_second"] / 1000000, + outliers=",".join(map(str, row["outlier_trials"])) or "-", **row)) + else: + lines = ["# libfastjson paired comparison", "", report["ratio_definition"], "", + "| Operation | Width | Position | Bytes | Speedup | MAD | >=10% | >5% regression |", + "|---|---:|---|---:|---:|---:|---|---|"] + for row in report["workloads"]: + lines.append("| {operation} | {width} | {position} | {value_bytes} | {median_speedup_ratio:.4f} | " + "{mad:.4f} | {improves_at_least_10_percent} | {regresses_over_5_percent} |".format(**row)) + return "\n".join(lines) + "\n" + + +def write_report(path, content): + if path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + +def main(argv=None): + args = parse_args(argv) + source = Path(__file__).with_name("benchmark.c") + harness_hash = hashlib.sha256(source.read_bytes() + Path(__file__).read_bytes()).hexdigest() + matrix = workloads(args.operation, args.width, args.position, args.value_bytes, + args.case_sensitive, args.smoke) + config = {"trials": args.trials, "calibrations": args.calibrations, "target_ms": args.target_ms, + "workloads": matrix} + + primary_metadata = build_metadata(args.build_dir, args.label, harness_hash) + primary = load_or_create(args.output, primary_metadata, config) + artifact_root = args.output.parent / "artifacts" + primary_binary = artifact_root / args.label / "benchmark" + compile_benchmark(args.build_dir, primary_binary, source) + builds = [(args.label, primary_binary, primary, args.output)] + + if args.pair_build_dir: + pair_metadata = build_metadata(args.pair_build_dir, args.pair_label, harness_hash) + pair = load_or_create(args.pair_output, pair_metadata, config) + pair_binary = artifact_root / args.pair_label / "benchmark" + compile_benchmark(args.pair_build_dir, pair_binary, source) + builds.append((args.pair_label, pair_binary, pair, args.pair_output)) + + for spec in matrix: + calibration_matches = [] + for label, binary, document, path in builds: + matches = [record for record in document["calibrations"] + if workload_key(record) == workload_key(spec)] + if len(matches) > 1: + raise SystemExit(f"duplicate calibration records for {label} and {workload_key(spec)}") + calibration_matches.append((label, binary, document, path, matches)) + existing_iterations = {matches[0]["selected_iterations"] + for _, _, _, _, matches in calibration_matches if matches} + if len(existing_iterations) > 1: + raise SystemExit(f"inconsistent calibrated iterations for {workload_key(spec)}") + if existing_iterations: + iterations = existing_iterations.pop() + for label, binary, document, path, matches in calibration_matches: + if matches: + continue + record = run_once(binary, spec, 10000) + record.update({"label": label, "measured": False, "trial": 0, + "selected_iterations": iterations}) + document["calibrations"].append(record) + atomic_json(path, document) + else: + binary_specs = [(label, binary) for label, binary, _, _ in builds] + iterations, calibration_records = calibrated_iterations(binary_specs, spec, args.target_ms) + for record in calibration_records: + record["selected_iterations"] = iterations + for name, value in spec.items(): + record[name] = value + for label, _, document, path in builds: + if record["label"] == label: + document["calibrations"].append(record) + atomic_json(path, document) + + for trial in range(1, args.trials + 1): + order = trial_order(builds, trial) + for label, binary, document, path in order: + if any(workload_key(record) == workload_key(spec) and record["trial"] == trial + for record in document["records"]): + continue + record = run_once(binary, spec, iterations) + record.update({"label": label, "measured": True, "trial": trial}) + document["records"].append(record) + atomic_json(path, document) + + primary_summary = summarize(primary) + write_report(args.summary_json, json.dumps(primary_summary, indent=2, sort_keys=True) + "\n") + write_report(args.summary_markdown, markdown(primary_summary)) + if len(builds) == 2: + comparison = compare(primary, builds[1][2]) + write_report(args.comparison_json, json.dumps(comparison, indent=2, sort_keys=True) + "\n") + write_report(args.comparison_markdown, markdown(comparison)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/selftest.py b/benchmarks/selftest.py new file mode 100755 index 0000000..17f28eb --- /dev/null +++ b/benchmarks/selftest.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Deterministic tests for libfastjson benchmark plumbing.""" + +import contextlib +import io +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).parent)) +import runner # noqa: E402 + + +class BenchmarkTests(unittest.TestCase): + def test_full_matrix_is_stable_and_covers_boundaries(self): + matrix = runner.workloads() + self.assertEqual(len(matrix), 50) + self.assertIn(runner.workload("lookup", 32, "miss", 0, False), matrix) + self.assertIn(runner.workload("string", 0, "none", 31, False), matrix) + self.assertIn(runner.workload("string", 0, "none", 32, False), matrix) + self.assertIn(runner.workload("object", 9, "none", 0, False), matrix) + self.assertIn(runner.workload("object-new", 9, "none", 0, False), matrix) + + def test_smoke_matrix_is_reduced(self): + matrix = runner.workloads(smoke=True) + self.assertLess(len(matrix), len(runner.workloads())) + self.assertEqual({entry["operation"] for entry in matrix}, set(runner.OPERATIONS)) + + def test_filters(self): + matrix = runner.workloads(["lookup"], [8], ["last"], None) + self.assertEqual(matrix, [runner.workload("lookup", 8, "last", 0, False)]) + + def test_median_mad_and_outliers(self): + median, mad = runner.median_mad([10, 10, 11, 12, 100]) + self.assertEqual(median, 11) + self.assertEqual(mad, 1) + records = [{"trial": index + 1, "ns_per_operation": value} + for index, value in enumerate([10, 10, 11, 12, 100])] + self.assertEqual(runner.outlier_trials(records), [5]) + + def test_alternating_order_does_not_mutate(self): + builds = ["baseline", "candidate"] + self.assertEqual(runner.trial_order(builds, 1), builds) + self.assertEqual(runner.trial_order(builds, 2), list(reversed(builds))) + self.assertEqual(builds, ["baseline", "candidate"]) + + def test_comparison_ratio(self): + metadata = {"label": "x"} + spec = runner.workload("lookup", 1, "first", 0, False) + config = {"trials": 3, "workloads": [spec]} + baseline_records = [] + candidate_records = [] + for trial, baseline_ns, candidate_ns in ((1, 20, 10), (2, 22, 11), (3, 18, 9)): + baseline_records.append(dict(spec, trial=trial, ns_per_operation=baseline_ns)) + candidate_records.append(dict(spec, trial=trial, ns_per_operation=candidate_ns)) + baseline = {"metadata": metadata, "config": config, "records": baseline_records} + candidate = {"metadata": metadata, "config": config, "records": candidate_records} + row = runner.compare(baseline, candidate)["workloads"][0] + self.assertEqual(row["median_speedup_ratio"], 2) + self.assertEqual(row["mad"], 0) + + def test_reports_reject_incomplete_records(self): + spec = runner.workload("lookup", 1, "first", 0, False) + document = {"metadata": {}, "config": {"trials": 2, "workloads": [spec]}, + "records": [dict(spec, trial=1, ns_per_operation=10)]} + with self.assertRaises(SystemExit): + runner.summarize(document) + + def test_resume_rejects_changed_host_metadata(self): + fields = ("revision", "library_source_fingerprint", "library_sha256", "config_status_sha256", + "harness_sha256", "architecture", "cpu", "kernel", "python", "clock", "host_exclusive", + "compiler", "configure_args", "cflags") + metadata = {name: name for name in fields} + config = {"trials": 1} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory, "raw.json") + runner.atomic_json(path, {"schema_version": runner.SCHEMA_VERSION, + "metadata": metadata, "config": config, + "calibrations": [], "records": []}) + changed = dict(metadata, kernel="different") + with self.assertRaises(SystemExit): + runner.load_or_create(path, changed, config) + + def test_invalid_pair_arguments(self): + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + runner.parse_args(["--build-dir", ".", "--output", "a", "--pair-build-dir", "."]) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + runner.parse_args(["--build-dir", ".", "--output", "a", "--label", "same", + "--pair-build-dir", ".", "--pair-output", "b", "--pair-label", "same"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/json_object.c b/json_object.c index 8f79ded..7401947 100644 --- a/json_object.c +++ b/json_object.c @@ -366,24 +366,39 @@ struct fjson_object* fjson_object_new_object(void) } -/* finds the child with given key if it exists in a json object - * and returns a pointer to it. Returns NULL if not found. +/** + * \brief Find an object member by key. + * + * Object members are stored in fixed-size linked pages rather than in a hash + * table. Walk those pages and their occupied slots directly: this keeps the + * lookup linear, preserves the existing insertion and deletion semantics, and + * avoids iterator cross-page bookkeeping on this frequently used path. + * + * The explicit comparison also deliberately honours + * \c do_case_sensitive_comparison; a separate index would need the same + * case-mode-dependent key semantics. + * + * \return A pointer to the matching child, or \c NULL when no key matches. */ static struct _fjson_child* _fjson_find_child(struct fjson_object *const __restrict__ jso, const char *const key) { - struct fjson_object_iterator it = fjson_object_iter_begin(jso); - struct fjson_object_iterator itEnd = fjson_object_iter_end(jso); - while (!fjson_object_iter_equal(&it, &itEnd)) { - if (do_case_sensitive_comparison) { - if (!strcmp (key, fjson_object_iter_peek_name(&it))) - return _fjson_object_iter_peek_child(&it); - } else { - if (!strcasecmp (key, fjson_object_iter_peek_name(&it))) - return _fjson_object_iter_peek_child(&it); + struct _fjson_child_pg *pg = &jso->o.c_obj.pg; + while (pg != NULL) { + for (int i = 0; i < FJSON_OBJECT_CHLD_PG_SIZE; ++i) { + struct _fjson_child *const chld = &pg->children[i]; + if (chld->k == NULL) + continue; + if (do_case_sensitive_comparison) { + if (!strcmp(key, chld->k)) + return chld; + } else { + if (!strcasecmp(key, chld->k)) + return chld; + } } - fjson_object_iter_next(&it); + pg = pg->next; } return NULL; } diff --git a/tests/test_charcase.c b/tests/test_charcase.c index 3e8389d..efbf046 100644 --- a/tests/test_charcase.c +++ b/tests/test_charcase.c @@ -15,12 +15,14 @@ } static void test_case_parse(void); +static void test_object_lookup_case(void); int main(int __attribute__((unused)) argc, char __attribute__((unused)) **argv) { MC_SET_DEBUG(1); test_case_parse(); + test_object_lookup_case(); return 0; } @@ -46,3 +48,37 @@ static void test_case_parse(void) fjson_tokener_free(tok); } + +/* Exercise object lookup across pages and deleted slots in both comparison modes. */ +static void test_object_lookup_case(void) +{ + char key[16]; + fjson_object *obj = fjson_object_new_object(); + fjson_object *value; + int i; + + CHK(obj != NULL); + for (i = 0; i < 10; ++i) { + snprintf(key, sizeof(key), "key-%02d", i); + fjson_object_object_add(obj, key, fjson_object_new_int(i)); + } + fjson_object_object_del(obj, "key-02"); + fjson_object_object_del(obj, "key-08"); + CHK(fjson_object_object_length(obj) == 8); + CHK(fjson_object_object_get_ex(obj, "key-09", &value)); + CHK(fjson_object_get_int(value) == 9); + + fjson_global_do_case_sensitive_comparison(0); + CHK(fjson_object_object_get_ex(obj, "KEY-09", &value)); + CHK(fjson_object_get_int(value) == 9); + fjson_object_object_add(obj, "KEY-09", fjson_object_new_int(90)); + CHK(fjson_object_object_length(obj) == 8); + CHK(fjson_object_object_get_ex(obj, "key-09", &value)); + CHK(fjson_object_get_int(value) == 90); + + fjson_global_do_case_sensitive_comparison(1); + CHK(!fjson_object_object_get_ex(obj, "KEY-09", &value)); + CHK(value == NULL); + CHK(fjson_object_object_get_ex(obj, "key-09", &value)); + fjson_object_put(obj); +}