From 22807ea3529b63dad1548dc4fc8bfa352dab7472 Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Tue, 21 Jul 2026 20:39:42 +0100
Subject: [PATCH 1/9] refactor: extract the region-conversion core into an
 internal engine

Move the iso3-explosion conversion core of convert_region() verbatim to
convert_region_via_iso3() in R/convert_region_legacy.R. convert_region()
keeps validation, mapping resolution and the pass-through shortcut, then
delegates. No behaviour change; this isolates the reference engine so a
faster engine can be added and tested for equivalence against it.
---
 R/convert_region.R        | 124 ++------------------------------
 R/convert_region_legacy.R | 144 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 149 insertions(+), 119 deletions(-)
 create mode 100644 R/convert_region_legacy.R

diff --git a/R/convert_region.R b/R/convert_region.R
index df09dfe..80db976 100644
--- a/R/convert_region.R
+++ b/R/convert_region.R
@@ -129,123 +129,9 @@ convert_region <- function(.x,
     stop(paste0("to_reg == iso3 is not yet implemented."))
   }
 
-  # Add iso3 and data_reg mapping
-  if (rname0 == "iso3") {
-    .x <- merge(.x, rmap1, by = "iso3")
-  } else {
-    .r <- merge(rmap0, rmap1, by = "iso3")
-    .x <- merge(.x, .r, by = rname0, allow.cartesian = TRUE)
-  }
-
-  # Add weight
-  .x <- merge(.x, agg_weight, by = "iso3")
-  .x <- .x[!is.na(get(rname1))]
-
-  dkeys <- function(dd) {
-    return(c(colnames(dd)[!colnames(dd) %in% c(
-      "value",
-      "weight", "sum_weight",
-      "iso3", rname0, rname1
-    )]))
-  }
-
-  # Disaggregation
-  if (rname0 != "iso3") {
-    if (agg_operator %in% c("sum","sumby")) {
-      # total weights are computed because of missing zeros values
-      .w <- merge(rmap0, agg_weight, by = "iso3")
-      .w <- .w[iso3 %in% unique(.x$iso3)]
-      .w <- .w[, list(sum_weight = sum(weight)), by = rname0]
-      .x <- merge(.x, .w, by = rname0)
-      .x <- .x[, list(iso3,
-        rname1 = get(rname1),
-        value = value * weight / sum_weight,
-        weight
-      ),
-      by = c(dkeys(.x), rname0)
-      ]
-      if (agg_operator %in% c("sum")) {
-        .x[, weight := NULL]
-      }
-    } else {
-      if (agg_operator %in% c("mean", "set1", "min", "minw", "max", "maxw")) {
-        .x <- .x[, .(iso3,
-          rname1 = get(rname1),
-          value,
-          weight
-        ),
-        by = c(dkeys(.x), rname0)
-        ]
-      } else {
-        stop(paste("Operator ", agg_operator, "not implemented"))
-      }
-    }
-  } else {
-    data.table::setnames(.x, rname1, "rname1")
-  }
-
-  # informed share
-  .info_share <- NULL
-  if (agg_operator %in% c("sumby")) {
-    .w <- merge(rmap1, agg_weight, by = "iso3")
-    .w <- .w[, .(sum_weight = sum(weight)), by = rname1]
-    data.table::setnames(.w, rname1, "rname1")
-    .x <- merge(.x, .w, by = "rname1")
-    .info_share <- .x[, .(value = sum(weight) / mean(sum_weight)),
-      by = c(dkeys(.x))
-    ]
-    data.table::setnames(.info_share, "rname1", rname1)
-  }
-
-  # Aggregation
-  if (agg_operator %in% c("sum", "sumby")) {
-    .x <- .x[, .(value = sum(value)), by = c(dkeys(.x))]
-  } else {
-    .w <- merge(rmap1, agg_weight, by = "iso3")
-    .w <- .w[, .(sum_weight = sum(weight)), by = rname1]
-    data.table::setnames(.w, rname1, "rname1")
-    .x <- merge(.x, .w, by = "rname1")
-    if (agg_operator == "mean") {
-      if (agg_missing == "zero") {
-        .x <- .x[, .(value = sum(value * weight / sum_weight)),
-          by = c(dkeys(.x))
-        ]
-      }
-      if (agg_missing == "NA") {
-        .x <- .x[!is.na(value), .(value = sum(value * weight / sum(weight))),
-          by = c(dkeys(.x))
-        ]
-      }
-    } else if (agg_operator == "set1") {
-      if (agg_missing == "zero") {
-        .x <- .x[, .(value = round(sum(value * weight / sum_weight))),
-          by = c(dkeys(.x))
-        ]
-      }
-      if (agg_missing == "NA") {
-        .x <- .x[, .(value = round(sum(value * weight / sum(weight)))),
-          by = c(dkeys(.x))
-        ]
-      }
-    } else if (agg_operator %in% c("min", "minw")) {
-      .x <- .x[, .(value = min(value[which(weight == min(weight))])),
-        by = c(dkeys(.x))
-      ]
-    } else if (agg_operator %in% c("max", "maxw")) {
-      .x <- .x[, .(value = max(value[which(weight == max(weight))])),
-        by = c(dkeys(.x))
-      ]
-    } else {
-      stop(paste("Operator", agg_operator, "not implemented"))
-    }
-  }
-
-  # Change the region column name
-  data.table::setnames(.x, "rname1", rname1)
-
-  if (info) {
-    return(list(data = .x, info = .info_share))
-  } else {
-    return(.x)
-  }
+  # Convert via the legacy iso3-explosion engine.
+  # A pair-coefficient "fast" engine is added in a later commit; until then
+  # every route goes through the reference implementation.
+  convert_region_via_iso3(.x, rmap0, rname0, rmap1, rname1,
+                          agg_operator, agg_weight, agg_missing, info)
 }
diff --git a/R/convert_region_legacy.R b/R/convert_region_legacy.R
new file mode 100644
index 0000000..522a471
--- /dev/null
+++ b/R/convert_region_legacy.R
@@ -0,0 +1,144 @@
+# Legacy region-conversion engine.
+#
+# This is the original convert_region() core, moved verbatim from
+# R/convert_region.R. It downscales the table to iso3 country level
+# (exploding each region row to all its member countries) and aggregates back
+# into the target mapping. It is kept as the reference implementation: the
+# equivalence tests compare the fast pair-coefficient engine against it, and
+# it remains reachable via options(witchtools.convert_region_engine = "legacy").
+#
+# Arguments are the already-resolved pieces computed by convert_region():
+# the input table, both mapping tables and their names, and the aggregation
+# settings. Validation and the pass-through shortcut live in the dispatcher.
+
+#' @noRd
+convert_region_via_iso3 <- function(.x,
+                                    rmap0, rname0,
+                                    rmap1, rname1,
+                                    agg_operator,
+                                    agg_weight,
+                                    agg_missing,
+                                    info) {
+
+  iso3 <- weight <- sum_weight <- value <- NULL # due to NSE notes in R CMD check
+
+  # Add iso3 and data_reg mapping
+  if (rname0 == "iso3") {
+    .x <- merge(.x, rmap1, by = "iso3")
+  } else {
+    .r <- merge(rmap0, rmap1, by = "iso3")
+    .x <- merge(.x, .r, by = rname0, allow.cartesian = TRUE)
+  }
+
+  # Add weight
+  .x <- merge(.x, agg_weight, by = "iso3")
+  .x <- .x[!is.na(get(rname1))]
+
+  dkeys <- function(dd) {
+    return(c(colnames(dd)[!colnames(dd) %in% c(
+      "value",
+      "weight", "sum_weight",
+      "iso3", rname0, rname1
+    )]))
+  }
+
+  # Disaggregation
+  if (rname0 != "iso3") {
+    if (agg_operator %in% c("sum","sumby")) {
+      # total weights are computed because of missing zeros values
+      .w <- merge(rmap0, agg_weight, by = "iso3")
+      .w <- .w[iso3 %in% unique(.x$iso3)]
+      .w <- .w[, list(sum_weight = sum(weight)), by = rname0]
+      .x <- merge(.x, .w, by = rname0)
+      .x <- .x[, list(iso3,
+        rname1 = get(rname1),
+        value = value * weight / sum_weight,
+        weight
+      ),
+      by = c(dkeys(.x), rname0)
+      ]
+      if (agg_operator %in% c("sum")) {
+        .x[, weight := NULL]
+      }
+    } else {
+      if (agg_operator %in% c("mean", "set1", "min", "minw", "max", "maxw")) {
+        .x <- .x[, .(iso3,
+          rname1 = get(rname1),
+          value,
+          weight
+        ),
+        by = c(dkeys(.x), rname0)
+        ]
+      } else {
+        stop(paste("Operator ", agg_operator, "not implemented"))
+      }
+    }
+  } else {
+    data.table::setnames(.x, rname1, "rname1")
+  }
+
+  # informed share
+  .info_share <- NULL
+  if (agg_operator %in% c("sumby")) {
+    .w <- merge(rmap1, agg_weight, by = "iso3")
+    .w <- .w[, .(sum_weight = sum(weight)), by = rname1]
+    data.table::setnames(.w, rname1, "rname1")
+    .x <- merge(.x, .w, by = "rname1")
+    .info_share <- .x[, .(value = sum(weight) / mean(sum_weight)),
+      by = c(dkeys(.x))
+    ]
+    data.table::setnames(.info_share, "rname1", rname1)
+  }
+
+  # Aggregation
+  if (agg_operator %in% c("sum", "sumby")) {
+    .x <- .x[, .(value = sum(value)), by = c(dkeys(.x))]
+  } else {
+    .w <- merge(rmap1, agg_weight, by = "iso3")
+    .w <- .w[, .(sum_weight = sum(weight)), by = rname1]
+    data.table::setnames(.w, rname1, "rname1")
+    .x <- merge(.x, .w, by = "rname1")
+    if (agg_operator == "mean") {
+      if (agg_missing == "zero") {
+        .x <- .x[, .(value = sum(value * weight / sum_weight)),
+          by = c(dkeys(.x))
+        ]
+      }
+      if (agg_missing == "NA") {
+        .x <- .x[!is.na(value), .(value = sum(value * weight / sum(weight))),
+          by = c(dkeys(.x))
+        ]
+      }
+    } else if (agg_operator == "set1") {
+      if (agg_missing == "zero") {
+        .x <- .x[, .(value = round(sum(value * weight / sum_weight))),
+          by = c(dkeys(.x))
+        ]
+      }
+      if (agg_missing == "NA") {
+        .x <- .x[, .(value = round(sum(value * weight / sum(weight)))),
+          by = c(dkeys(.x))
+        ]
+      }
+    } else if (agg_operator %in% c("min", "minw")) {
+      .x <- .x[, .(value = min(value[which(weight == min(weight))])),
+        by = c(dkeys(.x))
+      ]
+    } else if (agg_operator %in% c("max", "maxw")) {
+      .x <- .x[, .(value = max(value[which(weight == max(weight))])),
+        by = c(dkeys(.x))
+      ]
+    } else {
+      stop(paste("Operator", agg_operator, "not implemented"))
+    }
+  }
+
+  # Change the region column name
+  data.table::setnames(.x, "rname1", rname1)
+
+  if (info) {
+    return(list(data = .x, info = .info_share))
+  } else {
+    return(.x)
+  }
+}

From b31e83a5bdbc05e10153b6852e6d2e0a65295cb8 Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Tue, 21 Jul 2026 20:41:36 +0100
Subject: [PATCH 2/9] test: add engine-equivalence test infrastructure

with_engine() option helper, a GLOBIOM-shaped input generator covering the
edge cases (NA values, per-id missing regions, duplicated rows, factor and
character-period columns), and expect_region_equal() asserting identical
names/classes and tolerance-compared values. Smoke-tested legacy-vs-legacy
until the fast engine lands.
---
 tests/testthat/helper-convert-region.R        | 94 +++++++++++++++++++
 .../test-convert-region-equivalence.R         | 32 +++++++
 2 files changed, 126 insertions(+)
 create mode 100644 tests/testthat/helper-convert-region.R
 create mode 100644 tests/testthat/test-convert-region-equivalence.R

diff --git a/tests/testthat/helper-convert-region.R b/tests/testthat/helper-convert-region.R
new file mode 100644
index 0000000..6fbf96a
--- /dev/null
+++ b/tests/testthat/helper-convert-region.R
@@ -0,0 +1,94 @@
+# Helpers for the convert_region engine-equivalence tests.
+
+# Run expr with a given region-conversion engine ("legacy" or "fast").
+with_engine <- function(engine, expr) {
+  withr::with_options(
+    list(witchtools.convert_region_engine = engine),
+    expr
+  )
+}
+
+# Build a GLOBIOM-shaped input table.
+#
+# Columns mimic the real GLOBIOM reporting tables: character id columns
+# (variable, scenario), one factor id column (ghg_class), a character period
+# column (t), a region column named after `route`, and a numeric value.
+#
+# route: "witch17" (region-level input) or "iso3" (country-level input).
+# n_var: number of distinct `variable` values (drives table size).
+# na_frac: fraction of values set to NA.
+# missing_region_per_id: randomly drop some regions within some id-groups
+#   (exercises the data-dependence edge case).
+# drop_region: name of one region to remove from the whole table ("" = none).
+# dup_rows: duplicate a handful of (id, region) rows.
+make_input <- function(route = "witch17",
+                       n_var = 4,
+                       seed = 42,
+                       na_frac = 0,
+                       missing_region_per_id = FALSE,
+                       drop_region = "",
+                       dup_rows = FALSE) {
+  set.seed(seed)
+  regs <- if (route == "iso3") {
+    witchtools::region_mappings[["witch17"]]$iso3
+  } else {
+    unique(witchtools::region_mappings[[route]][[route]])
+  }
+  dt <- data.table::CJ(
+    variable = paste0("var", seq_len(n_var)),
+    scenario = c("ssp1_lu1", "ssp2_lu2"),
+    ghg_class = factor(c("GHG001", "GHG002")),
+    t = as.character(c(2L, 5L, 10L)),
+    region = regs,
+    sorted = FALSE
+  )
+  dt[, value := round(stats::runif(.N, 0, 1000), 3)]
+  if (na_frac > 0) {
+    idx <- sample(nrow(dt), ceiling(na_frac * nrow(dt)))
+    dt[idx, value := NA_real_]
+  }
+  if (missing_region_per_id) {
+    # For a third of the variables, drop a random subset of regions entirely.
+    for (v in unique(dt$variable)[seq_len(max(1L, n_var %/% 3L))]) {
+      gone <- sample(regs, max(1L, length(regs) %/% 4L))
+      dt <- dt[!(variable == v & region %in% gone)]
+    }
+  }
+  if (nzchar(drop_region)) {
+    dt <- dt[region != drop_region]
+  }
+  if (dup_rows) {
+    dt <- rbind(dt, dt[sample(nrow(dt), 5L)])
+  }
+  data.table::setnames(dt, "region", route)
+  return(dt)
+}
+
+# Compare two convert_region() results (plain data.table, or list(data, info)
+# when info = TRUE). Column names, order and classes must be identical; values
+# are compared after sorting by all id columns, with a tolerance for the
+# floating-point reassociation between engines.
+expect_region_equal <- function(old, new, tol = 1e-12) {
+  if (is.list(old) && !data.table::is.data.table(old)) {
+    testthat::expect_true(is.list(new) && !data.table::is.data.table(new))
+    expect_region_equal(old$data, new$data, tol = tol)
+    if (is.null(old$info)) {
+      testthat::expect_null(new$info)
+    } else {
+      expect_region_equal(old$info, new$info, tol = tol)
+    }
+    return(invisible(NULL))
+  }
+  testthat::expect_identical(names(old), names(new))
+  testthat::expect_identical(
+    lapply(old, class),
+    lapply(new, class)
+  )
+  keys <- setdiff(names(old), "value")
+  o <- data.table::setorderv(data.table::copy(old), keys)
+  n <- data.table::setorderv(data.table::copy(new), keys)
+  testthat::expect_equal(as.data.frame(o), as.data.frame(n),
+    tolerance = tol,
+    ignore_attr = TRUE
+  )
+}
diff --git a/tests/testthat/test-convert-region-equivalence.R b/tests/testthat/test-convert-region-equivalence.R
new file mode 100644
index 0000000..adc574e
--- /dev/null
+++ b/tests/testthat/test-convert-region-equivalence.R
@@ -0,0 +1,32 @@
+# Engine-equivalence tests: the fast pair-coefficient engine must reproduce
+# the legacy iso3-explosion engine for every operator, agg_missing mode and
+# route. Until the fast engine lands, this file smoke-tests the helpers with
+# legacy-vs-legacy comparisons.
+
+test_that("equivalence helpers work (legacy vs legacy smoke)", {
+  dt <- make_input("witch17", seed = 1)
+  r1 <- with_engine("legacy", convert_region(dt, to_reg = "witch20"))
+  r2 <- with_engine("legacy", convert_region(dt, to_reg = "witch20"))
+  expect_region_equal(r1, r2)
+
+  ri <- with_engine(
+    "legacy",
+    suppressWarnings(
+      convert_region(dt, to_reg = "witch20", agg_operator = "sumby", info = TRUE)
+    )
+  )
+  expect_region_equal(ri, ri)
+  expect_true(data.table::is.data.table(ri$info))
+})
+
+test_that("make_input produces the expected shapes", {
+  dt <- make_input("witch17", na_frac = 0.1, missing_region_per_id = TRUE,
+                   dup_rows = TRUE, seed = 7)
+  expect_true("witch17" %in% names(dt))
+  expect_true(is.factor(dt$ghg_class))
+  expect_type(dt$t, "character")
+  expect_gt(sum(is.na(dt$value)), 0)
+
+  di <- make_input("iso3", seed = 7)
+  expect_true("iso3" %in% names(di))
+})

From 6613b3072803a3b4512c15af760e6c8fe3b1d94b Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Tue, 21 Jul 2026 20:47:18 +0100
Subject: [PATCH 3/9] perf: add pair-coefficient fast engine for region
 conversion

Region-to-region conversion no longer explodes the table to iso3 country
level (~x15 rows for witch17 input, held across four full-size
intermediates). The downscale->upscale is a linear map, so the fast
engine precomputes a per-(from, to) pair coefficient table (21 rows for
witch17->witch20) and converts with one small join (~x1.2 rows) and one
grouped aggregation. Covers sum, sumby (including the info coperture),
mean, min/minw and max/maxw for all agg_missing modes; column and row
order match the legacy engine exactly.

iso3-level input (already linear-size) and the set1 operator stay on the
legacy engine: set1's round() is discontinuous, so the fast engine's
summation reassociation could flip values sitting exactly on a .5
boundary (observed on synthetic data).

The legacy engine remains selectable via
options(witchtools.convert_region_engine = "legacy") and serves as the
reference in the new equivalence suite (127 assertions across operators,
agg_missing modes, routes, weights, NA patterns, per-id missing regions,
duplicated rows, non-partition mappings and row order).
---
 R/convert_region.R                            |  23 +-
 R/convert_region_fast.R                       | 162 ++++++++++++++
 man/convert_region.Rd                         |   9 +
 .../test-convert-region-equivalence.R         | 209 +++++++++++++++---
 4 files changed, 375 insertions(+), 28 deletions(-)
 create mode 100644 R/convert_region_fast.R

diff --git a/R/convert_region.R b/R/convert_region.R
index 80db976..d945873 100644
--- a/R/convert_region.R
+++ b/R/convert_region.R
@@ -20,6 +20,15 @@
 #' and another one named as the regional mapping (for region name).
 #' The name in the list should also be the regional mapping name.
 #'
+#' Region-to-region conversions use a fast engine that converts through a
+#' precomputed region-pair coefficient table instead of expanding the data to
+#' country level, which drastically reduces memory use and run time on large
+#' tables. Results are identical up to floating-point summation order
+#' (relative differences below 1e-12). The previous implementation remains
+#' available with \code{options(witchtools.convert_region_engine = "legacy")}.
+#' Country-level (iso3) input and the \code{set1} operator always use the
+#' legacy engine.
+#'
 #' @family conversion functions
 #' @seealso \code{\link{convert_table}},
 #' \code{\link{convert_gdx}}.
@@ -129,9 +138,17 @@ convert_region <- function(.x,
     stop(paste0("to_reg == iso3 is not yet implemented."))
   }
 
-  # Convert via the legacy iso3-explosion engine.
-  # A pair-coefficient "fast" engine is added in a later commit; until then
-  # every route goes through the reference implementation.
+  # Engine dispatch. The "fast" engine converts region->region input through
+  # a small pair-coefficient table instead of the iso3 explosion; iso3-level
+  # input is already linear-size and stays on the legacy engine. "set1" also
+  # stays on the legacy engine: its round() is discontinuous, so the summation
+  # reassociation of the fast engine could flip a value sitting exactly on a
+  # .5 boundary, and set1 tables are tiny anyway.
+  engine <- getOption("witchtools.convert_region_engine", "fast")
+  if (identical(engine, "fast") && rname0 != "iso3" && agg_operator != "set1") {
+    return(convert_region_fast(.x, rmap0, rname0, rmap1, rname1,
+                               agg_operator, agg_weight, agg_missing, info))
+  }
   convert_region_via_iso3(.x, rmap0, rname0, rmap1, rname1,
                           agg_operator, agg_weight, agg_missing, info)
 }
diff --git a/R/convert_region_fast.R b/R/convert_region_fast.R
new file mode 100644
index 0000000..b2b4d6e
--- /dev/null
+++ b/R/convert_region_fast.R
@@ -0,0 +1,162 @@
+# Fast region-conversion engine.
+#
+# The legacy engine downscales the table to iso3 country level (~x15 rows for
+# witch17 input) and re-aggregates. But the downscale->upscale is a linear
+# map: for every operator the result only depends on per-(from, to) pair
+# aggregates of the country weights. This engine precomputes a tiny pair
+# coefficient table (21 rows for witch17->witch20) and converts with a single
+# join (~x1.2 rows) and one grouped aggregation, instead of the iso3 explosion
+# and its four full-size intermediate copies.
+#
+# Numerical results match the legacy engine up to floating-point summation
+# order (tolerance ~1e-12, covered by the equivalence tests in
+# tests/testthat/test-convert-region-equivalence.R).
+
+# Build the per-(from, to) coefficient table.
+#
+# Column semantics (mirroring the legacy engine's intermediate quantities):
+#   w_pair   sum of country weights in (from n to)      [legacy :141 join]
+#   w_min/w_max  min/max country weight in the pair     [for min*/max* ops]
+#   iso3_min smallest iso3 of the pair                  [row-order emulation]
+#   sw_from  sum of w_pair by from-region               [legacy :156-158]
+#   sw_to    sum of weights over the FULL to-mapping    [legacy :190/:204]
+#
+# The legacy sum denominator filters countries to those present in the data
+# (`iso3 %in% unique(.x$iso3)`), but since the explosion expands every present
+# region to all its member countries and denominators are grouped by
+# from-region, the filter is data-independent: sw_from computed here is
+# identical for every from-region that appears in the data at all.
+#' @noRd
+build_region_coeff <- function(rmap0, rname0, rmap1, rname1, agg_weight) {
+  iso3 <- weight <- w_pair <- .from <- NULL # due to NSE notes in R CMD check
+
+  pw <- merge(rmap0, rmap1, by = "iso3")
+  pw <- merge(pw, agg_weight, by = "iso3")
+  pw <- pw[!is.na(get(rname1))]
+  cf <- pw[, list(
+    w_pair = sum(weight),
+    w_min = min(weight),
+    w_max = max(weight),
+    iso3_min = min(iso3)
+  ), by = c(rname0, rname1)]
+  data.table::setnames(cf, c(rname0, rname1), c(".from", ".to"))
+  cf[, sw_from := sum(w_pair), by = .from]
+  sw_to <- merge(rmap1, agg_weight, by = "iso3")
+  sw_to <- sw_to[, list(sw_to = sum(weight)), by = rname1]
+  data.table::setnames(sw_to, rname1, ".to")
+  cf <- merge(cf, sw_to, by = ".to")
+  return(cf)
+}
+
+#' @noRd
+convert_region_fast <- function(.x,
+                                rmap0, rname0,
+                                rmap1, rname1,
+                                agg_operator,
+                                agg_weight,
+                                agg_missing,
+                                info) {
+
+  value <- w_pair <- sw_from <- sw_to <- NULL # due to NSE notes in R CMD check
+  .row0 <- .from <- .to <- iso3_min <- w_min <- w_max <- gw <- NULL
+
+  known_ops <- c("sum", "sumby", "mean", "min", "minw", "max", "maxw")
+  if (!agg_operator %in% known_ops) {
+    if (agg_operator == "set1") {
+      # Dispatched to the legacy engine by convert_region(): round() is
+      # discontinuous, so reassociated summation could flip .5 boundaries.
+      stop("set1 is handled by the legacy engine.")
+    }
+    # Same message and trigger point as the legacy disaggregation branch.
+    stop(paste("Operator ", agg_operator, "not implemented"))
+  }
+
+  cf <- build_region_coeff(rmap0, rname0, rmap1, rname1, agg_weight)
+
+  idcols <- setdiff(names(.x), c(rname0, "value"))
+
+  # Transient input-row tag for row-order emulation; removed on exit so the
+  # caller's table is left untouched even on error.
+  .x[, .row0 := .I]
+  on.exit(
+    if (".row0" %in% names(.x)) .x[, .row0 := NULL],
+    add = TRUE
+  )
+
+  # One small join instead of the iso3 explosion. nomatch=NULL drops input
+  # rows whose region is absent from the crosswalk/weights, as the legacy
+  # inner merges do.
+  xt <- cf[.x, on = c(".from" = rname0), allow.cartesian = TRUE, nomatch = NULL]
+
+  if (agg_operator %in% c("sum", "sumby")) {
+
+    # Legacy row order: groups appear in (from-region, input-row) order for
+    # sum, with an extra leading to-region sort for sumby (its merge at :193).
+    if (agg_operator == "sum") {
+      data.table::setorder(xt, .from, .row0, iso3_min)
+      out <- xt[, list(value = sum(value * w_pair / sw_from)),
+        by = c(idcols, ".to")
+      ]
+      data.table::setnames(out, ".to", rname1)
+      data.table::setcolorder(out, c(idcols, rname1, "value"))
+      if (info) {
+        return(list(data = out, info = NULL))
+      }
+      return(out)
+    }
+
+    data.table::setorder(xt, .to, .from, .row0, iso3_min)
+    out <- xt[, list(value = sum(value * w_pair / sw_from)),
+      by = c(".to", idcols)
+    ]
+    data.table::setnames(out, ".to", rname1)
+    data.table::setcolorder(out, c(rname1, idcols, "value"))
+
+    .info_share <- NULL
+    .info_share <- xt[, list(value = sum(w_pair / sw_to)),
+      by = c(".to", idcols)
+    ]
+    data.table::setnames(.info_share, ".to", rname1)
+    data.table::setcolorder(.info_share, c(rname1, idcols, "value"))
+
+    if (info) {
+      return(list(data = out, info = .info_share))
+    }
+    return(out)
+  }
+
+  # mean / set1 / min* / max* : legacy replicates values to countries and
+  # aggregates with the full to-mapping weight sums (sw_to).
+  data.table::setorder(xt, .to, .from, .row0, iso3_min)
+
+  if (agg_operator == "mean") {
+    if (agg_missing == "zero") {
+      out <- xt[, list(value = sum(value * w_pair) / sw_to[1L]),
+        by = c(".to", idcols)
+      ]
+    } else {
+      # agg_missing == "NA": legacy filters NA rows before aggregating, so
+      # groups whose contributions are all NA disappear from the output.
+      out <- xt[!is.na(value),
+        list(value = sum(value * w_pair) / sum(w_pair)),
+        by = c(".to", idcols)
+      ]
+    }
+  } else if (agg_operator %in% c("min", "minw")) {
+    # Legacy: min(value[weight == min(weight)]) over country rows; replicated
+    # values make this expressible with the per-pair minimum weight.
+    xt[, gw := min(w_min), by = c(".to", idcols)]
+    out <- xt[w_min == gw, list(value = min(value)), by = c(".to", idcols)]
+  } else {
+    xt[, gw := max(w_max), by = c(".to", idcols)]
+    out <- xt[w_max == gw, list(value = max(value)), by = c(".to", idcols)]
+  }
+
+  data.table::setnames(out, ".to", rname1)
+  data.table::setcolorder(out, c(rname1, idcols, "value"))
+
+  if (info) {
+    return(list(data = out, info = NULL))
+  }
+  return(out)
+}
diff --git a/man/convert_region.Rd b/man/convert_region.Rd
index 1bb814a..d661a86 100644
--- a/man/convert_region.Rd
+++ b/man/convert_region.Rd
@@ -61,6 +61,15 @@ Regional mappings are 2-columns data.table with a column named 'iso3'
 (for country ISO3)
 and another one named as the regional mapping (for region name).
 The name in the list should also be the regional mapping name.
+
+Region-to-region conversions use a fast engine that converts through a
+precomputed region-pair coefficient table instead of expanding the data to
+country level, which drastically reduces memory use and run time on large
+tables. Results are identical up to floating-point summation order
+(relative differences below 1e-12). The previous implementation remains
+available with \code{options(witchtools.convert_region_engine = "legacy")}.
+Country-level (iso3) input and the \code{set1} operator always use the
+legacy engine.
 }
 \examples{
 # Aggregate country-level GDP into the 17 WITCH regions
diff --git a/tests/testthat/test-convert-region-equivalence.R b/tests/testthat/test-convert-region-equivalence.R
index adc574e..b87c0d6 100644
--- a/tests/testthat/test-convert-region-equivalence.R
+++ b/tests/testthat/test-convert-region-equivalence.R
@@ -1,32 +1,191 @@
 # Engine-equivalence tests: the fast pair-coefficient engine must reproduce
 # the legacy iso3-explosion engine for every operator, agg_missing mode and
-# route. Until the fast engine lands, this file smoke-tests the helpers with
-# legacy-vs-legacy comparisons.
-
-test_that("equivalence helpers work (legacy vs legacy smoke)", {
-  dt <- make_input("witch17", seed = 1)
-  r1 <- with_engine("legacy", convert_region(dt, to_reg = "witch20"))
-  r2 <- with_engine("legacy", convert_region(dt, to_reg = "witch20"))
-  expect_region_equal(r1, r2)
-
-  ri <- with_engine(
-    "legacy",
-    suppressWarnings(
-      convert_region(dt, to_reg = "witch20", agg_operator = "sumby", info = TRUE)
-    )
+# route (values within floating-point tolerance; identical column names,
+# order and classes).
+
+both_engines <- function(expr_fun) {
+  list(
+    old = with_engine("legacy", expr_fun()),
+    new = with_engine("fast", expr_fun())
   )
-  expect_region_equal(ri, ri)
-  expect_true(data.table::is.data.table(ri$info))
+}
+
+test_that("fast engine matches legacy across operators and agg_missing", {
+  for (op in c("sum", "mean", "set1", "min", "minw", "max", "maxw")) {
+    for (am in c("NA", "zero")) {
+      dt <- make_input("witch17", seed = 11)
+      r <- both_engines(function() {
+        convert_region(data.table::copy(dt),
+          to_reg = "witch20",
+          agg_operator = op, agg_missing = am
+        )
+      })
+      expect_region_equal(r$old, r$new)
+    }
+  }
+})
+
+test_that("fast engine matches legacy for sumby including info", {
+  dt <- make_input("witch17", seed = 12)
+  r <- both_engines(function() {
+    convert_region(data.table::copy(dt),
+      to_reg = "witch20",
+      agg_operator = "sumby", info = TRUE
+    )
+  })
+  expect_region_equal(r$old, r$new)
+  expect_true(data.table::is.data.table(r$new$info))
+})
+
+test_that("fast engine matches legacy on other routes and weights", {
+  for (route_to in c("witch20", "r5")) {
+    for (w in c("gdp", "cst", "pop")) {
+      dt <- make_input("witch17", seed = 13)
+      r <- both_engines(function() {
+        convert_region(data.table::copy(dt),
+          to_reg = route_to,
+          agg_weight = witchtools::default_weights[[w]]
+        )
+      })
+      expect_region_equal(r$old, r$new)
+    }
+  }
+})
+
+test_that("iso3 input routes through the legacy engine and is unchanged", {
+  dt <- make_input("iso3", seed = 14)
+  r <- both_engines(function() {
+    convert_region(data.table::copy(dt), to_reg = "witch17")
+  })
+  # Both engines must give identical results (same code path).
+  expect_identical(r$old, r$new)
+})
+
+test_that("regions missing per id-group are handled identically", {
+  dt <- make_input("witch17", seed = 15, missing_region_per_id = TRUE)
+  for (op in c("sum", "mean")) {
+    r <- both_engines(function() {
+      convert_region(data.table::copy(dt), to_reg = "witch20", agg_operator = op)
+    })
+    expect_region_equal(r$old, r$new)
+  }
+})
+
+test_that("a region absent from the whole table is handled identically", {
+  dt <- make_input("witch17", seed = 16, drop_region = "usa")
+  r <- both_engines(function() {
+    convert_region(data.table::copy(dt), to_reg = "witch20")
+  })
+  expect_region_equal(r$old, r$new)
+})
+
+test_that("NA values propagate identically (sum, mean zero, set1 NA)", {
+  dt <- make_input("witch17", seed = 17, na_frac = 0.15)
+  for (case in list(
+    c("sum", "NA"), c("mean", "zero"), c("mean", "NA"),
+    c("set1", "NA"), c("set1", "zero"), c("min", "NA"), c("max", "NA")
+  )) {
+    r <- both_engines(function() {
+      convert_region(data.table::copy(dt),
+        to_reg = "witch20",
+        agg_operator = case[1], agg_missing = case[2]
+      )
+    })
+    expect_region_equal(r$old, r$new)
+  }
+})
+
+test_that("all-NA groups vanish under mean/'NA' in both engines", {
+  dt <- make_input("witch17", seed = 18)
+  dt[variable == "var1", value := NA_real_]
+  r <- both_engines(function() {
+    convert_region(data.table::copy(dt),
+      to_reg = "witch20",
+      agg_operator = "mean", agg_missing = "NA"
+    )
+  })
+  expect_false("var1" %in% r$old$variable)
+  expect_region_equal(r$old, r$new)
 })
 
-test_that("make_input produces the expected shapes", {
-  dt <- make_input("witch17", na_frac = 0.1, missing_region_per_id = TRUE,
-                   dup_rows = TRUE, seed = 7)
-  expect_true("witch17" %in% names(dt))
-  expect_true(is.factor(dt$ghg_class))
-  expect_type(dt$t, "character")
-  expect_gt(sum(is.na(dt$value)), 0)
+test_that("duplicated rows double-count identically (sum and sumby info)", {
+  dt <- make_input("witch17", seed = 19, dup_rows = TRUE)
+  r <- both_engines(function() {
+    convert_region(data.table::copy(dt),
+      to_reg = "witch20",
+      agg_operator = "sumby", info = TRUE
+    )
+  })
+  expect_region_equal(r$old, r$new)
+})
 
-  di <- make_input("iso3", seed = 7)
-  expect_true("iso3" %in% names(di))
+test_that("iso3 missing from the weight table is handled identically", {
+  w <- data.table::copy(witchtools::default_weights[["gdp"]])
+  w <- w[!iso3 %in% c("FRA", "CHN", "BRA")]
+  dt <- make_input("witch17", seed = 20)
+  r <- both_engines(function() {
+    convert_region(data.table::copy(dt), to_reg = "witch20", agg_weight = w)
+  })
+  expect_region_equal(r$old, r$new)
+})
+
+test_that("NA regions in the target mapping are handled identically", {
+  rmap <- data.table::copy(witchtools::region_mappings[["witch20"]])
+  rmap[iso3 %in% c("USA", "CAN"), witch20 := NA_character_]
+  regions <- witchtools::region_mappings
+  regions[["witch20"]] <- rmap
+  dt <- make_input("witch17", seed = 21)
+  r <- both_engines(function() {
+    convert_region(data.table::copy(dt), to_reg = "witch20", regions = regions)
+  })
+  expect_region_equal(r$old, r$new)
+})
+
+test_that("a non-partition from-mapping is handled identically", {
+  # One country in two from-regions: defensive check of the sw_from
+  # derivation, which assumes per-region denominators.
+  rmap <- data.table::copy(witchtools::region_mappings[["witch17"]])
+  rmap <- rbind(rmap, data.table::data.table(witch17 = "usa", iso3 = "MEX"))
+  regions <- witchtools::region_mappings
+  regions[["witch17"]] <- rmap
+  dt <- make_input("witch17", seed = 22)
+  r <- both_engines(function() {
+    convert_region(data.table::copy(dt),
+      from_reg = "witch17", to_reg = "witch20", regions = regions
+    )
+  })
+  expect_region_equal(r$old, r$new)
+})
+
+test_that("fast engine preserves legacy row order", {
+  dt <- make_input("witch17", seed = 23)
+  for (op in c("sum", "mean", "min", "max")) {
+    r <- both_engines(function() {
+      convert_region(data.table::copy(dt), to_reg = "witch20", agg_operator = op)
+    })
+    expect_identical(
+      as.data.frame(r$old)[, setdiff(names(r$old), "value")],
+      as.data.frame(r$new)[, setdiff(names(r$new), "value")]
+    )
+  }
+})
+
+test_that("fast engine does not mutate the input table", {
+  dt <- make_input("witch17", seed = 24)
+  before <- data.table::copy(dt)
+  invisible(with_engine("fast", convert_region(dt, to_reg = "witch20")))
+  expect_identical(names(dt), names(before))
+  expect_equal(as.data.frame(dt), as.data.frame(before))
+})
+
+test_that("unknown operators fail with the same error in both engines", {
+  dt <- make_input("witch17", seed = 25)
+  for (eng in c("legacy", "fast")) {
+    expect_error(
+      with_engine(eng, convert_region(data.table::copy(dt),
+        to_reg = "witch20", agg_operator = "bogus"
+      )),
+      "not implemented"
+    )
+  }
 })

From 6cc56ba8b1c21214893d8af66656c3ac19a18d08 Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Tue, 21 Jul 2026 20:54:05 +0100
Subject: [PATCH 4/9] perf: replace base subset() with direct indexing in
 conversion paths

subset() on a data.table or character vector allocates through the
data-frame path; plain [ filtering is faster and copies less. Four call
sites: the index-column selection in convert_gdx/convert_duckdb/
convert_sqlite and the missing-period filter in convert_time_period.
---
 R/convert_duckdb.R      | 2 +-
 R/convert_gdx.R         | 2 +-
 R/convert_sqlite.R      | 2 +-
 R/convert_time_period.R | 5 +----
 4 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/R/convert_duckdb.R b/R/convert_duckdb.R
index 7fdc014..91e6f4e 100644
--- a/R/convert_duckdb.R
+++ b/R/convert_duckdb.R
@@ -113,7 +113,7 @@ convert_duckdb <- function(duckdb,
 
     # add an additional parameter '_info' when using sumby
     if (!is.null(.info_share)) {
-      indices <- subset(colnames(.data), colnames(.data) != "value")
+      indices <- colnames(.data)[colnames(.data) != "value"]
       data.table::setcolorder(.info_share, data_indices)
       names(.info_share) <- c(indices, "value")
       .i <- list(.info_share)
diff --git a/R/convert_gdx.R b/R/convert_gdx.R
index 6b704ed..b1eaeb3 100644
--- a/R/convert_gdx.R
+++ b/R/convert_gdx.R
@@ -109,7 +109,7 @@ convert_gdx <- function(gdxfile,
     if (length(colnames(.data)) == 1) {
       names(.data) <- "value"
     } else {
-      indices <- subset(colnames(.data), colnames(.data) != "value")
+      indices <- colnames(.data)[colnames(.data) != "value"]
       indices <- ifelse(indices %in% c(region_name, "t"), indices, "*")
       names(.data) <- c(indices, "value")
     }
diff --git a/R/convert_sqlite.R b/R/convert_sqlite.R
index 6d1987b..a767a67 100644
--- a/R/convert_sqlite.R
+++ b/R/convert_sqlite.R
@@ -113,7 +113,7 @@ convert_sqlite <- function(sqlitedb,
 
     # add an additional parameter '_info' when using sumby
     if (!is.null(.info_share)) {
-      indices <- subset(colnames(.data), colnames(.data) != "value")
+      indices <- colnames(.data)[colnames(.data) != "value"]
       data.table::setcolorder(.info_share, data_indices)
       names(.info_share) <- c(indices, "value")
       .i <- list(.info_share)
diff --git a/R/convert_time_period.R b/R/convert_time_period.R
index e0217bb..8fd83dc 100644
--- a/R/convert_time_period.R
+++ b/R/convert_time_period.R
@@ -128,10 +128,7 @@ convert_time_period <- function(.x,
     }
     .rt <- unique(time_mapping$tperiod)
     missing_t <- .rt[!.rt %in% .x$t]
-    missing_time <- subset(
-      time_mapping,
-      tperiod %in% missing_t & refyear == year
-    )
+    missing_time <- time_mapping[tperiod %in% missing_t & refyear == year]
 
     if (!do_extrap) {
       if (do_past_extrap) {

From 1faebc375f00bdc1aeea5e7d67ab41e57e78e71a Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Tue, 21 Jul 2026 20:54:05 +0100
Subject: [PATCH 5/9] bench: add memory-capped engine benchmark harness with
 results

benchmarks/run_all.sh runs each cell in a fresh Rscript subprocess under
a systemd scope (MemoryMax, swap disabled) so over-budget runs are
OOM-killed and recorded as DNF instead of swap-thrashing the host; peak
RSS comes from GNU time. Matrix: synthetic GLOBIOM-shaped tables (1e5,
1e6, 4e6 rows) x {sum, mean, sumby} witch17->witch20, a witch17
pass-through, iso3->witch17, and two real 2.14M-row tables from the
witch-master GLOBIOM DuckDB, each x {legacy, fast}.

Results (6G cap, committed in benchmarks/results/results.md): the
legacy engine is OOM-killed on every 4e6-row case and on BOTH real
GLOBIOM tables, while the fast engine converts them in ~6s at <1GB
peak. Where both complete: 4.5-5.5x faster, 3.2-9.7x lower peak RSS,
identical checksums. Pass-through and iso3 routes unchanged (1.0x).
Coefficient-table build is ~14ms, so caching it across the convert_gdx
item loop is not worth the added state (decision recorded here).
---
 .Rbuildignore                  |  1 +
 benchmarks/.gitignore          |  4 ++
 benchmarks/README.md           | 13 ++++++
 benchmarks/cases.R             | 38 +++++++++++++++++
 benchmarks/results/results.csv | 15 +++++++
 benchmarks/results/results.md  | 28 +++++++++++++
 benchmarks/run_all.sh          | 62 ++++++++++++++++++++++++++++
 benchmarks/run_case.R          | 74 ++++++++++++++++++++++++++++++++++
 benchmarks/summarize.R         | 65 +++++++++++++++++++++++++++++
 9 files changed, 300 insertions(+)
 create mode 100644 benchmarks/.gitignore
 create mode 100644 benchmarks/README.md
 create mode 100644 benchmarks/cases.R
 create mode 100644 benchmarks/results/results.csv
 create mode 100644 benchmarks/results/results.md
 create mode 100755 benchmarks/run_all.sh
 create mode 100644 benchmarks/run_case.R
 create mode 100644 benchmarks/summarize.R

diff --git a/.Rbuildignore b/.Rbuildignore
index 5a42d78..70707be 100644
--- a/.Rbuildignore
+++ b/.Rbuildignore
@@ -10,3 +10,4 @@
 ^\.github$
 ^CLAUDE\.md$
 ^\.claude$
+^benchmarks$
diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore
new file mode 100644
index 0000000..7734f87
--- /dev/null
+++ b/benchmarks/.gitignore
@@ -0,0 +1,4 @@
+lib/
+results/raw/
+results/cells.csv
+results/dnf.txt
diff --git a/benchmarks/README.md b/benchmarks/README.md
new file mode 100644
index 0000000..a1fd0f3
--- /dev/null
+++ b/benchmarks/README.md
@@ -0,0 +1,13 @@
+# Region-conversion benchmarks
+
+Compares the `fast` (pair-coefficient) and `legacy` (iso3-explosion) engines
+of `convert_region()` on synthetic GLOBIOM-shaped tables and on real tables
+from the witch-master GLOBIOM DuckDB database.
+
+    benchmarks/run_all.sh [MemoryMax]   # default cap 6G
+
+Each cell runs in a fresh Rscript subprocess inside a systemd scope
+(`MemoryMax`, swap disabled) so an over-budget run is OOM-killed and recorded
+as DNF instead of swap-thrashing the machine. Peak RSS comes from
+`/usr/bin/time -v`. Set `WITCH_GLOBIOM_DB` to point at a different GLOBIOM
+database. Summary lands in `results/results.md`.
diff --git a/benchmarks/cases.R b/benchmarks/cases.R
new file mode 100644
index 0000000..7d49600
--- /dev/null
+++ b/benchmarks/cases.R
@@ -0,0 +1,38 @@
+# Benchmark input builders. Sourced by run_case.R.
+#
+# Synthetic tables mimic the GLOBIOM reporting shape: character id columns,
+# a character period column t, a region column and a numeric value.
+
+make_synthetic <- function(route, n_rows, seed = 1) {
+  set.seed(seed)
+  regs <- if (route == "iso3") {
+    witchtools::region_mappings[["witch17"]]$iso3
+  } else {
+    unique(witchtools::region_mappings[[route]][[route]])
+  }
+  k <- ceiling(n_rows / (length(regs) * 30L))
+  dt <- data.table::CJ(
+    variable = paste0("v", seq_len(k)),
+    scenario = c("s1", "s2"),
+    t = as.character(1:15),
+    region = regs,
+    sorted = FALSE
+  )
+  dt[, value := stats::runif(.N, 0, 1000)]
+  dt <- dt[seq_len(min(.N, n_rows))]
+  data.table::setnames(dt, "region", route)
+  return(dt)
+}
+
+# Load one table from a GLOBIOM DuckDB database (region column `n`, witch17).
+# The witchtools region column name is required by convert_region, so `n`
+# is renamed to `witch17` as convert_item would after region guessing.
+load_globiom_table <- function(db, table) {
+  con <- DBI::dbConnect(duckdb::duckdb(), dbdir = db, read_only = TRUE)
+  on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE)
+  dt <- data.table::setDT(
+    DBI::dbGetQuery(con, paste0("SELECT * FROM \"", table, "\""))
+  )
+  data.table::setnames(dt, "n", "witch17")
+  return(dt)
+}
diff --git a/benchmarks/results/results.csv b/benchmarks/results/results.csv
new file mode 100644
index 0000000..ac6be7a
--- /dev/null
+++ b/benchmarks/results/results.csv
@@ -0,0 +1,15 @@
+case,table,rows_in,op,route,wall_s_fast,wall_s_legacy,peak_mb_fast,peak_mb_legacy,checksum_fast,checksum_legacy,n_out_fast,n_out_legacy,speedup,mem_ratio,checksum_ok
+globiom,mean_gdpcap1,2142000,mean,witch17->witch20,6.393,,973.046875,,6058735.850763,,2520000,,,,
+globiom,sum_forest2,2142000,sum,witch17->witch20,5.587,,788.05859375,,205884097.075209,,2520000,,,,
+passthrough,"",1000000,sum,witch17->witch17,0.001,0.001,162.29296875,161.77734375,499922276.016161,499922276.016161,1000000,1000000,1,1,TRUE
+synthetic,"",100000,mean,witch17->witch20,0.26,1.314,145.55078125,462.43359375,58765467.46314,58765467.46314,117648,117648,5.1,3.2,TRUE
+synthetic,"",100000,sum,witch17->witch20,0.238,1.206,137.7890625,523.35546875,49962465.521146,49962465.521146,117648,117648,5.1,3.8,TRUE
+synthetic,"",100000,sumby,witch17->witch20,0.427,1.908,139.69921875,522.921875,49962465.521146,49962465.521146,117648,117648,4.5,3.7,TRUE
+synthetic,"",250000,sum,iso3->witch17,0.06,0.056,140.3515625,140.60546875,125007518.692503,125007518.692503,17000,17000,0.9,1,TRUE
+synthetic,"",1000000,mean,witch17->witch20,2.504,12.684,396.29296875,2729.66015625,588178669.527252,588178669.527252,1176471,1176471,5.1,6.9,TRUE
+synthetic,"",1000000,sum,iso3->witch17,0.338,0.347,284.5625,284.46875,499922276.016161,499922276.016161,68000,68000,1,1,TRUE
+synthetic,"",1000000,sum,witch17->witch20,2.322,12.673,347.828125,3362.203125,499922276.016161,499922276.016161,1176471,1176471,5.5,9.7,TRUE
+synthetic,"",1000000,sumby,witch17->witch20,4.185,20.097,425.35546875,3222.44140625,499922276.016161,499922276.016161,1176471,1176471,4.8,7.6,TRUE
+synthetic,"",4000000,mean,witch17->witch20,9.967,,1279.703125,,2352290164.43051,,4705884,,,,
+synthetic,"",4000000,sum,witch17->witch20,8.684,,1115.14453125,,1999407318.48482,,4705884,,,,
+synthetic,"",4000000,sumby,witch17->witch20,16.252,,1401.515625,,1999407318.48482,,4705884,,,,
diff --git a/benchmarks/results/results.md b/benchmarks/results/results.md
new file mode 100644
index 0000000..0c93de8
--- /dev/null
+++ b/benchmarks/results/results.md
@@ -0,0 +1,28 @@
+# convert_region engine benchmark
+
+Run: 2026-07-21 host cap: see run_all.sh
+
+| case | rows | op | route | legacy s | fast s | speedup | legacy MB | fast MB | mem x | checksums |
+|---|---|---|---|---|---|---|---|---|---|---|
+| globiom:mean_gdpcap1 | 2,142,000 | mean | witch17->witch20 | NA | 6.39 | NAx | NA | 973 | NAx | NA |
+| globiom:sum_forest2 | 2,142,000 | sum | witch17->witch20 | NA | 5.59 | NAx | NA | 788 | NAx | NA |
+| passthrough | 1,000,000 | sum | witch17->witch17 | 0.00 | 0.00 | 1.0x | 162 | 162 | 1.0x | match |
+| synthetic |   100,000 | mean | witch17->witch20 | 1.31 | 0.26 | 5.1x | 462 | 146 | 3.2x | match |
+| synthetic |   100,000 | sum | witch17->witch20 | 1.21 | 0.24 | 5.1x | 523 | 138 | 3.8x | match |
+| synthetic |   100,000 | sumby | witch17->witch20 | 1.91 | 0.43 | 4.5x | 523 | 140 | 3.7x | match |
+| synthetic |   250,000 | sum | iso3->witch17 | 0.06 | 0.06 | 0.9x | 141 | 140 | 1.0x | match |
+| synthetic | 1,000,000 | mean | witch17->witch20 | 12.68 | 2.50 | 5.1x | 2730 | 396 | 6.9x | match |
+| synthetic | 1,000,000 | sum | iso3->witch17 | 0.35 | 0.34 | 1.0x | 284 | 285 | 1.0x | match |
+| synthetic | 1,000,000 | sum | witch17->witch20 | 12.67 | 2.32 | 5.5x | 3362 | 348 | 9.7x | match |
+| synthetic | 1,000,000 | sumby | witch17->witch20 | 20.10 | 4.18 | 4.8x | 3222 | 425 | 7.6x | match |
+| synthetic | 4,000,000 | mean | witch17->witch20 | NA | 9.97 | NAx | NA | 1280 | NAx | NA |
+| synthetic | 4,000,000 | sum | witch17->witch20 | NA | 8.68 | NAx | NA | 1115 | NAx | NA |
+| synthetic | 4,000,000 | sumby | witch17->witch20 | NA | 16.25 | NAx | NA | 1402 | NAx | NA |
+
+## Did not finish (OOM-killed under the memory cap)
+
+- DNF syn_4000000_sum_legacy
+- DNF syn_4000000_mean_legacy
+- DNF syn_4000000_sumby_legacy
+- DNF globiom_mean_gdpcap1_legacy
+- DNF globiom_sum_forest2_legacy
diff --git a/benchmarks/run_all.sh b/benchmarks/run_all.sh
new file mode 100755
index 0000000..a8b0643
--- /dev/null
+++ b/benchmarks/run_all.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Run the full region-conversion benchmark matrix.
+#
+# Every cell runs in a fresh Rscript subprocess inside a systemd scope with a
+# hard memory cap and swap disabled, so an over-budget run is OOM-killed
+# (recorded as DNF) instead of thrashing the machine. Peak RSS is taken from
+# GNU time. All artifacts stay under benchmarks/ (never /tmp, which is tmpfs).
+#
+# Usage: benchmarks/run_all.sh [MemoryMax, default 6G]
+set -u
+cd "$(dirname "$0")/.."
+
+MEM="${1:-6G}"
+RES=benchmarks/results
+RAW=$RES/raw
+CSV=$RES/cells.csv
+mkdir -p "$RAW"
+rm -f "$CSV" "$RAW"/*.time
+
+# One-time install of the current source into a private library.
+mkdir -p benchmarks/lib
+R CMD INSTALL --no-docs --no-help . -l benchmarks/lib >/dev/null
+
+GLOBIOM_DB="${WITCH_GLOBIOM_DB:-/home/lolow/Sync/03-WITCH/witch-master/data_witch17/data_globiom2025_rep.duckdb}"
+
+run_cell() {
+  local name="$1"; shift
+  echo "== $name"
+  systemd-run --user --scope --same-dir -q \
+    -p "MemoryMax=$MEM" -p MemorySwapMax=0 \
+    /usr/bin/time -v -o "$RAW/$name.time" \
+    Rscript benchmarks/run_case.R --out="$CSV" "$@" \
+    || echo "DNF $name" >> "$RES/dnf.txt"
+}
+
+for engine in legacy fast; do
+  # Synthetic witch17 -> witch20
+  for rows in 100000 1000000 4000000; do
+    for op in sum mean sumby; do
+      run_cell "syn_${rows}_${op}_${engine}" \
+        --engine=$engine --case=synthetic --rows=$rows --op=$op
+    done
+  done
+  # Pass-through no-regression
+  run_cell "passthrough_1000000_${engine}" \
+    --engine=$engine --case=passthrough --rows=1000000
+  # iso3 upscale
+  for rows in 250000 1000000; do
+    run_cell "iso3_${rows}_sum_${engine}" \
+      --engine=$engine --case=synthetic --rows=$rows --route=iso3 --to=witch17
+  done
+  # Real GLOBIOM tables
+  if [ -f "$GLOBIOM_DB" ]; then
+    for tab in mean_gdpcap1 sum_forest2; do
+      op=${tab%%_*}
+      run_cell "globiom_${tab}_${engine}" \
+        --engine=$engine --case=globiom --db="$GLOBIOM_DB" --table="$tab" --op="$op"
+    done
+  fi
+done
+
+Rscript benchmarks/summarize.R
diff --git a/benchmarks/run_case.R b/benchmarks/run_case.R
new file mode 100644
index 0000000..c56b090
--- /dev/null
+++ b/benchmarks/run_case.R
@@ -0,0 +1,74 @@
+#!/usr/bin/env Rscript
+"Run one region-conversion benchmark cell.
+
+Usage:
+  run_case.R --engine=<e> --case=<c> --out=<csv> [--rows=<n>] [--op=<op>] [--route=<r>] [--to=<t>] [--table=<tab>] [--db=<db>] [--seed=<s>]
+
+Options:
+  --engine=<e>   Conversion engine: fast or legacy.
+  --case=<c>     Case name: synthetic | globiom | passthrough.
+  --out=<csv>    CSV file to append the result row to.
+  --rows=<n>     Synthetic row count [default: 1000000].
+  --op=<op>      Aggregation operator [default: sum].
+  --route=<r>    Input region mapping [default: witch17].
+  --to=<t>       Target region mapping [default: witch20].
+  --table=<tab>  GLOBIOM table name (case=globiom).
+  --db=<db>      GLOBIOM DuckDB path (case=globiom).
+  --seed=<s>     RNG seed [default: 1].
+" -> doc
+
+opts <- docopt::docopt(doc)
+
+lib <- file.path(dirname(sub("--file=", "", grep("--file=", commandArgs(), value = TRUE))), "lib")
+if (dir.exists(lib)) .libPaths(c(lib, .libPaths()))
+suppressMessages(library(witchtools))
+suppressMessages(library(data.table))
+source(file.path(dirname(sub("--file=", "", grep("--file=", commandArgs(), value = TRUE))), "cases.R"))
+
+engine <- opts$engine
+op <- opts$op
+route <- opts$route
+to <- opts$to
+
+if (opts$case == "globiom") {
+  dt <- load_globiom_table(opts$db, opts$table)
+} else {
+  dt <- make_synthetic(route, as.numeric(opts$rows), as.integer(opts$seed))
+}
+if (opts$case == "passthrough") to <- route
+n_in <- nrow(dt)
+
+# Coefficient-build time in isolation (informs the caching decision).
+coeff_s <- NA_real_
+if (engine == "fast" && route != "iso3" && to != route) {
+  t0 <- proc.time()
+  cf <- witchtools:::build_region_coeff(
+    witchtools::region_mappings[[route]], route,
+    witchtools::region_mappings[[to]], to,
+    witchtools::default_weights[["gdp"]]
+  )
+  coeff_s <- (proc.time() - t0)[["elapsed"]]
+}
+
+options(witchtools.convert_region_engine = engine)
+gc(reset = TRUE)
+t0 <- proc.time()
+res <- convert_region(dt, from_reg = route, to_reg = to, agg_operator = op,
+                      info = (op == "sumby"))
+wall <- (proc.time() - t0)[["elapsed"]]
+
+out <- if (is.list(res) && !data.table::is.data.table(res)) res$data else res
+row <- data.frame(
+  case = opts$case,
+  table = if (is.null(opts$table)) "" else opts$table,
+  engine = engine, rows_in = n_in, op = op,
+  route = paste0(route, "->", to),
+  wall_s = round(wall, 3),
+  n_out = nrow(out),
+  checksum = sprintf("%.6f", sum(out$value, na.rm = TRUE)),
+  coeff_s = round(coeff_s, 4)
+)
+write.table(row, opts$out, sep = ",", row.names = FALSE,
+            col.names = !file.exists(opts$out), append = file.exists(opts$out))
+cat(sprintf("%s %s %s rows=%d wall=%.2fs\n",
+            opts$case, engine, op, n_in, wall))
diff --git a/benchmarks/summarize.R b/benchmarks/summarize.R
new file mode 100644
index 0000000..aaebb2d
--- /dev/null
+++ b/benchmarks/summarize.R
@@ -0,0 +1,65 @@
+#!/usr/bin/env Rscript
+# Join the per-cell CSV with peak-RSS figures from the GNU time files and
+# write results.csv + results.md (side-by-side engines, speedup, mem ratio,
+# checksum agreement).
+suppressMessages(library(data.table))
+
+res_dir <- file.path("benchmarks", "results")
+cells <- fread(file.path(res_dir, "cells.csv"))
+
+rss_of <- function(f) {
+  ln <- grep("Maximum resident set size", readLines(f, warn = FALSE), value = TRUE)
+  if (!length(ln)) return(NA_real_)
+  as.numeric(sub(".*: ", "", ln)) / 1024 # MB
+}
+cells[, cell := paste0(
+  ifelse(case == "globiom", paste0("globiom_", table),
+    ifelse(case == "passthrough", paste0("passthrough_", rows_in),
+      paste0("syn_", rows_in, "_", op))),
+  ifelse(case == "synthetic" & route == "iso3->witch17",
+         "", "")
+)]
+# time-file names were written by run_all.sh; recover them from its scheme
+cells[, timefile := sprintf(
+  "%s/raw/%s_%s.time", res_dir,
+  ifelse(case == "globiom", paste0("globiom_", table),
+    ifelse(case == "passthrough", paste0("passthrough_", rows_in),
+      ifelse(route == "iso3->witch17", paste0("iso3_", rows_in, "_", op),
+        paste0("syn_", rows_in, "_", op)))),
+  engine
+)]
+cells[, peak_mb := sapply(timefile, rss_of)]
+
+wide <- dcast(cells,
+  case + table + rows_in + op + route ~ engine,
+  value.var = c("wall_s", "peak_mb", "checksum", "n_out")
+)
+wide[, speedup := round(wall_s_legacy / wall_s_fast, 1)]
+wide[, mem_ratio := round(peak_mb_legacy / peak_mb_fast, 1)]
+wide[, checksum_ok := checksum_legacy == checksum_fast |
+  abs(as.numeric(checksum_legacy) - as.numeric(checksum_fast)) <
+    1e-6 * pmax(abs(as.numeric(checksum_legacy)), 1)]
+
+fwrite(wide, file.path(res_dir, "results.csv"))
+
+md <- c(
+  "# convert_region engine benchmark",
+  "",
+  sprintf("Run: %s host cap: see run_all.sh", format(Sys.time(), "%Y-%m-%d")),
+  "",
+  "| case | rows | op | route | legacy s | fast s | speedup | legacy MB | fast MB | mem x | checksums |",
+  "|---|---|---|---|---|---|---|---|---|---|---|",
+  wide[, sprintf("| %s%s | %s | %s | %s | %.2f | %.2f | %.1fx | %.0f | %.0f | %.1fx | %s |",
+    case, ifelse(table == "", "", paste0(":", table)),
+    format(rows_in, big.mark = ","), op, route,
+    wall_s_legacy, wall_s_fast, speedup,
+    peak_mb_legacy, peak_mb_fast, mem_ratio,
+    ifelse(checksum_ok, "match", "MISMATCH"))]
+)
+dnf <- file.path(res_dir, "dnf.txt")
+if (file.exists(dnf)) {
+  md <- c(md, "", "## Did not finish (OOM-killed under the memory cap)", "",
+          paste0("- ", readLines(dnf)))
+}
+writeLines(md, file.path(res_dir, "results.md"))
+cat("Wrote", file.path(res_dir, "results.md"), "\n")

From d4e5a7223a3908b11aaa248e7b6eb249c571eac1 Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Tue, 21 Jul 2026 20:56:36 +0100
Subject: [PATCH 6/9] docs: NEWS entry for the fast region-conversion engine

Also declare the sw_from NSE symbol flagged by R CMD check.
---
 NEWS.md                 | 16 ++++++++++++++++
 R/convert_region_fast.R |  2 +-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/NEWS.md b/NEWS.md
index 4be98c8..96aa32b 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,5 +1,21 @@
 # witchtools (development version)
 
+## Performance
+
+- Region-to-region conversion (`convert_region()`, and `convert_table()` /
+  `convert_gdx()` / `convert_duckdb()` / `convert_sqlite()` through it) no
+  longer expands the data to country level. A new engine converts through a
+  precomputed region-pair coefficient table: on 1M-row tables it is ~5x
+  faster with up to ~10x lower peak memory, and it converts the 2.1M-row
+  GLOBIOM reporting tables witch17->witch20 in ~6s within 1 GB where the
+  previous implementation exhausted 6 GB (see `benchmarks/results/`).
+  Results are identical up to floating-point summation order (< 1e-12).
+  The previous engine remains available with
+  `options(witchtools.convert_region_engine = "legacy")`; country-level
+  (iso3) input and the `set1` operator always use it (`set1`'s rounding is
+  discontinuous, so summation reassociation could flip values at .5).
+- `subset()` replaced by direct indexing in the conversion hot paths.
+
 # witchtools 0.5.1
 
 ## Bug fixes
diff --git a/R/convert_region_fast.R b/R/convert_region_fast.R
index b2b4d6e..5c27d19 100644
--- a/R/convert_region_fast.R
+++ b/R/convert_region_fast.R
@@ -28,7 +28,7 @@
 # identical for every from-region that appears in the data at all.
 #' @noRd
 build_region_coeff <- function(rmap0, rname0, rmap1, rname1, agg_weight) {
-  iso3 <- weight <- w_pair <- .from <- NULL # due to NSE notes in R CMD check
+  iso3 <- weight <- w_pair <- .from <- sw_from <- NULL # due to NSE notes in R CMD check
 
   pw <- merge(rmap0, rmap1, by = "iso3")
   pw <- merge(pw, agg_weight, by = "iso3")

From 4e4334826bf9ca7949bc425013f5eff638457134 Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Tue, 21 Jul 2026 22:26:21 +0100
Subject: [PATCH 7/9] feat: add japan, korea and southafrica region sets

Emit is_japan, is_korea and is_southafrica in the generated
regions.inc, alongside the existing is_* sets. Needed by WITCH bunker
modules to set region-specific bounds that stay valid across region
mappings (jpnkor under witch17, japan/korea under witch20).

Selection uses the same GDP-majority rule as the other region sets,
so under witch17 jpnkor belongs to is_japan and is_korea is empty.
---
 NAMESPACE                  |  3 +++
 R/region_sets.R            | 52 ++++++++++++++++++++++++++++++++++++++
 R/witch_write_gams.R       | 12 +++++++++
 man/brazil_regions.Rd      |  3 +++
 man/china_regions.Rd       |  3 +++
 man/eu27_regions.Rd        |  3 +++
 man/eu28_regions.Rd        |  3 +++
 man/eu_regions.Rd          |  3 +++
 man/europe_regions.Rd      |  3 +++
 man/india_regions.Rd       |  3 +++
 man/indonesia_regions.Rd   |  3 +++
 man/japan_regions.Rd       | 39 ++++++++++++++++++++++++++++
 man/korea_regions.Rd       | 39 ++++++++++++++++++++++++++++
 man/oceania_regions.Rd     |  3 +++
 man/oecd_regions.Rd        |  3 +++
 man/region_id.Rd           |  3 +++
 man/southafrica_regions.Rd | 40 +++++++++++++++++++++++++++++
 man/ssa_regions.Rd         |  3 +++
 man/usa_regions.Rd         |  3 +++
 19 files changed, 224 insertions(+)
 create mode 100644 man/japan_regions.Rd
 create mode 100644 man/korea_regions.Rd
 create mode 100644 man/southafrica_regions.Rd

diff --git a/NAMESPACE b/NAMESPACE
index fc72d37..88cb878 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -19,12 +19,15 @@ export(iamc_native_regions_yml)
 export(iamc_region_mappings_yml)
 export(india_regions)
 export(indonesia_regions)
+export(japan_regions)
+export(korea_regions)
 export(oceania_regions)
 export(oecd_regions)
 export(premise_region_mapping)
 export(region_id)
 export(require_gdxtools)
 export(require_package)
+export(southafrica_regions)
 export(ssa_regions)
 export(usa_regions)
 export(witch_data)
diff --git a/R/region_sets.R b/R/region_sets.R
index 4365cf8..fe651dd 100644
--- a/R/region_sets.R
+++ b/R/region_sets.R
@@ -91,6 +91,58 @@ usa_regions <- function(region_mapping) {
   return(any_regions(usa_iso3,region_mapping))
 }
 
+#' Japan regions for a given region mapping.
+#'
+#' \code{japan_regions} returns a vector of region representing Japan.
+#'
+#' @family misc functions
+#'
+#' @param region_mapping a data.table of regional mapping.
+#' @return a vector of region name.
+#'
+#' @export
+#' @examples
+#' japan_regions(region_mappings[["witch17"]])
+japan_regions <- function(region_mapping) {
+  japan_iso3 <- c("JPN")
+  return(any_regions(japan_iso3,region_mapping))
+}
+
+#' Korea regions for a given region mapping.
+#'
+#' \code{korea_regions} returns a vector of region representing South Korea.
+#'
+#' @family misc functions
+#'
+#' @param region_mapping a data.table of regional mapping.
+#' @return a vector of region name.
+#'
+#' @export
+#' @examples
+#' korea_regions(region_mappings[["witch17"]])
+korea_regions <- function(region_mapping) {
+  korea_iso3 <- c("KOR")
+  return(any_regions(korea_iso3,region_mapping))
+}
+
+#' South Africa regions for a given region mapping.
+#'
+#' \code{southafrica_regions} returns a vector of region representing
+#' South Africa.
+#'
+#' @family misc functions
+#'
+#' @param region_mapping a data.table of regional mapping.
+#' @return a vector of region name.
+#'
+#' @export
+#' @examples
+#' southafrica_regions(region_mappings[["witch17"]])
+southafrica_regions <- function(region_mapping) {
+  southafrica_iso3 <- c("ZAF")
+  return(any_regions(southafrica_iso3,region_mapping))
+}
+
 #' China regions for a given region mapping.
 #'
 #' \code{china_regions} returns a vector of region representing China,
diff --git a/R/witch_write_gams.R b/R/witch_write_gams.R
index a5e9e1e..fdf2e7c 100644
--- a/R/witch_write_gams.R
+++ b/R/witch_write_gams.R
@@ -99,6 +99,18 @@ witch_write_gams <- function(region_mapping,
   writeLines("set is_ssa(n) 'Sub-Saharan Africa regions' /", finc)
   writeLines(ssa_regions(region_mapping), finc)
   writeLines("/;", finc)
+  # set is_japan
+  writeLines("set is_japan(n) 'Japan regions' /", finc)
+  writeLines(japan_regions(region_mapping), finc)
+  writeLines("/;", finc)
+  # set is_korea
+  writeLines("set is_korea(n) 'Korea regions' /", finc)
+  writeLines(korea_regions(region_mapping), finc)
+  writeLines("/;", finc)
+  # set is_southafrica
+  writeLines("set is_southafrica(n) 'South Africa regions' /", finc)
+  writeLines(southafrica_regions(region_mapping), finc)
+  writeLines("/;", finc)
 
   close(finc)
 
diff --git a/man/brazil_regions.Rd b/man/brazil_regions.Rd
index 919af2e..ebbbea1 100644
--- a/man/brazil_regions.Rd
+++ b/man/brazil_regions.Rd
@@ -27,9 +27,12 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/china_regions.Rd b/man/china_regions.Rd
index 1a74914..4cc760f 100644
--- a/man/china_regions.Rd
+++ b/man/china_regions.Rd
@@ -28,9 +28,12 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/eu27_regions.Rd b/man/eu27_regions.Rd
index 797e5bd..34b66f7 100644
--- a/man/eu27_regions.Rd
+++ b/man/eu27_regions.Rd
@@ -30,9 +30,12 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/eu28_regions.Rd b/man/eu28_regions.Rd
index cfeb3b2..ca90503 100644
--- a/man/eu28_regions.Rd
+++ b/man/eu28_regions.Rd
@@ -30,9 +30,12 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/eu_regions.Rd b/man/eu_regions.Rd
index 9ad09a1..f76c6c4 100644
--- a/man/eu_regions.Rd
+++ b/man/eu_regions.Rd
@@ -30,9 +30,12 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/europe_regions.Rd b/man/europe_regions.Rd
index 89a8324..17e386f 100644
--- a/man/europe_regions.Rd
+++ b/man/europe_regions.Rd
@@ -30,9 +30,12 @@ Other misc functions:
 \code{\link[=eu_regions]{eu_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/india_regions.Rd b/man/india_regions.Rd
index ccfd903..5431bd1 100644
--- a/man/india_regions.Rd
+++ b/man/india_regions.Rd
@@ -27,9 +27,12 @@ Other misc functions:
 \code{\link[=eu_regions]{eu_regions()}},
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/indonesia_regions.Rd b/man/indonesia_regions.Rd
index 264d256..e0a9213 100644
--- a/man/indonesia_regions.Rd
+++ b/man/indonesia_regions.Rd
@@ -27,9 +27,12 @@ Other misc functions:
 \code{\link[=eu_regions]{eu_regions()}},
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/japan_regions.Rd b/man/japan_regions.Rd
new file mode 100644
index 0000000..9cc42ad
--- /dev/null
+++ b/man/japan_regions.Rd
@@ -0,0 +1,39 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/region_sets.R
+\name{japan_regions}
+\alias{japan_regions}
+\title{Japan regions for a given region mapping.}
+\usage{
+japan_regions(region_mapping)
+}
+\arguments{
+\item{region_mapping}{a data.table of regional mapping.}
+}
+\value{
+a vector of region name.
+}
+\description{
+\code{japan_regions} returns a vector of region representing Japan.
+}
+\examples{
+japan_regions(region_mappings[["witch17"]])
+}
+\seealso{
+Other misc functions:
+\code{\link[=brazil_regions]{brazil_regions()}},
+\code{\link[=china_regions]{china_regions()}},
+\code{\link[=eu27_regions]{eu27_regions()}},
+\code{\link[=eu28_regions]{eu28_regions()}},
+\code{\link[=eu_regions]{eu_regions()}},
+\code{\link[=europe_regions]{europe_regions()}},
+\code{\link[=india_regions]{india_regions()}},
+\code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
+\code{\link[=oceania_regions]{oceania_regions()}},
+\code{\link[=oecd_regions]{oecd_regions()}},
+\code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
+\code{\link[=ssa_regions]{ssa_regions()}},
+\code{\link[=usa_regions]{usa_regions()}}
+}
+\concept{misc functions}
diff --git a/man/korea_regions.Rd b/man/korea_regions.Rd
new file mode 100644
index 0000000..5ccb09d
--- /dev/null
+++ b/man/korea_regions.Rd
@@ -0,0 +1,39 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/region_sets.R
+\name{korea_regions}
+\alias{korea_regions}
+\title{Korea regions for a given region mapping.}
+\usage{
+korea_regions(region_mapping)
+}
+\arguments{
+\item{region_mapping}{a data.table of regional mapping.}
+}
+\value{
+a vector of region name.
+}
+\description{
+\code{korea_regions} returns a vector of region representing South Korea.
+}
+\examples{
+korea_regions(region_mappings[["witch17"]])
+}
+\seealso{
+Other misc functions:
+\code{\link[=brazil_regions]{brazil_regions()}},
+\code{\link[=china_regions]{china_regions()}},
+\code{\link[=eu27_regions]{eu27_regions()}},
+\code{\link[=eu28_regions]{eu28_regions()}},
+\code{\link[=eu_regions]{eu_regions()}},
+\code{\link[=europe_regions]{europe_regions()}},
+\code{\link[=india_regions]{india_regions()}},
+\code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=oceania_regions]{oceania_regions()}},
+\code{\link[=oecd_regions]{oecd_regions()}},
+\code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
+\code{\link[=ssa_regions]{ssa_regions()}},
+\code{\link[=usa_regions]{usa_regions()}}
+}
+\concept{misc functions}
diff --git a/man/oceania_regions.Rd b/man/oceania_regions.Rd
index b2cac8e..7e91b2d 100644
--- a/man/oceania_regions.Rd
+++ b/man/oceania_regions.Rd
@@ -28,8 +28,11 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/oecd_regions.Rd b/man/oecd_regions.Rd
index 58d9b1e..e54d674 100644
--- a/man/oecd_regions.Rd
+++ b/man/oecd_regions.Rd
@@ -31,8 +31,11 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/region_id.Rd b/man/region_id.Rd
index 5299dc4..9ab4fce 100644
--- a/man/region_id.Rd
+++ b/man/region_id.Rd
@@ -28,8 +28,11 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
diff --git a/man/southafrica_regions.Rd b/man/southafrica_regions.Rd
new file mode 100644
index 0000000..946881e
--- /dev/null
+++ b/man/southafrica_regions.Rd
@@ -0,0 +1,40 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/region_sets.R
+\name{southafrica_regions}
+\alias{southafrica_regions}
+\title{South Africa regions for a given region mapping.}
+\usage{
+southafrica_regions(region_mapping)
+}
+\arguments{
+\item{region_mapping}{a data.table of regional mapping.}
+}
+\value{
+a vector of region name.
+}
+\description{
+\code{southafrica_regions} returns a vector of region representing
+South Africa.
+}
+\examples{
+southafrica_regions(region_mappings[["witch17"]])
+}
+\seealso{
+Other misc functions:
+\code{\link[=brazil_regions]{brazil_regions()}},
+\code{\link[=china_regions]{china_regions()}},
+\code{\link[=eu27_regions]{eu27_regions()}},
+\code{\link[=eu28_regions]{eu28_regions()}},
+\code{\link[=eu_regions]{eu_regions()}},
+\code{\link[=europe_regions]{europe_regions()}},
+\code{\link[=india_regions]{india_regions()}},
+\code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
+\code{\link[=oceania_regions]{oceania_regions()}},
+\code{\link[=oecd_regions]{oecd_regions()}},
+\code{\link[=region_id]{region_id()}},
+\code{\link[=ssa_regions]{ssa_regions()}},
+\code{\link[=usa_regions]{usa_regions()}}
+}
+\concept{misc functions}
diff --git a/man/ssa_regions.Rd b/man/ssa_regions.Rd
index fa853fa..0b199a2 100644
--- a/man/ssa_regions.Rd
+++ b/man/ssa_regions.Rd
@@ -28,9 +28,12 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=usa_regions]{usa_regions()}}
 }
 \concept{misc functions}
diff --git a/man/usa_regions.Rd b/man/usa_regions.Rd
index 807af6b..39fd5de 100644
--- a/man/usa_regions.Rd
+++ b/man/usa_regions.Rd
@@ -28,9 +28,12 @@ Other misc functions:
 \code{\link[=europe_regions]{europe_regions()}},
 \code{\link[=india_regions]{india_regions()}},
 \code{\link[=indonesia_regions]{indonesia_regions()}},
+\code{\link[=japan_regions]{japan_regions()}},
+\code{\link[=korea_regions]{korea_regions()}},
 \code{\link[=oceania_regions]{oceania_regions()}},
 \code{\link[=oecd_regions]{oecd_regions()}},
 \code{\link[=region_id]{region_id()}},
+\code{\link[=southafrica_regions]{southafrica_regions()}},
 \code{\link[=ssa_regions]{ssa_regions()}}
 }
 \concept{misc functions}

From 8c441ee0c782ff84c657c2496c149e0e0c640c11 Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Tue, 21 Jul 2026 22:53:02 +0100
Subject: [PATCH 8/9] test: expect the three new region sets in regions.inc

witch_write_gams() now emits is_japan, is_korea and is_southafrica,
so regions.inc declares 17 sets, not 14. Also check is_japan
membership against japan_regions().
---
 tests/testthat/test-witch-write-gams.R | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/testthat/test-witch-write-gams.R b/tests/testthat/test-witch-write-gams.R
index eecfe21..1271f05 100644
--- a/tests/testthat/test-witch-write-gams.R
+++ b/tests/testthat/test-witch-write-gams.R
@@ -43,7 +43,7 @@ test_that("witch_write_gams writes regions.inc with the iso3 and region sets", {
   # every declared set is closed
   n_open <- sum(grepl("^set .*/$", lines))
   expect_equal(n_open, sum(lines == "/;"))
-  expect_equal(n_open, 14L)
+  expect_equal(n_open, 17L)
 
   # a few membership checks against the region_sets helpers
   block <- function(header) {
@@ -53,6 +53,8 @@ test_that("witch_write_gams writes regions.inc with the iso3 and region sets", {
   }
   expect_setequal(block("set is_usa(n) 'USA regions' /"), usa_regions(rm))
   expect_setequal(block("set eu27(n) 'EU27 regions' /"), eu27_regions(rm))
+  expect_setequal(block("set is_japan(n) 'Japan regions' /"),
+                  japan_regions(rm))
 })
 
 test_that("witch_write_gams writes the coalition config and include", {

From 0b2ea80f7b0e752f3b74d0e8d9d9e0df7d075adf Mon Sep 17 00:00:00 2001
From: Laurent Drouet <ldrouet@pm.me>
Date: Tue, 21 Jul 2026 22:55:40 +0100
Subject: [PATCH 9/9] chore: bump version to 0.6.0 with NEWS entry

---
 DESCRIPTION |  2 +-
 NEWS.md     | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/DESCRIPTION b/DESCRIPTION
index 3ed2ed0..1a91c1f 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: witchtools
 Title: Data Management for Integrated Assessment Models
-Version: 0.5.1.9000
+Version: 0.6.0
 Authors@R: 
     c(person(given = "Laurent",
            family = "Drouet",
diff --git a/NEWS.md b/NEWS.md
index 96aa32b..b6d24a2 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,4 +1,12 @@
-# witchtools (development version)
+# witchtools 0.6.0
+
+## New features
+
+- New region sets `japan_regions()`, `korea_regions()` and
+  `southafrica_regions()`, emitted as `is_japan`, `is_korea` and
+  `is_southafrica` in the generated `regions.inc`. Selection uses the same
+  GDP-majority rule as the other region sets, so under `witch17` the
+  `jpnkor` region belongs to `is_japan` and `is_korea` is empty.
 
 ## Performance