Add functions to spatially group data with kmeans (and add UV)

README.md

@@ -411,6 +411,32 @@ training:
 
 For detailed information, see [docs/HYPERPARAMETER_TUNING.md](docs/HYPERPARAMETER_TUNING.md)
 
+### Grouping without a group column: K-means spatial blocks
+
+Grouped splits and grouped CV (above) need a `cv_group_column` — e.g. `patch_id` or
+`site` — to keep spatially autocorrelated samples together. If your data has
+coordinates but no ready-made group column, set `spatial_blocks` under `data:` and
+EcoNetToolkit will cluster samples into spatially coherent blocks with K-means and
+generate that column for you:
+
+```yaml
+data:
+  path: data/mangrove.csv
+  cv_group_column: spatial_block   # name for the generated column
+  spatial_blocks:
+    lon_col: pu_x                  # default: "longitude"
+    lat_col: pu_y                  # default: "latitude"
+    n_blocks: 10                   # default: 10 (capped at number of rows)
+
+  n_train_groups: 6
+  n_val_groups: 2
+  n_test_groups: 2
+```
+
+- `cv_group_column` must still be set (it's the name K-means writes its block labels to) — it just no longer needs to already exist in the CSV.
+- Each row is assigned to one of `n_blocks` clusters based on `lon_col`/`lat_col`, so neighbouring points end up in the same block and are never split across train/val/test.
+- Works with both grouped train/val/test splits (`prepare_grouped_splits`) and `GroupKFold` cross-validation.
+
 ## Using your own data
 
 1. Place your CSV file in the `data` folder.

ecosci/data/__init__.py

@@ -1,5 +1,6 @@
 """Data loading, preprocessing, and splitting utilities."""
 
 from .loader import CSVDataLoader
+from .spatial import assign_spatial_blocks
 
-__all__ = ["CSVDataLoader"]
+__all__ = ["CSVDataLoader", "assign_spatial_blocks"]

ecosci/data/loader.py

@@ -10,6 +10,7 @@
     encode_labels,
 )
 from .splitting import prepare_cv_folds, prepare_grouped_splits
+from .spatial import assign_spatial_blocks
 
 
 class CSVDataLoader:
@@ -50,6 +51,13 @@ class CSVDataLoader:
         "classification" or "regression"
     cv_group_column : str, optional
         Column name for grouping in cross-validation
+    spatial_blocks : dict, optional
+        If set, automatically derives `cv_group_column` from coordinates
+        instead of requiring a pre-existing column. Keys: `lon_col`,
+        `lat_col` (default "longitude"/"latitude"), `n_blocks` (default 10).
+        Samples are clustered into spatially coherent blocks with KMeans, so
+        grouping/CV on `cv_group_column` keeps spatially autocorrelated
+        neighbours together rather than splitting them across train/test.
     """
 
     def __init__(
@@ -65,6 +73,7 @@ def __init__(
         impute_strategy: str = "mean",
         problem_type: str = "classification",
         cv_group_column: Optional[str] = None,
+        spatial_blocks: Optional[dict] = None,
     ):
         self.path = path
         self.features = features
@@ -85,10 +94,31 @@ def __init__(
         self.impute_strategy = impute_strategy
         self.problem_type = problem_type
         self.cv_group_column = cv_group_column
+        self.spatial_blocks = spatial_blocks
 
     def load(self) -> pd.DataFrame:
-        """Read the CSV into a DataFrame."""
+        """Read the CSV into a DataFrame, adding spatial-block groups if configured."""
         df = pd.read_csv(self.path)
+        if self.spatial_blocks is not None:
+            if self.cv_group_column is None:
+                raise ValueError(
+                    "cv_group_column must be set when spatial_blocks is configured"
+                )
+            df[self.cv_group_column] = assign_spatial_blocks(
+                df,
+                lon_col=self.spatial_blocks.get("lon_col", "longitude"),
+                lat_col=self.spatial_blocks.get("lat_col", "latitude"),
+                n_blocks=self.spatial_blocks.get("n_blocks", 10),
+                random_state=self.spatial_blocks.get(
+                    "random_state", self.random_state
+                ),
+            )
+            print(
+                f"\nAssigned {df[self.cv_group_column].nunique()} spatial blocks "
+                f"from '{self.spatial_blocks.get('lon_col', 'longitude')}'/"
+                f"'{self.spatial_blocks.get('lat_col', 'latitude')}' "
+                f"-> '{self.cv_group_column}'"
+            )
         return df
 
     def _get_feature_names_after_transform(self, preprocessor, numeric_cols, cat_cols):

ecosci/data/spatial.py

@@ -0,0 +1,48 @@
+"""Spatial partitioning utilities.
+
+Random train/test splits on geo-referenced ecological data tend to place
+spatially autocorrelated neighbours (e.g. adjacent plots) on both sides of the
+split, which lets models "cheat" by memorising local conditions rather than
+learning a relationship that generalises to new sites. Grouping samples into
+spatial blocks first, then splitting/cross-validating on those blocks (e.g.
+via GroupKFold), keeps neighbours together on one side of the split.
+"""
+
+import pandas as pd
+
+
+def assign_spatial_blocks(
+    df: pd.DataFrame,
+    lon_col: str = "longitude",
+    lat_col: str = "latitude",
+    n_blocks: int = 10,
+    random_state: int = 42,
+) -> pd.Series:
+    """Cluster samples into spatially coherent blocks using KMeans on coordinates.
+
+    Parameters
+    ----------
+    df : DataFrame
+        Must contain `lon_col` and `lat_col`.
+    lon_col, lat_col : str
+        Coordinate column names.
+    n_blocks : int
+        Number of spatial blocks (i.e. groups) to create. Capped at the number
+        of rows if there are fewer samples than blocks.
+    random_state : int
+        Seed for KMeans initialisation.
+
+    Returns
+    -------
+    Series
+        Integer block label per row, named "spatial_block", aligned to `df`'s index.
+    """
+    from sklearn.cluster import KMeans
+
+    coords = df[[lon_col, lat_col]].to_numpy()
+    n_blocks = max(1, min(n_blocks, len(df)))
+
+    km = KMeans(n_clusters=n_blocks, random_state=random_state, n_init=10)
+    labels = km.fit_predict(coords)
+
+    return pd.Series(labels, index=df.index, name="spatial_block")

pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "econet"
+version = "0.1.0"
+requires-python = ">=3.10,<3.11"
+dependencies = [
+    "numpy>=1.20",
+    "pandas>=1.2",
+    "scikit-learn>=1.0",
+    "xgboost>=1.6",
+    "matplotlib>=3.9",
+    "seaborn>=0.11",
+    "pyyaml>=6.0",
+    "joblib>=1.0",
+    "pytest>=7.0",
+    "tqdm>=4.0",
+]

run.py

@@ -41,6 +41,7 @@
     impute_strategy=data_cfg.get("impute_strategy", "mean"),
     problem_type=problem_type,
     cv_group_column=cv_group_column,
+    spatial_blocks=data_cfg.get("spatial_blocks"),
 )
 
 # Get output directory