From ad711f5811e35da5ff8903af909fac189d2c51f4 Mon Sep 17 00:00:00 2001 From: Chris van den Berg <11998981+bergvca@users.noreply.github.com> Date: Mon, 22 Jun 2026 21:32:22 +0200 Subject: [PATCH 1/9] added sp_matmul_rs in pyproject.toml and updated docstring --- pyproject.toml | 1 + string_grouper/string_grouper.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 22710f3d..d17a3a0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "numpy>=2.0", "sparse-dot-topn>=1.1.0", "loguru>0.7.0", + "sp_matmul_rs>=0.0.1" ] [build-system] diff --git a/string_grouper/string_grouper.py b/string_grouper/string_grouper.py index 6dff2d1e..b65c97fb 100644 --- a/string_grouper/string_grouper.py +++ b/string_grouper/string_grouper.py @@ -2,7 +2,6 @@ import numpy as np import re import multiprocessing -import warnings from sklearn.feature_extraction.text import TfidfVectorizer from scipy.sparse import vstack from scipy.sparse import csr_matrix @@ -21,6 +20,7 @@ DEFAULT_MIN_SIMILARITY: float = 0.8 # minimum cosine similarity for an item to be considered a match DEFAULT_N_PROCESSES: int = multiprocessing.cpu_count() - 1 DEFAULT_IGNORE_CASE: bool = True # ignores case by default +DEFAULT_USE_SP_MATMUL_RS: bool = True # use sp_matmul_rs or the sparse_dot_topn as matrix multiplication library DEFAULT_DROP_INDEX: bool = False # includes index-columns in output DEFAULT_REPLACE_NA: bool = False # when finding the most similar strings, does not replace NaN values in most # similar string index-columns with corresponding duplicates-index values @@ -49,7 +49,7 @@ GROUP_REP_PREFIX: str = 'group_rep_' # used to prefix and name columns of the output of StringGrouper._deduplicate -# High level functions +# High-level functions def compute_pairwise_similarities(string_series_1: pd.Series, @@ -170,6 +170,9 @@ class StringGrouperConfig(NamedTuple): :param number_of_processes: int. The number of processes used by the cosine similarity calculation. Defaults to number of cores on a machine - 1. :param ignore_case: bool. Whether or not case should be ignored. Defaults to True (ignore case). + use_sp_matmul_rs: bool. Whether or not to use sp_matmul_rs or the sparse_dot_topn as matrix multiplication library. + sp_matmul_rs does the chunking internally and has further optimizations, but is not battle-tested as much. + Defaults to True. :param ignore_index: whether or not to exclude string Series index-columns in output. Defaults to False. :param include_zeroes: when the minimum cosine similarity <=0, determines whether zero-similarity matches appear in the output. Defaults to True. @@ -193,6 +196,7 @@ class StringGrouperConfig(NamedTuple): min_similarity: float = DEFAULT_MIN_SIMILARITY number_of_processes: int = DEFAULT_N_PROCESSES ignore_case: bool = DEFAULT_IGNORE_CASE + use_sp_matmul_rs: bool = DEFAULT_USE_SP_MATMUL_RS ignore_index: bool = DEFAULT_DROP_INDEX include_zeroes: bool = DEFAULT_INCLUDE_ZEROES replace_na: bool = DEFAULT_REPLACE_NA From 68fe88c824fb6608f70a23b61e6b1aeec7232554 Mon Sep 17 00:00:00 2001 From: Chris van den Berg <11998981+bergvca@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:16:45 +0200 Subject: [PATCH 2/9] added support for sp_matmul_rs, this is now the default to perform cosine similarity calculations --- string_grouper/string_grouper.py | 106 +++++++++++++-------- string_grouper/test/test_string_grouper.py | 29 +++--- 2 files changed, 82 insertions(+), 53 deletions(-) diff --git a/string_grouper/string_grouper.py b/string_grouper/string_grouper.py index b65c97fb..1825cb31 100644 --- a/string_grouper/string_grouper.py +++ b/string_grouper/string_grouper.py @@ -3,12 +3,13 @@ import re import multiprocessing from sklearn.feature_extraction.text import TfidfVectorizer -from scipy.sparse import vstack +from scipy.sparse import vstack, csr_matrix from scipy.sparse import csr_matrix from scipy.sparse import lil_matrix from scipy.sparse.csgraph import connected_components from typing import Tuple, NamedTuple, List, Optional, Union from sparse_dot_topn import sp_matmul_topn, zip_sp_matmul_topn +from sp_matmul_rs import sp_matmul_topn as sp_matmul_topn_rs from functools import wraps from unicodedata import normalize from loguru import logger @@ -170,8 +171,8 @@ class StringGrouperConfig(NamedTuple): :param number_of_processes: int. The number of processes used by the cosine similarity calculation. Defaults to number of cores on a machine - 1. :param ignore_case: bool. Whether or not case should be ignored. Defaults to True (ignore case). - use_sp_matmul_rs: bool. Whether or not to use sp_matmul_rs or the sparse_dot_topn as matrix multiplication library. - sp_matmul_rs does the chunking internally and has further optimizations, but is not battle-tested as much. + :param use_sp_matmul_rs: bool. Whether or not to use sp_matmul_rs or the sparse_dot_topn as matrix multiplication + library. sp_matmul_rs does the chunking internally and has further optimizations, but is not battle-tested. Defaults to True. :param ignore_index: whether or not to exclude string Series index-columns in output. Defaults to False. :param include_zeroes: when the minimum cosine similarity <=0, determines whether zero-similarity matches @@ -303,7 +304,7 @@ def _set_options(self, **kwargs): self._validate_group_rep_specs() self._validate_tfidf_matrix_dtype() self._validate_replace_na_and_drop() - StringGrouper._validate_n_blocks(self._config.n_blocks) + self._validate_n_blocks() self.is_build = False def _build_corpus(self): @@ -388,35 +389,10 @@ def fit(self): """ master_matrix, duplicate_matrix = self._get_tf_idf_matrices() - b_left = max(1, round(len(self._left_Series)/1e6)) # arbitrary, big enough not to split both left and right often - b_right = max(1, round(len(self._right_Series)/4e3)) # based on tests and observations - size_guess_block = (b_left, b_right) # inversion of left and right series was introduced in 0.6 ? - - if self._n_blocks is None: - if size_guess_block != (1,1): - logger.info("n_blocks parameter is not set so data will be split into smaller chunks, n_blocks = (" + str(size_guess_block[0]) +","+ str(size_guess_block[1])+")") - self._n_blocks = size_guess_block - - # do the matching - if self._n_blocks == (1,1): - try: - matches = self._build_matches(master_matrix, duplicate_matrix, self._n_blocks) - except OverflowError: - logger.warning( - "An OverflowError occurred but is being " + - "handled. The input data will be automatically " + - "split-up into smaller chunks which will then be " + - "processed one chunk at a time. To prevent " + - "OverflowError, use the n_blocks parameter to split-up " + - "the data manually into small enough chunks" + - ", n_blocks = (" + - str(size_guess_block[0]), - ",", - str(size_guess_block[1])+")" - ) - matches = self._build_matches(master_matrix, duplicate_matrix, size_guess_block) + if self._config.use_sp_matmul_rs: + matches = self._build_matches_rs(duplicate_matrix, master_matrix) else: - matches = self._build_matches(master_matrix, duplicate_matrix, self._n_blocks) + matches = self._calc_blocks_and_build_matches(duplicate_matrix, master_matrix) self._true_max_n_matches = np.diff(matches.indptr).max() @@ -434,6 +410,44 @@ def fit(self): self.is_build = True return self + def _calc_blocks_and_build_matches(self, duplicate_matrix: csr_matrix, master_matrix: csr_matrix) -> csr_matrix: + """ + Calculates the optimal blocks and builds matching data from the provided matrices. Uses the legacy + sp_dot_topn function to calculate matches. + """ + b_left = max(1, round(len(self._left_Series) / 1e6)) # arbitrary, big enough not to split both left and right often + b_right = max(1, round(len(self._right_Series) / 4e3)) # based on tests and observations + size_guess_block = (b_left, b_right) # inversion of left and right series was introduced in 0.6 ? + + if self._n_blocks is None: + if size_guess_block != (1, 1): + logger.info( + "n_blocks parameter is not set so data will be split into smaller chunks, n_blocks = (" + + str(size_guess_block[0]) + "," + str(size_guess_block[1]) + ")") + self._n_blocks = size_guess_block + + # do the matching + if self._n_blocks == (1, 1): + try: + matches = self._build_matches(master_matrix, duplicate_matrix, self._n_blocks) + except OverflowError: + logger.warning( + "An OverflowError occurred but is being " + + "handled. The input data will be automatically " + + "split-up into smaller chunks which will then be " + + "processed one chunk at a time. To prevent " + + "OverflowError, use the n_blocks parameter to split-up " + + "the data manually into small enough chunks" + + ", n_blocks = (" + + str(size_guess_block[0]), + ",", + str(size_guess_block[1]) + ")" + ) + matches = self._build_matches(master_matrix, duplicate_matrix, size_guess_block) + else: + matches = self._build_matches(master_matrix, duplicate_matrix, self._n_blocks) + return matches + def dot(self) -> pd.Series: """Computes the row-wise similarity scores between strings in _master and _duplicates""" if len(self._master) != len(self._duplicates): @@ -755,6 +769,18 @@ def chunk_list(lst, n): return C + def _build_matches_rs(self, + master_matrix: csr_matrix, + duplicate_matrix: csr_matrix) -> csr_matrix: + """Builds the cossine similarity matrix of two csr matrices using sp_matmul_topn_rs for faster computation""" + return sp_matmul_topn_rs( + master_matrix, + duplicate_matrix.transpose(), + top_n = self._max_n_matches, + threshold = self._config.min_similarity, + sort = True, + n_threads = self._config.number_of_processes + ) def _get_matches_list(self, matches: csr_matrix @@ -940,19 +966,20 @@ def _validate_replace_na_and_drop(self): "index if the number of index-levels does not equal the number of index-columns." ) - @staticmethod - def _validate_n_blocks(n_blocks): + def _validate_n_blocks(self): errmsg = "Invalid option value for parameter n_blocks: " "n_blocks must be None or a tuple of 2 integers greater than 0." - if n_blocks is None: + if self._config.n_blocks is None: return - if not isinstance(n_blocks, tuple): + if self._config.n_blocks is not None and self._config.use_sp_matmul_rs: + raise Exception("If sp_matmul_rs is True, n_blocks is cannot be set and is calculated automatically.") + if not isinstance(self._config.n_blocks, tuple): raise Exception(errmsg) - if len(n_blocks) != 2: + if len(self._config.n_blocks) != 2: raise Exception(errmsg) - if not (isinstance(n_blocks[0], int) and isinstance(n_blocks[1], int)): + if not (isinstance(self._config.n_blocks[0], int) and isinstance(self._config.n_blocks[1], int)): raise Exception(errmsg) - if (n_blocks[0] < 1) or (n_blocks[1] < 1): + if (self._config.n_blocks[0] < 1) or (self._config.n_blocks[1] < 1): raise Exception(errmsg) @staticmethod @@ -1012,3 +1039,4 @@ def _validate_id_data(master, duplicates, master_id, duplicates_id): raise Exception('Both master and master_id must be pandas.Series of the same length.') if duplicates is not None and duplicates_id is not None and len(duplicates) != len(duplicates_id): raise Exception('Both duplicates and duplicates_id must be pandas.Series of the same length.') + diff --git a/string_grouper/test/test_string_grouper.py b/string_grouper/test/test_string_grouper.py index b15a7493..11222404 100644 --- a/string_grouper/test/test_string_grouper.py +++ b/string_grouper/test/test_string_grouper.py @@ -6,7 +6,7 @@ DEFAULT_REGEX, DEFAULT_NGRAM_SIZE, DEFAULT_N_PROCESSES, DEFAULT_IGNORE_CASE, \ StringGrouperConfig, StringGrouper, StringGrouperNotFitException, \ match_most_similar, group_similar_strings, match_strings, \ - compute_pairwise_similarities + compute_pairwise_similarities, DEFAULT_USE_SP_MATMUL_RS from unittest.mock import patch, Mock @@ -100,6 +100,7 @@ def test_config_defaults(self): self.assertEqual(config.ngram_size, DEFAULT_NGRAM_SIZE) self.assertEqual(config.number_of_processes, DEFAULT_N_PROCESSES) self.assertEqual(config.ignore_case, DEFAULT_IGNORE_CASE) + self.assertEqual(config.use_sp_matmul_rs, DEFAULT_USE_SP_MATMUL_RS) def test_config_immutable(self): """Configurations should be immutable""" @@ -133,11 +134,11 @@ def fix_row_order(df): df1 = simple_example.customers_df2['Customer Name'] # first do manual blocking - sg = StringGrouper(df1, min_similarity=0.1) + sg = StringGrouper(df1, min_similarity=0.1, use_sp_matmul_rs=False) pd.testing.assert_series_equal(sg.master, df1) self.assertEqual(sg.duplicates, None) - matches = fix_row_order(sg.match_strings(df1, n_blocks=(1, 1))) + matches = fix_row_order(sg.match_strings(df1, n_blocks=(1, 1), use_sp_matmul_rs=False)) self.assertEqual(sg._config.n_blocks, (1, 1)) # Create a custom wrapper for this StringGrouper instance's @@ -201,43 +202,43 @@ def fix_row_order(df): matches11 = fix_row_order(match_strings(df1, min_similarity=0.1)) matches12 = fix_row_order( - match_strings(df1, n_blocks=(1, 2), min_similarity=0.1)) + match_strings(df1, n_blocks=(1, 2), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches12) matches13 = fix_row_order( - match_strings(df1, n_blocks=(1, 3), min_similarity=0.1)) + match_strings(df1, n_blocks=(1, 3), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches13) matches14 = fix_row_order( - match_strings(df1, n_blocks=(1, 4), min_similarity=0.1)) + match_strings(df1, n_blocks=(1, 4), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches14) matches15 = fix_row_order( - match_strings(df1, n_blocks=(1, 5), min_similarity=0.1)) + match_strings(df1, n_blocks=(1, 5), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches15) matches16 = fix_row_order( - match_strings(df1, n_blocks=(1, 6), min_similarity=0.1)) + match_strings(df1, n_blocks=(1, 6), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches16) matches17 = fix_row_order( - match_strings(df1, n_blocks=(1, 7), min_similarity=0.1)) + match_strings(df1, n_blocks=(1, 7), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches17) matches18 = fix_row_order( - match_strings(df1, n_blocks=(1, 8), min_similarity=0.1)) + match_strings(df1, n_blocks=(1, 8), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches18) matches21 = fix_row_order( - match_strings(df1, n_blocks=(2, 1), min_similarity=0.1)) + match_strings(df1, n_blocks=(2, 1), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches21) matches22 = fix_row_order( - match_strings(df1, n_blocks=(2, 2), min_similarity=0.1)) + match_strings(df1, n_blocks=(2, 2), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches22) matches32 = fix_row_order( - match_strings(df1, n_blocks=(3, 2), min_similarity=0.1)) + match_strings(df1, n_blocks=(3, 2), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches32) # Create a custom wrapper for this StringGrouper instance's @@ -296,7 +297,7 @@ def fix_row_order(df): matches11 = fix_row_order(match_strings(df1, df2, min_similarity=0.1)) matches12 = fix_row_order( - match_strings(df1, df2, n_blocks=(1, 2), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(1, 2), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches12) matches13 = fix_row_order( From e7cdb1fced80f8db5a1c14b64c7913820fe5c5ae Mon Sep 17 00:00:00 2001 From: Chris van den Berg <11998981+bergvca@users.noreply.github.com> Date: Fri, 3 Jul 2026 09:34:33 +0200 Subject: [PATCH 3/9] fixed issue with argument order and made unittest pass --- string_grouper/string_grouper.py | 6 +++--- string_grouper/test/test_string_grouper.py | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/string_grouper/string_grouper.py b/string_grouper/string_grouper.py index 1825cb31..a567d5e3 100644 --- a/string_grouper/string_grouper.py +++ b/string_grouper/string_grouper.py @@ -390,9 +390,9 @@ def fit(self): master_matrix, duplicate_matrix = self._get_tf_idf_matrices() if self._config.use_sp_matmul_rs: - matches = self._build_matches_rs(duplicate_matrix, master_matrix) + matches = self._build_matches_rs(master_matrix, duplicate_matrix) else: - matches = self._calc_blocks_and_build_matches(duplicate_matrix, master_matrix) + matches = self._calc_blocks_and_build_matches(master_matrix, duplicate_matrix) self._true_max_n_matches = np.diff(matches.indptr).max() @@ -410,7 +410,7 @@ def fit(self): self.is_build = True return self - def _calc_blocks_and_build_matches(self, duplicate_matrix: csr_matrix, master_matrix: csr_matrix) -> csr_matrix: + def _calc_blocks_and_build_matches(self, master_matrix: csr_matrix, duplicate_matrix: csr_matrix) -> csr_matrix: """ Calculates the optimal blocks and builds matching data from the provided matrices. Uses the legacy sp_dot_topn function to calculate matches. diff --git a/string_grouper/test/test_string_grouper.py b/string_grouper/test/test_string_grouper.py index 11222404..c040f802 100644 --- a/string_grouper/test/test_string_grouper.py +++ b/string_grouper/test/test_string_grouper.py @@ -272,9 +272,9 @@ def test_overflow_error_with(OverflowThreshold, n_blocks): + (1 if len(df1) % n_blocks[1] > 0 else 0)) if (max_left_block_size + max_right_block_size) > OverflowThreshold: with self.assertRaises(Exception): - _ = sg.match_strings(df1, n_blocks=n_blocks) + _ = sg.match_strings(df1, n_blocks=n_blocks, use_sp_matmul_rs=False) else: - matches_manual = fix_row_order(sg.match_strings(df1, n_blocks=n_blocks)) + matches_manual = fix_row_order(sg.match_strings(df1, n_blocks=n_blocks, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches_manual) test_overflow_error_with(OverflowThreshold=20, n_blocks=(1, 1)) @@ -301,39 +301,39 @@ def fix_row_order(df): pd.testing.assert_frame_equal(matches11, matches12) matches13 = fix_row_order( - match_strings(df1, df2, n_blocks=(1, 3), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(1, 3), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches13) matches14 = fix_row_order( - match_strings(df1, df2, n_blocks=(1, 4), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(1, 4), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches14) matches15 = fix_row_order( - match_strings(df1, df2, n_blocks=(1, 5), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(1, 5), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches15) matches16 = fix_row_order( - match_strings(df1, df2, n_blocks=(1, 6), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(1, 6), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches16) matches17 = fix_row_order( - match_strings(df1, df2, n_blocks=(1, 7), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(1, 7), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches17) matches18 = fix_row_order( - match_strings(df1, df2, n_blocks=(1, 8), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(1, 8), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches18) matches21 = fix_row_order( - match_strings(df1, df2, n_blocks=(2, 1), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(2, 1), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches21) matches22 = fix_row_order( - match_strings(df1, df2, n_blocks=(2, 2), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(2, 2), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches22) matches32 = fix_row_order( - match_strings(df1, df2, n_blocks=(3, 2), min_similarity=0.1)) + match_strings(df1, df2, n_blocks=(3, 2), min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches11, matches32) def test_n_blocks_bad_option_value(self): From 73e61d0b28d10a5a5f49c85b8bce3281f9857b87 Mon Sep 17 00:00:00 2001 From: Chris van den Berg <11998981+bergvca@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:40:54 +0200 Subject: [PATCH 4/9] Added set of tests to verify sp_matmul_rs backend equivalence --- string_grouper/string_grouper.py | 2 +- string_grouper/test/test_string_grouper.py | 82 ++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/string_grouper/string_grouper.py b/string_grouper/string_grouper.py index a567d5e3..9a544558 100644 --- a/string_grouper/string_grouper.py +++ b/string_grouper/string_grouper.py @@ -172,7 +172,7 @@ class StringGrouperConfig(NamedTuple): Defaults to number of cores on a machine - 1. :param ignore_case: bool. Whether or not case should be ignored. Defaults to True (ignore case). :param use_sp_matmul_rs: bool. Whether or not to use sp_matmul_rs or the sparse_dot_topn as matrix multiplication - library. sp_matmul_rs does the chunking internally and has further optimizations, but is not battle-tested. + library. sp_matmul_rs does the chunking internally and has further optimizations but is not battle-tested. Defaults to True. :param ignore_index: whether or not to exclude string Series index-columns in output. Defaults to False. :param include_zeroes: when the minimum cosine similarity <=0, determines whether zero-similarity matches diff --git a/string_grouper/test/test_string_grouper.py b/string_grouper/test/test_string_grouper.py index c040f802..d5c7d685 100644 --- a/string_grouper/test/test_string_grouper.py +++ b/string_grouper/test/test_string_grouper.py @@ -1045,5 +1045,87 @@ def test_prior_matches_added(self): self.assertEqual(1, len(df.deduped.unique())) +class SpMatmulRsEquivalenceTest(unittest.TestCase): + """Tests that the sp_matmul_rs backend (use_sp_matmul_rs=True) yields the same results + as the legacy sparse_dot_topn backend (use_sp_matmul_rs=False)""" + + sort_cols = ['right_index', 'left_index'] + + def fix_row_order(self, df): + return df.sort_values(self.sort_cols).reset_index(drop=True) + + def test_match_strings_single_series(self): + """match_strings on a single Series (self-join) should be backend-independent""" + simple_example = SimpleExample() + df1 = simple_example.customers_df2['Customer Name'] + matches_rs = self.fix_row_order( + match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=True)) + matches_legacy = self.fix_row_order( + match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=False)) + pd.testing.assert_frame_equal(matches_legacy, matches_rs) + + def test_match_strings_two_series(self): + """match_strings on two Series should be backend-independent""" + simple_example = SimpleExample() + df1 = simple_example.customers_df['Customer Name'] + df2 = simple_example.customers_df2['Customer Name'] + matches_rs = self.fix_row_order( + match_strings(df1, df2, min_similarity=0.1, use_sp_matmul_rs=True)) + matches_legacy = self.fix_row_order( + match_strings(df1, df2, min_similarity=0.1, use_sp_matmul_rs=False)) + pd.testing.assert_frame_equal(matches_legacy, matches_rs) + + def test_match_strings_with_ids(self): + """match_strings with master_id and duplicates_id should be backend-independent""" + simple_example = SimpleExample() + matches_rs = self.fix_row_order( + match_strings(simple_example.customers_df['Customer Name'], + simple_example.customers_df2['Customer Name'], + master_id=simple_example.customers_df['Customer ID'], + duplicates_id=simple_example.customers_df2['Customer ID'], + min_similarity=0.1, + use_sp_matmul_rs=True)) + matches_legacy = self.fix_row_order( + match_strings(simple_example.customers_df['Customer Name'], + simple_example.customers_df2['Customer Name'], + master_id=simple_example.customers_df['Customer ID'], + duplicates_id=simple_example.customers_df2['Customer ID'], + min_similarity=0.1, + use_sp_matmul_rs=False)) + pd.testing.assert_frame_equal(matches_legacy, matches_rs) + + def test_match_most_similar(self): + """match_most_similar should be backend-independent""" + test_series_1 = pd.Series(['foooo', 'bar', 'baz']) + test_series_2 = pd.Series(['foooo', 'bar', 'baz', 'foooob']) + result_rs = match_most_similar(test_series_1, test_series_2, + ignore_index=True, use_sp_matmul_rs=True) + result_legacy = match_most_similar(test_series_1, test_series_2, + ignore_index=True, use_sp_matmul_rs=False) + pd.testing.assert_series_equal(result_legacy, result_rs) + + def test_group_similar_strings(self): + """group_similar_strings should be backend-independent""" + simple_example = SimpleExample() + df1 = simple_example.customers_df['Customer Name'] + result_rs = group_similar_strings(df1, min_similarity=0.6, ignore_index=True, + use_sp_matmul_rs=True) + result_legacy = group_similar_strings(df1, min_similarity=0.6, ignore_index=True, + use_sp_matmul_rs=False) + pd.testing.assert_series_equal(result_legacy, result_rs) + # sanity-check against the known expected grouping + pd.testing.assert_series_equal(simple_example.expected_result_centroid, result_rs) + + def test_zero_min_similarity(self): + """zero-similarity matches should be included by both backends when min_similarity <= 0""" + simple_example = SimpleExample() + s_master = simple_example.customers_df['Customer Name'] + s_dup = simple_example.whatever_series_1 + matches_rs = match_strings(s_master, s_dup, min_similarity=0, use_sp_matmul_rs=True) + matches_legacy = match_strings(s_master, s_dup, min_similarity=0, use_sp_matmul_rs=False) + pd.testing.assert_frame_equal(matches_legacy, matches_rs) + pd.testing.assert_frame_equal(simple_example.expected_result_with_zeroes, matches_rs) + + if __name__ == '__main__': unittest.main() From e4acd37e455ef8942c6d6e241b589e7f325a3f32 Mon Sep 17 00:00:00 2001 From: Chris van den Berg <11998981+bergvca@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:06:38 +0200 Subject: [PATCH 5/9] updated README.md to reflect changes --- README.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3ea6544e..acb839ac 100644 --- a/README.md +++ b/README.md @@ -36,24 +36,24 @@ within a single list or between two lists of strings. The full process is descri ## Speed -**`string_grouper`** leverages the blazingly fast [sparse_dot_topn](https://github.com/ing-bank/sparse_dot_topn) libary +**`string_grouper`** leverages the blazingly fast [sp_matmul_rs](https://github.com/Bergvca/sp_matmul_rs) (originally based on: [sparse_dot_topn](https://github.com/ing-bank/sparse_dot_topn)) to calculate cosine similarities. ```python s = datetime.datetime.now() -matches = match_strings(names['Company Name'], number_of_processes = 4) - +matches = match_strings(names["name"], number_of_processes=15) e = datetime.datetime.now() -diff = (e - s) -str(diff) + +diff = e - s +print(diff) ``` Results in: -`00:05:34.65` On an Intel i7-6500U CPU @ 2.50GHz, where `len(names)` = 663 000 +`00:17.80` On an m5 pro, where `len(names)` = 663 000 *in other words*, -the library is able to perform fuzzy matching of 663 000 names in _five and a half minutes_ -on a 2015 consumer CPU using 4 cores. +the library is able to perform fuzzy matching of 663 000 names in _less then 18 seconds_ +on a 2026 consumer CPU using 15 cores. ## Simple Match @@ -100,3 +100,10 @@ companies.groupby('name_deduped')['Line Number'].count().sort_values(ascending=F ## Documentation The documentation can be found [here](https://bergvca.github.io/string_grouper/) + +## Backends + +The library was originally developed using the [sparse_dot_topn](https://github.com/ing-bank/sparse_dot_topn) library, +but has since been rewritten to use the [sp_matmul_rs](https://github.com/Bergvca/sp_matmul_rs) library, which is a Rust +implementation of the sparse matrix multiplication algorithm optimized with _Claude Fable_. To run the library with the +original sparse_dot_topn backend, set `use_sp_matmul_rs` to `False`. \ No newline at end of file From f18963ac103053f14084e0325a6b8dad7c395294 Mon Sep 17 00:00:00 2001 From: Chris van den Berg <11998981+bergvca@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:56:58 +0200 Subject: [PATCH 6/9] bumped sp_matmul_rs version to 0.0.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d17a3a0e..5147debe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "numpy>=2.0", "sparse-dot-topn>=1.1.0", "loguru>0.7.0", - "sp_matmul_rs>=0.0.1" + "sp_matmul_rs>=0.0.2" ] [build-system] From 3d2bfc222a2158e33d47649f17032f672893f02e Mon Sep 17 00:00:00 2001 From: Chris van den Berg <11998981+bergvca@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:56:58 +0200 Subject: [PATCH 7/9] bumped sp_matmul_rs version to 0.2.1 and added relevant code and documentation. --- CHANGELOG.md | 26 ++++++++++++++++++++++ pyproject.toml | 2 +- string_grouper/string_grouper.py | 21 +++++++++++++++-- string_grouper/test/test_string_grouper.py | 19 ++++++++++++++++ 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40640660..1469c79d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,32 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.0] - Unreleased + +### Added + +* New [sp_matmul_rs](https://github.com/Bergvca/sp_matmul_rs) backend for the cosine-similarity calculation — a Rust + reimplementation of the sparse top-n matrix multiplication. It performs the block/chunk splitting internally and + adds further optimizations. This is now the **default** backend. +* New `use_sp_matmul_rs` configuration keyword (default `True`). Set it to `False` to fall back to the original + `sparse_dot_topn` backend. This is slower but battle tested and thus more stable. +* New dependency on `sp_matmul_rs>=0.2.0`. +* Added a test suite verifying that the `sp_matmul_rs` and `sparse_dot_topn` backends produce equivalent results. +* New `chunk_cols` configuration keyword (default `None`) — the `sp_matmul_rs` counterpart to `n_blocks`. It sets the + column-chunk width of the backend's cache-blocked kernel. `chunk_cols` can be used to tune performance for matrices that +are denser than the matrices normally expected in the string-matching use case. Only valid when `use_sp_matmul_rs=True`; +`None` lets the backend derive the width from the detected L1d cache size. + +### Changed + +* Cosine similarities are now computed with `sp_matmul_rs` by default, yielding a large speed-up (e.g. fuzzy matching + of 663 000 names in under 18 seconds on a m5 pro using 15 cores). +* When `use_sp_matmul_rs=True`, block splitting is handled internally by the backend; the automatic `n_blocks` + guesstimate and `OverflowError` fallback are only used with the `sparse_dot_topn` backend. +* Setting `n_blocks` explicitly while `use_sp_matmul_rs=True` now raises an exception, since blocking is calculated + automatically by `sp_matmul_rs`. + + ## [0.7.2] - 2026-05-22 ### Changed diff --git a/pyproject.toml b/pyproject.toml index 5147debe..a7c096c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "numpy>=2.0", "sparse-dot-topn>=1.1.0", "loguru>0.7.0", - "sp_matmul_rs>=0.0.2" + "sp_matmul_rs>=0.2.1" ] [build-system] diff --git a/string_grouper/string_grouper.py b/string_grouper/string_grouper.py index 9a544558..7c00c212 100644 --- a/string_grouper/string_grouper.py +++ b/string_grouper/string_grouper.py @@ -35,6 +35,7 @@ # to account for symmetry thus compensating for those numerical errors that violate symmetry due to loss of # significance DEFAULT_N_BLOCKS: Optional[Tuple[int, int]] = None # Option value to use to split dataset(s) into roughly equal-sized blocks +DEFAULT_CHUNK_COLS: Optional[int] = None # sp_matmul_rs cache-tile width; None auto-derives from the L1d cache size DEFAULT_NORMALIZE_TO_ASCII: bool = True # The following string constants are used by (but aren't [yet] options passed to) StringGrouper @@ -187,7 +188,12 @@ class StringGrouperConfig(NamedTuple): :param n_blocks: (int, int) This parameter is provided to help boost performance, if possible, of processing large DataFrames, by splitting the DataFrames into n_blocks[0] blocks for the left operand (of the underlying matrix multiplication) and into n_blocks[1] blocks for the right operand - before performing the string-comparisons block-wise. Defaults to None. + before performing the string-comparisons block-wise. Only applies to the sparse_dot_topn backend + (use_sp_matmul_rs=False). Defaults to None. + :param chunk_cols: int. The sp_matmul_rs counterpart to n_blocks: the column-chunk width of the + cache-blocked kernel. This is a performance knob only; any value yields identical results. Only + applies to the sp_matmul_rs backend (use_sp_matmul_rs=True). Defaults to None, which lets + sp_matmul_rs derive the width from the detected L1d cache size. """ ngram_size: int = DEFAULT_NGRAM_SIZE @@ -204,6 +210,7 @@ class StringGrouperConfig(NamedTuple): group_rep: str = DEFAULT_GROUP_REP force_symmetries: bool = DEFAULT_FORCE_SYMMETRIES n_blocks: Tuple[int, int] = DEFAULT_N_BLOCKS + chunk_cols: Optional[int] = DEFAULT_CHUNK_COLS normalize_to_ascii: bool = DEFAULT_NORMALIZE_TO_ASCII def validate_is_fit(f): @@ -305,6 +312,7 @@ def _set_options(self, **kwargs): self._validate_tfidf_matrix_dtype() self._validate_replace_na_and_drop() self._validate_n_blocks() + self._validate_chunk_cols() self.is_build = False def _build_corpus(self): @@ -779,7 +787,8 @@ def _build_matches_rs(self, top_n = self._max_n_matches, threshold = self._config.min_similarity, sort = True, - n_threads = self._config.number_of_processes + n_threads = self._config.number_of_processes, + chunk_cols = self._config.chunk_cols ) def _get_matches_list(self, @@ -982,6 +991,14 @@ def _validate_n_blocks(self): if (self._config.n_blocks[0] < 1) or (self._config.n_blocks[1] < 1): raise Exception(errmsg) + def _validate_chunk_cols(self): + if self._config.chunk_cols is None: + return + if not self._config.use_sp_matmul_rs: + raise Exception("chunk_cols only applies when use_sp_matmul_rs is True.") + if not isinstance(self._config.chunk_cols, int) or self._config.chunk_cols < 1: + raise Exception("Invalid option value for parameter chunk_cols: chunk_cols must be None or an integer greater than 0.") + @staticmethod def _fix_diagonal(m: lil_matrix) -> lil_matrix: r = np.arange(m.shape[0]) diff --git a/string_grouper/test/test_string_grouper.py b/string_grouper/test/test_string_grouper.py index d5c7d685..93aead76 100644 --- a/string_grouper/test/test_string_grouper.py +++ b/string_grouper/test/test_string_grouper.py @@ -1126,6 +1126,25 @@ def test_zero_min_similarity(self): pd.testing.assert_frame_equal(matches_legacy, matches_rs) pd.testing.assert_frame_equal(simple_example.expected_result_with_zeroes, matches_rs) + def test_chunk_cols_result_invariant(self): + """chunk_cols is an sp_matmul_rs performance knob only: any value must yield identical results""" + df1 = SimpleExample().customers_df2['Customer Name'] + base = self.fix_row_order(match_strings(df1, min_similarity=0.1)) + for chunk_cols in (1, 7, 64, 100000): + tuned = self.fix_row_order(match_strings(df1, min_similarity=0.1, chunk_cols=chunk_cols)) + pd.testing.assert_frame_equal(base, tuned) + + def test_chunk_cols_validation(self): + """chunk_cols must be None or a positive int, and only valid with the sp_matmul_rs backend""" + df1 = SimpleExample().customers_df2['Customer Name'] + # chunk_cols applies only to the sp_matmul_rs backend + with self.assertRaises(Exception): + match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=False, n_blocks=(1, 1), chunk_cols=64) + # chunk_cols must be a positive integer + for bad in (0, -1, 2.5, 'x'): + with self.assertRaises(Exception): + match_strings(df1, min_similarity=0.1, chunk_cols=bad) + if __name__ == '__main__': unittest.main() From a03bd621f4cb70c13263d23f5cdf9cbf865bb586 Mon Sep 17 00:00:00 2001 From: Chris van den Berg <11998981+bergvca@users.noreply.github.com> Date: Mon, 20 Jul 2026 21:06:17 +0200 Subject: [PATCH 8/9] Some changes based on code review --- CHANGELOG.md | 22 +++- README.md | 4 +- docs/performance.md | 5 + docs/references/options_kwargs.md | 4 +- string_grouper/string_grouper.py | 104 ++++++++++----- string_grouper/test/test_string_grouper.py | 142 ++++++++++++++------- 6 files changed, 192 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1469c79d..20d97305 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,21 +14,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 adds further optimizations. This is now the **default** backend. * New `use_sp_matmul_rs` configuration keyword (default `True`). Set it to `False` to fall back to the original `sparse_dot_topn` backend. This is slower but battle tested and thus more stable. -* New dependency on `sp_matmul_rs>=0.2.0`. +* New dependency on `sp_matmul_rs>=0.2.1`. * Added a test suite verifying that the `sp_matmul_rs` and `sparse_dot_topn` backends produce equivalent results. * New `chunk_cols` configuration keyword (default `None`) — the `sp_matmul_rs` counterpart to `n_blocks`. It sets the column-chunk width of the backend's cache-blocked kernel. `chunk_cols` can be used to tune performance for matrices that -are denser than the matrices normally expected in the string-matching use case. Only valid when `use_sp_matmul_rs=True`; -`None` lets the backend derive the width from the detected L1d cache size. +are denser than the matrices normally expected in the string-matching use case. Only used when `use_sp_matmul_rs=True` +(ignored, with a warning, by the `sparse_dot_topn` backend); `None` lets the backend derive the width from the detected +L1d cache size. ### Changed * Cosine similarities are now computed with `sp_matmul_rs` by default, yielding a large speed-up (e.g. fuzzy matching of 663 000 names in under 18 seconds on a m5 pro using 15 cores). * When `use_sp_matmul_rs=True`, block splitting is handled internally by the backend; the automatic `n_blocks` - guesstimate and `OverflowError` fallback are only used with the `sparse_dot_topn` backend. -* Setting `n_blocks` explicitly while `use_sp_matmul_rs=True` now raises an exception, since blocking is calculated - automatically by `sp_matmul_rs`. + guesstimate and `OverflowError` fallback are only used with the `sparse_dot_topn` backend. Should the + `sp_matmul_rs` backend overflow its 32-bit result indices (`OverflowError`), `fit()` transparently retries it + with 64-bit indices (`idx_dtype=np.int64`), staying on the fast backend. Only if that retry still fails, or on + a `MemoryError`, does `fit()` fall back to the `sparse_dot_topn` backend with automatic block splitting. +* Setting `n_blocks` explicitly while `use_sp_matmul_rs=True` logs a warning and ignores `n_blocks`, since blocking is + calculated automatically by `sp_matmul_rs`. Existing code that tunes `n_blocks` keeps working; set + `use_sp_matmul_rs=False` to make `n_blocks` effective again. + +### Fixed + +* `n_blocks` passed through the instance-method variants (e.g. `StringGrouper.match_strings`) is now honored; previously + a stale value captured at construction time (or a prior fit's automatic guess) was silently used instead. ## [0.7.2] - 2026-05-22 diff --git a/README.md b/README.md index acb839ac..392acb6c 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Results in: `00:17.80` On an m5 pro, where `len(names)` = 663 000 *in other words*, -the library is able to perform fuzzy matching of 663 000 names in _less then 18 seconds_ +the library is able to perform fuzzy matching of 663 000 names in _less than 18 seconds_ on a 2026 consumer CPU using 15 cores. ## Simple Match @@ -106,4 +106,4 @@ The documentation can be found [here](https://bergvca.github.io/string_grouper/) The library was originally developed using the [sparse_dot_topn](https://github.com/ing-bank/sparse_dot_topn) library, but has since been rewritten to use the [sp_matmul_rs](https://github.com/Bergvca/sp_matmul_rs) library, which is a Rust implementation of the sparse matrix multiplication algorithm optimized with _Claude Fable_. To run the library with the -original sparse_dot_topn backend, set `use_sp_matmul_rs` to `False`. \ No newline at end of file +original sparse_dot_topn backend, set `use_sp_matmul_rs` to `False`. diff --git a/docs/performance.md b/docs/performance.md index 0e03a46f..5f60ee6e 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -1,5 +1,10 @@ ## Performance +> **Note:** this page describes the `n_blocks` block-splitting mechanism of the legacy +> `sparse_dot_topn` backend, which is used only when `use_sp_matmul_rs=False`. The default +> `sp_matmul_rs` backend (since version 0.8.0) performs block/chunk splitting internally and +> ignores `n_blocks` (with a warning); its performance can instead be tuned with the +> `chunk_cols` option. Semilogx plots of run-times of `match_strings()` vs the number of blocks (`n_blocks[1]`) into which the right matrix-operand of the dataset (663 000 strings from sec__edgar_company_info.csv) was split before performing the string comparison. As shown in the legend, each plot corresponds to the number `n_blocks[0]` of blocks into which the left matrix-operand was split. ![Semilogx](https://raw.githubusercontent.com/Bergvca/string_grouper/master/images/BlockNumberSpaceExploration1.png) diff --git a/docs/references/options_kwargs.md b/docs/references/options_kwargs.md index 4dfe0ebc..68ed985b 100644 --- a/docs/references/options_kwargs.md +++ b/docs/references/options_kwargs.md @@ -24,7 +24,9 @@ All keyword arguments not mentioned in the function definitions above are used t * **`number_of_processes`**: The number of processes used by the cosine similarity calculation. Defaults to `number of cores on a machine - 1.` -* **`n_blocks`**: This parameter is a tuple of two `int`s provided to help boost performance, if possible, of processing large DataFrames (see [Subsection Performance](#perf)), by splitting the DataFrames into `n_blocks[0]` blocks for the left operand (of the underlying matrix multiplication) and into `n_blocks[1]` blocks for the right operand before performing the string-comparisons block-wise. Defaults to `None`, in which case automatic splitting occurs if an `OverflowError` would otherwise occur. +* **`use_sp_matmul_rs`**: Selects the backend used for the cosine-similarity matrix multiplication. If `True` (the default), the Rust [sp_matmul_rs](https://github.com/Bergvca/sp_matmul_rs) library is used, which performs block/chunk splitting internally. Set it to `False` to fall back to the original [sparse_dot_topn](https://github.com/ing-bank/sparse_dot_topn) backend (slower, but battle-tested). +* **`n_blocks`**: This parameter is a tuple of two `int`s provided to help boost performance, if possible, of processing large DataFrames (see [Subsection Performance](#perf)), by splitting the DataFrames into `n_blocks[0]` blocks for the left operand (of the underlying matrix multiplication) and into `n_blocks[1]` blocks for the right operand before performing the string-comparisons block-wise. Defaults to `None`, in which case automatic splitting occurs if an `OverflowError` would otherwise occur. Only used by the `sparse_dot_topn` backend (`use_sp_matmul_rs=False`); the default `sp_matmul_rs` backend splits the data internally and ignores `n_blocks` with a warning. +* **`chunk_cols`**: The `sp_matmul_rs` counterpart to `n_blocks`: an `int` setting the column-chunk width of the backend's cache-blocked kernel. This is a performance knob only — any value yields identical results. Defaults to `None`, which lets `sp_matmul_rs` derive the width from the detected L1d cache size. Only used by the `sp_matmul_rs` backend (`use_sp_matmul_rs=True`); the `sparse_dot_topn` backend ignores it with a warning. ## Other settings diff --git a/string_grouper/string_grouper.py b/string_grouper/string_grouper.py index 7c00c212..7381e9a4 100644 --- a/string_grouper/string_grouper.py +++ b/string_grouper/string_grouper.py @@ -4,7 +4,6 @@ import multiprocessing from sklearn.feature_extraction.text import TfidfVectorizer from scipy.sparse import vstack, csr_matrix -from scipy.sparse import csr_matrix from scipy.sparse import lil_matrix from scipy.sparse.csgraph import connected_components from typing import Tuple, NamedTuple, List, Optional, Union @@ -188,12 +187,13 @@ class StringGrouperConfig(NamedTuple): :param n_blocks: (int, int) This parameter is provided to help boost performance, if possible, of processing large DataFrames, by splitting the DataFrames into n_blocks[0] blocks for the left operand (of the underlying matrix multiplication) and into n_blocks[1] blocks for the right operand - before performing the string-comparisons block-wise. Only applies to the sparse_dot_topn backend - (use_sp_matmul_rs=False). Defaults to None. + before performing the string-comparisons block-wise. Only used by the sparse_dot_topn backend + (use_sp_matmul_rs=False); ignored, with a warning, when use_sp_matmul_rs=True. Defaults to None. :param chunk_cols: int. The sp_matmul_rs counterpart to n_blocks: the column-chunk width of the cache-blocked kernel. This is a performance knob only; any value yields identical results. Only - applies to the sp_matmul_rs backend (use_sp_matmul_rs=True). Defaults to None, which lets - sp_matmul_rs derive the width from the detected L1d cache size. + used by the sp_matmul_rs backend (use_sp_matmul_rs=True); ignored, with a warning, when + use_sp_matmul_rs=False. Defaults to None, which lets sp_matmul_rs derive the width from the + detected L1d cache size. """ ngram_size: int = DEFAULT_NGRAM_SIZE @@ -271,8 +271,6 @@ def __init__(self, master: pd.Series, self._config: StringGrouperConfig = StringGrouperConfig(**kwargs) - self._n_blocks = self._config.n_blocks - # initialize the members: self._set_data(master, duplicates, master_id, duplicates_id) self._set_options(**kwargs) @@ -398,7 +396,7 @@ def fit(self): master_matrix, duplicate_matrix = self._get_tf_idf_matrices() if self._config.use_sp_matmul_rs: - matches = self._build_matches_rs(master_matrix, duplicate_matrix) + matches = self._build_matches_rs_with_recovery(master_matrix, duplicate_matrix) else: matches = self._calc_blocks_and_build_matches(master_matrix, duplicate_matrix) @@ -427,33 +425,28 @@ def _calc_blocks_and_build_matches(self, master_matrix: csr_matrix, duplicate_ma b_right = max(1, round(len(self._right_Series) / 4e3)) # based on tests and observations size_guess_block = (b_left, b_right) # inversion of left and right series was introduced in 0.6 ? - if self._n_blocks is None: + n_blocks = self._config.n_blocks + if n_blocks is None: if size_guess_block != (1, 1): logger.info( "n_blocks parameter is not set so data will be split into smaller chunks, n_blocks = (" + str(size_guess_block[0]) + "," + str(size_guess_block[1]) + ")") - self._n_blocks = size_guess_block + n_blocks = size_guess_block # do the matching - if self._n_blocks == (1, 1): + if n_blocks == (1, 1): try: - matches = self._build_matches(master_matrix, duplicate_matrix, self._n_blocks) + matches = self._build_matches(master_matrix, duplicate_matrix, n_blocks) except OverflowError: logger.warning( - "An OverflowError occurred but is being " + - "handled. The input data will be automatically " + - "split-up into smaller chunks which will then be " + - "processed one chunk at a time. To prevent " + - "OverflowError, use the n_blocks parameter to split-up " + - "the data manually into small enough chunks" + - ", n_blocks = (" + - str(size_guess_block[0]), - ",", - str(size_guess_block[1]) + ")" - ) + "An OverflowError occurred but is being handled. The input data will be " + "automatically split-up into smaller chunks which will then be processed one " + "chunk at a time. To prevent OverflowError, use the n_blocks parameter to " + "split-up the data manually into small enough chunks, " + f"n_blocks = ({size_guess_block[0]},{size_guess_block[1]})") matches = self._build_matches(master_matrix, duplicate_matrix, size_guess_block) else: - matches = self._build_matches(master_matrix, duplicate_matrix, self._n_blocks) + matches = self._build_matches(master_matrix, duplicate_matrix, n_blocks) return matches def dot(self) -> pd.Series: @@ -735,7 +728,7 @@ def _fit_vectorizer(self) -> TfidfVectorizer: def _build_matches(self, master_matrix: csr_matrix, duplicate_matrix: csr_matrix, n_blocks: Tuple[int, int]) -> csr_matrix: - """Builds the cossine similarity matrix of two csr matrices""" + """Builds the cosine similarity matrix of two csr matrices""" def define_chunks(length_to_split, n_chunks): @@ -779,8 +772,13 @@ def chunk_list(lst, n): def _build_matches_rs(self, master_matrix: csr_matrix, - duplicate_matrix: csr_matrix) -> csr_matrix: - """Builds the cossine similarity matrix of two csr matrices using sp_matmul_topn_rs for faster computation""" + duplicate_matrix: csr_matrix, + idx_dtype=None) -> csr_matrix: + """Builds the cosine similarity matrix of two csr matrices using sp_matmul_topn_rs for faster computation. + + idx_dtype controls the integer width of the result's index arrays; None (the default) lets + sp_matmul_rs use 32-bit indices, and np.int64 is used to retry after a 32-bit index overflow. + """ return sp_matmul_topn_rs( master_matrix, duplicate_matrix.transpose(), @@ -788,9 +786,39 @@ def _build_matches_rs(self, threshold = self._config.min_similarity, sort = True, n_threads = self._config.number_of_processes, - chunk_cols = self._config.chunk_cols + chunk_cols = self._config.chunk_cols, + idx_dtype = idx_dtype ) + def _build_matches_rs_with_recovery(self, master_matrix: csr_matrix, duplicate_matrix: csr_matrix) -> csr_matrix: + """Runs the sp_matmul_rs backend, recovering from failures without leaving the fast path when possible. + + On an OverflowError (the result's 32-bit index arrays overflowed) the multiplication is retried + with 64-bit indices, which addresses larger result matrices while staying on the Rust backend. + A MemoryError, or a failure that persists with 64-bit indices, falls back to the blocked + sparse_dot_topn backend, whose automatic chunk-splitting keeps peak memory bounded. + """ + try: + return self._build_matches_rs(master_matrix, duplicate_matrix) + except OverflowError: + logger.warning( + "The sp_matmul_rs backend overflowed its 32-bit result indices; retrying with 64-bit " + "indices (idx_dtype=np.int64).") + try: + return self._build_matches_rs(master_matrix, duplicate_matrix, idx_dtype=np.int64) + except (OverflowError, MemoryError) as error: + logger.warning( + f"The sp_matmul_rs backend still failed with 64-bit indices ({error!r}); falling back to " + "the sparse_dot_topn backend with automatic block splitting. Set use_sp_matmul_rs=False " + "(optionally with the n_blocks parameter) to skip the failing backend on future runs.") + return self._calc_blocks_and_build_matches(master_matrix, duplicate_matrix) + except MemoryError as error: + logger.warning( + f"The sp_matmul_rs backend ran out of memory ({error!r}); falling back to the sparse_dot_topn " + "backend with automatic block splitting. Set use_sp_matmul_rs=False (optionally with the " + "n_blocks parameter) to skip the failing backend on future runs.") + return self._calc_blocks_and_build_matches(master_matrix, duplicate_matrix) + def _get_matches_list(self, matches: csr_matrix ) -> pd.DataFrame: @@ -976,12 +1004,10 @@ def _validate_replace_na_and_drop(self): ) def _validate_n_blocks(self): - errmsg = "Invalid option value for parameter n_blocks: " - "n_blocks must be None or a tuple of 2 integers greater than 0." + errmsg = ("Invalid option value for parameter n_blocks: " + "n_blocks must be None or a tuple of 2 integers greater than 0.") if self._config.n_blocks is None: return - if self._config.n_blocks is not None and self._config.use_sp_matmul_rs: - raise Exception("If sp_matmul_rs is True, n_blocks is cannot be set and is calculated automatically.") if not isinstance(self._config.n_blocks, tuple): raise Exception(errmsg) if len(self._config.n_blocks) != 2: @@ -990,14 +1016,22 @@ def _validate_n_blocks(self): raise Exception(errmsg) if (self._config.n_blocks[0] < 1) or (self._config.n_blocks[1] < 1): raise Exception(errmsg) + if self._config.use_sp_matmul_rs: + logger.warning( + "n_blocks is ignored when use_sp_matmul_rs=True: block splitting is handled internally by " + "sp_matmul_rs. Set use_sp_matmul_rs=False to use n_blocks with the sparse_dot_topn backend.") def _validate_chunk_cols(self): if self._config.chunk_cols is None: return + if (isinstance(self._config.chunk_cols, bool) + or not isinstance(self._config.chunk_cols, (int, np.integer)) + or self._config.chunk_cols < 1): + raise Exception("Invalid option value for parameter chunk_cols: " + "chunk_cols must be None or an integer greater than 0.") if not self._config.use_sp_matmul_rs: - raise Exception("chunk_cols only applies when use_sp_matmul_rs is True.") - if not isinstance(self._config.chunk_cols, int) or self._config.chunk_cols < 1: - raise Exception("Invalid option value for parameter chunk_cols: chunk_cols must be None or an integer greater than 0.") + logger.warning( + "chunk_cols is ignored when use_sp_matmul_rs=False: it only applies to the sp_matmul_rs backend.") @staticmethod def _fix_diagonal(m: lil_matrix) -> lil_matrix: diff --git a/string_grouper/test/test_string_grouper.py b/string_grouper/test/test_string_grouper.py index 93aead76..466f25d9 100644 --- a/string_grouper/test/test_string_grouper.py +++ b/string_grouper/test/test_string_grouper.py @@ -14,6 +14,11 @@ def mock_symmetrize_matrix(x: csr_matrix) -> csr_matrix: return x +def fix_row_order(df): + """Sorts match rows canonically so DataFrames can be compared independently of row order""" + return df.sort_values(['right_index', 'left_index']).reset_index(drop=True) + + class SimpleExample(object): def __init__(self): self.customers_df = pd.DataFrame( @@ -125,11 +130,6 @@ def test_auto_blocking_single_DataFrame(self): # OverflowThreshold. This will in turn trigger automatic splitting # of the Series/matrices into smaller blocks when n_blocks = None - sort_cols = ['right_index', 'left_index'] - - def fix_row_order(df): - return df.sort_values(sort_cols).reset_index(drop=True) - simple_example = SimpleExample() df1 = simple_example.customers_df2['Customer Name'] @@ -191,11 +191,6 @@ def do_test_with(OverflowThreshold): def test_n_blocks_single_DataFrame(self): """tests whether manual blocking yields consistent results""" - sort_cols = ['right_index', 'left_index'] - - def fix_row_order(df): - return df.sort_values(sort_cols).reset_index(drop=True) - simple_example = SimpleExample() df1 = simple_example.customers_df2['Customer Name'] @@ -285,11 +280,6 @@ def test_overflow_error_with(OverflowThreshold, n_blocks): def test_n_blocks_both_DataFrames(self): """tests whether manual blocking yields consistent results""" - sort_cols = ['right_index', 'left_index'] - - def fix_row_order(df): - return df.sort_values(sort_cols).reset_index(drop=True) - simple_example = SimpleExample() df1 = simple_example.customers_df['Customer Name'] df2 = simple_example.customers_df2['Customer Name'] @@ -337,19 +327,14 @@ def fix_row_order(df): pd.testing.assert_frame_equal(matches11, matches32) def test_n_blocks_bad_option_value(self): - """Tests that bad option values for n_blocks are caught""" + """Tests that bad option values for n_blocks are caught, regardless of the backend""" simple_example = SimpleExample() df1 = simple_example.customers_df2['Customer Name'] - with self.assertRaises(Exception): - _ = match_strings(df1, n_blocks=2) - with self.assertRaises(Exception): - _ = match_strings(df1, n_blocks=(0, 2)) - with self.assertRaises(Exception): - _ = match_strings(df1, n_blocks=(1, 2.5)) - with self.assertRaises(Exception): - _ = match_strings(df1, n_blocks=(1, 2, 3)) - with self.assertRaises(Exception): - _ = match_strings(df1, n_blocks=(1, )) + for use_rs in (True, False): + for bad_n_blocks in (2, (0, 2), (1, 2.5), (1, 2, 3), (1, )): + with self.assertRaises(Exception) as cm: + _ = match_strings(df1, n_blocks=bad_n_blocks, use_sp_matmul_rs=use_rs) + self.assertIn('tuple of 2 integers greater than 0', str(cm.exception)) def test_tfidf_dtype_bad_option_value(self): """Tests that bad option values for n_blocks are caught""" @@ -1049,18 +1034,13 @@ class SpMatmulRsEquivalenceTest(unittest.TestCase): """Tests that the sp_matmul_rs backend (use_sp_matmul_rs=True) yields the same results as the legacy sparse_dot_topn backend (use_sp_matmul_rs=False)""" - sort_cols = ['right_index', 'left_index'] - - def fix_row_order(self, df): - return df.sort_values(self.sort_cols).reset_index(drop=True) - def test_match_strings_single_series(self): """match_strings on a single Series (self-join) should be backend-independent""" simple_example = SimpleExample() df1 = simple_example.customers_df2['Customer Name'] - matches_rs = self.fix_row_order( + matches_rs = fix_row_order( match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=True)) - matches_legacy = self.fix_row_order( + matches_legacy = fix_row_order( match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches_legacy, matches_rs) @@ -1069,23 +1049,23 @@ def test_match_strings_two_series(self): simple_example = SimpleExample() df1 = simple_example.customers_df['Customer Name'] df2 = simple_example.customers_df2['Customer Name'] - matches_rs = self.fix_row_order( + matches_rs = fix_row_order( match_strings(df1, df2, min_similarity=0.1, use_sp_matmul_rs=True)) - matches_legacy = self.fix_row_order( + matches_legacy = fix_row_order( match_strings(df1, df2, min_similarity=0.1, use_sp_matmul_rs=False)) pd.testing.assert_frame_equal(matches_legacy, matches_rs) def test_match_strings_with_ids(self): """match_strings with master_id and duplicates_id should be backend-independent""" simple_example = SimpleExample() - matches_rs = self.fix_row_order( + matches_rs = fix_row_order( match_strings(simple_example.customers_df['Customer Name'], simple_example.customers_df2['Customer Name'], master_id=simple_example.customers_df['Customer ID'], duplicates_id=simple_example.customers_df2['Customer ID'], min_similarity=0.1, use_sp_matmul_rs=True)) - matches_legacy = self.fix_row_order( + matches_legacy = fix_row_order( match_strings(simple_example.customers_df['Customer Name'], simple_example.customers_df2['Customer Name'], master_id=simple_example.customers_df['Customer ID'], @@ -1129,21 +1109,93 @@ def test_zero_min_similarity(self): def test_chunk_cols_result_invariant(self): """chunk_cols is an sp_matmul_rs performance knob only: any value must yield identical results""" df1 = SimpleExample().customers_df2['Customer Name'] - base = self.fix_row_order(match_strings(df1, min_similarity=0.1)) + base = fix_row_order(match_strings(df1, min_similarity=0.1)) for chunk_cols in (1, 7, 64, 100000): - tuned = self.fix_row_order(match_strings(df1, min_similarity=0.1, chunk_cols=chunk_cols)) + tuned = fix_row_order(match_strings(df1, min_similarity=0.1, chunk_cols=chunk_cols)) pd.testing.assert_frame_equal(base, tuned) def test_chunk_cols_validation(self): - """chunk_cols must be None or a positive int, and only valid with the sp_matmul_rs backend""" + """chunk_cols must be None or a positive int; the legacy backend ignores it with a warning""" df1 = SimpleExample().customers_df2['Customer Name'] - # chunk_cols applies only to the sp_matmul_rs backend - with self.assertRaises(Exception): - match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=False, n_blocks=(1, 1), chunk_cols=64) - # chunk_cols must be a positive integer - for bad in (0, -1, 2.5, 'x'): + # chunk_cols is ignored (with a warning) by the sparse_dot_topn backend + base = fix_row_order( + match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=False, n_blocks=(1, 1))) + ignored = fix_row_order( + match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=False, n_blocks=(1, 1), chunk_cols=64)) + pd.testing.assert_frame_equal(base, ignored) + # chunk_cols must be a positive integer (bools are not integers here) + for bad in (0, -1, 2.5, 'x', True): with self.assertRaises(Exception): match_strings(df1, min_similarity=0.1, chunk_cols=bad) + # numpy integers are valid + rs_base = fix_row_order(match_strings(df1, min_similarity=0.1)) + np_tuned = fix_row_order(match_strings(df1, min_similarity=0.1, chunk_cols=np.int64(64))) + pd.testing.assert_frame_equal(rs_base, np_tuned) + + def test_rs_backend_retries_with_64bit_indices_on_overflow(self): + """An OverflowError (32-bit index overflow) must be recovered by retrying sp_matmul_rs + with idx_dtype=np.int64, staying on the fast backend rather than falling back""" + df1 = SimpleExample().customers_df2['Customer Name'] + expected = fix_row_order(match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=False)) + + original_build = StringGrouper._build_matches_rs + idx_dtypes = [] + + def flaky(self, master_matrix, duplicate_matrix, idx_dtype=None): + idx_dtypes.append(idx_dtype) + if len(idx_dtypes) == 1: # first (32-bit) attempt overflows + raise OverflowError + return original_build(self, master_matrix, duplicate_matrix, idx_dtype=idx_dtype) + + with patch.object(StringGrouper, '_build_matches_rs', autospec=True, side_effect=flaky): + result = fix_row_order(match_strings(df1, min_similarity=0.1)) + pd.testing.assert_frame_equal(expected, result) + # the first attempt used default (32-bit) indices, the retry used 64-bit indices + self.assertEqual(idx_dtypes, [None, np.int64]) + + def test_rs_backend_falls_back_when_recovery_exhausted(self): + """fit() must fall back to the blocked sparse_dot_topn path when sp_matmul_rs keeps failing + (OverflowError even at 64-bit) or runs out of memory, yielding the same results""" + df1 = SimpleExample().customers_df2['Customer Name'] + expected = fix_row_order(match_strings(df1, min_similarity=0.1, use_sp_matmul_rs=False)) + for error in (OverflowError, MemoryError): + with patch.object(StringGrouper, '_build_matches_rs', side_effect=error): + fallback = fix_row_order(match_strings(df1, min_similarity=0.1)) + pd.testing.assert_frame_equal(expected, fallback) + + def test_n_blocks_ignored_by_rs_backend(self): + """n_blocks is ignored (with a warning) by the sp_matmul_rs backend instead of raising, + so pre-0.8 code that tunes n_blocks keeps working under the new default backend""" + df1 = SimpleExample().customers_df2['Customer Name'] + base = fix_row_order(match_strings(df1, min_similarity=0.1)) + with_n_blocks = fix_row_order(match_strings(df1, min_similarity=0.1, n_blocks=(1, 2))) + pd.testing.assert_frame_equal(base, with_n_blocks) + # malformed n_blocks values are still rejected, regardless of backend + with self.assertRaises(Exception): + match_strings(df1, min_similarity=0.1, n_blocks=(0, 2)) + + def test_backend_switch_on_reused_instance(self): + """switching backends via the instance methods must not trip validators on options + carried over from earlier calls (n_blocks / chunk_cols of the other backend)""" + df1 = SimpleExample().customers_df2['Customer Name'] + sg = StringGrouper(df1, min_similarity=0.1) + legacy = fix_row_order(sg.match_strings(df1, use_sp_matmul_rs=False, n_blocks=(1, 2))) + # stale n_blocks from the previous call must not raise under the rs backend + rs = fix_row_order(sg.match_strings(df1, use_sp_matmul_rs=True)) + pd.testing.assert_frame_equal(legacy, rs) + # and stale chunk_cols must not raise when switching back to the legacy backend + sg2 = StringGrouper(df1, min_similarity=0.1, chunk_cols=64) + back_to_legacy = fix_row_order(sg2.match_strings(df1, use_sp_matmul_rs=False)) + pd.testing.assert_frame_equal(legacy, back_to_legacy) + + def test_n_blocks_honored_via_instance_methods(self): + """n_blocks passed through an instance method must reach the legacy matmul, not a stale + value captured at construction time""" + df1 = SimpleExample().customers_df2['Customer Name'] + sg = StringGrouper(df1, min_similarity=0.1, use_sp_matmul_rs=False) + with patch.object(sg, '_build_matches', wraps=sg._build_matches) as spy: + sg.match_strings(df1, use_sp_matmul_rs=False, n_blocks=(1, 2)) + self.assertEqual(spy.call_args.args[2], (1, 2)) if __name__ == '__main__': From ff2aae89e5008f23b141dfbde56306bd4f4d4901 Mon Sep 17 00:00:00 2001 From: Chris van den Berg <11998981+bergvca@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:15:21 +0200 Subject: [PATCH 9/9] Added note on latest version release status --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 392acb6c..f185b046 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,9 @@ Results in: the library is able to perform fuzzy matching of 663 000 names in _less than 18 seconds_ on a 2026 consumer CPU using 15 cores. +**The latest version (0.8.0) with a significant speed up is not released on pypi yet.** Use this repository to install +if you want to use the latest and greatest. + ## Simple Match ```python