diff --git a/cpp/src/gandiva/precompiled/string_ops.cc b/cpp/src/gandiva/precompiled/string_ops.cc index b7c6b518b45..d31e8002758 100644 --- a/cpp/src/gandiva/precompiled/string_ops.cc +++ b/cpp/src/gandiva/precompiled/string_ops.cc @@ -3099,6 +3099,19 @@ static char mappings[] = {'0', '1', '2', '3', '0', '1', '2', '0', '0', // 4. If the string have too few letters in the word that you can't assign three // numbers, append with zeros until there are three numbers. If you have four or more // numbers, retain only the first three. +// +// The mappings table below is defined only for the 26 ASCII letters, so the +// classification and case-folding used to index it must stay ASCII-only. The +// locale-sensitive isalpha/toupper let bytes 0x80-0xFF count as letters under a +// non-C locale, and toupper on such a byte yields an index past mappings. +static FORCE_INLINE bool is_ascii_letter(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); +} + +static FORCE_INLINE char ascii_toupper(char c) { + return (c >= 'a' && c <= 'z') ? static_cast(c - ('a' - 'A')) : c; +} + FORCE_INLINE const char* soundex_utf8(gdv_int64 context, const char* in, gdv_int32 in_len, bool in_validity, bool* out_valid, int32_t* out_len) { @@ -3125,9 +3138,9 @@ const char* soundex_utf8(gdv_int64 context, const char* in, gdv_int32 in_len, int start_idx = 0; for (int i = 0; i < in_len; ++i) { - if (isalpha(static_cast(in[i])) > 0) { + if (is_ascii_letter(in[i])) { // Retain the first letter - ret[0] = toupper(static_cast(in[i])); + ret[0] = ascii_toupper(in[i]); start_idx = i + 1; break; } @@ -3143,8 +3156,8 @@ const char* soundex_utf8(gdv_int64 context, const char* in, gdv_int32 in_len, soundex[0] = '\0'; // Replace consonants with digits and special letters with 0 for (int i = start_idx; i < in_len; i++) { - if (isalpha(static_cast(in[i])) > 0) { - c = toupper(static_cast(in[i])) - 65; + if (is_ascii_letter(in[i])) { + c = ascii_toupper(in[i]) - 'A'; if (mappings[c] != soundex[si - 1]) { soundex[si] = mappings[c]; si++; diff --git a/cpp/src/gandiva/precompiled/string_ops_test.cc b/cpp/src/gandiva/precompiled/string_ops_test.cc index 3faeb316231..695b0200447 100644 --- a/cpp/src/gandiva/precompiled/string_ops_test.cc +++ b/cpp/src/gandiva/precompiled/string_ops_test.cc @@ -18,9 +18,12 @@ #include #include +#include +#include #include #include #include +#include #include "gandiva/execution_context.h" #include "gandiva/precompiled/types.h" @@ -3046,6 +3049,51 @@ TEST(TestStringOps, TestSoundex) { EXPECT_EQ(validity, true); } +TEST(TestStringOps, TestSoundexNonAsciiNoOverread) { + gandiva::ExecutionContext ctx; + auto ctx_ptr = reinterpret_cast(&ctx); + int32_t out_len = 0; + bool validity = false; + const char* out; + + // ASCII input is unaffected by the fix. + out = soundex_utf8(ctx_ptr, "Robert", 6, true, &validity, &out_len); + EXPECT_EQ(std::string(out, out_len), "R163"); + + // The mappings table has 26 entries (A-Z). Under a non-C locale isalpha + // accepts bytes 0x80-0xFF and toupper leaves them past 'Z', which the old + // code used to index the table out of bounds. Find such a byte if the + // platform provides a locale for it, then confirm it is now treated as a + // separator instead of over-reading. + const char* candidates[] = {"en_US.UTF-8", "en_US.ISO8859-1", "C.UTF-8", + "de_DE.ISO8859-1", ""}; + std::string saved = setlocale(LC_CTYPE, nullptr); + int trigger = -1; + for (const char* loc : candidates) { + if (setlocale(LC_CTYPE, loc) == nullptr) { + continue; + } + for (int b = 0x80; b < 0x100; ++b) { + if (isalpha(b) && (toupper(b) - 'A' < 0 || toupper(b) - 'A' > 25)) { + trigger = b; + break; + } + } + if (trigger >= 0) { + break; + } + } + + if (trigger >= 0) { + char buf[] = {'R', 'o', 'b', static_cast(trigger), 'e', 'r', 't'}; + out = soundex_utf8(ctx_ptr, buf, static_cast(sizeof(buf)), true, &validity, + &out_len); + EXPECT_EQ(std::string(out, out_len), "R163"); + EXPECT_EQ(validity, true); + } + setlocale(LC_CTYPE, saved.c_str()); +} + TEST(TestStringOps, TestInstr) { std::string s1 = "hello world!"; auto s1_len = static_cast(s1.size());