diff --git a/src/google/adk/memory/in_memory_memory_service.py b/src/google/adk/memory/in_memory_memory_service.py index 94639964c3..d8b31291a6 100644 --- a/src/google/adk/memory/in_memory_memory_service.py +++ b/src/google/adk/memory/in_memory_memory_service.py @@ -32,6 +32,7 @@ _UNKNOWN_SESSION_ID = '__unknown_session_id__' _MAX_SEARCH_RESULTS = 10 +_SCRIPT_RUN_PATTERN = re.compile(r'[\x00-\x7f]+|[^\x00-\x7f]+') def _user_key(app_name: str, user_id: str) -> tuple[str, str]: @@ -43,6 +44,25 @@ def _extract_words_lower(text: str) -> set[str]: return set(word.lower() for word in re.findall(r'\w+', text)) +def _extract_searchable_words(text: str) -> set[str]: + """Extracts the words an event can be matched on, in lowercase. + + The tokens of _extract_words_lower, plus, for a token that mixes scripts, + each of its ASCII and non-ASCII runs. Japanese and Chinese are written + without spaces, so 私はPythonを使う is a single \\w+ token and a query for + Python matches nothing. Splitting on the boundary between scripts makes the + embedded Latin word a token of its own, while still keeping a partial word + such as thon from matching. + + Args: + text: The text of an event. + """ + words = _extract_words_lower(text) + for word in [word for word in words if not word.isascii()]: + words.update(_SCRIPT_RUN_PATTERN.findall(word)) + return words + + class InMemoryMemoryService(BaseMemoryService): """An in-memory memory service for prototyping purpose only. @@ -128,7 +148,7 @@ async def search_memory( event_text = ' '.join( [part.text for part in event.content.parts if part.text] ) - words_in_event = _extract_words_lower(event_text) + words_in_event = _extract_searchable_words(event_text) if not words_in_event: continue diff --git a/tests/unittests/memory/test_in_memory_memory_service.py b/tests/unittests/memory/test_in_memory_memory_service.py index 5524226b27..f3473fccaf 100644 --- a/tests/unittests/memory/test_in_memory_memory_service.py +++ b/tests/unittests/memory/test_in_memory_memory_service.py @@ -384,8 +384,15 @@ async def test_search_memory_does_not_collide_on_slash_in_identifiers(): # Mixed: non-Latin substring + Latin token in same event ('太郎 works at ABC Corp', '太郎', 1), ('太郎 works at ABC Corp', 'ABC', 1), + # Latin word inside an unspaced script (no space to tokenize on) + ('私はPythonでADKを使っています', 'Python', 1), + ('私はPythonでADKを使っています', 'adk', 1), + ('我用Python写代码', 'python', 1), + ('私はPythonでADKを使っています', '使って', 1), + ('私はPythonでADKを使っています', 'Java', 0), # Latin partial-word must NOT match (regression guard) ('I like to code in Python.', 'thon', 0), + ('私はPythonでADKを使っています', 'thon', 0), ], ) async def test_search_memory_non_latin(event_text, query, expected_count):