From 2f8d931f3e551b924084bf8b75589df13ad9a497 Mon Sep 17 00:00:00 2001 From: Tom Windell Date: Mon, 22 Jun 2026 13:27:57 +0200 Subject: [PATCH 1/3] Change the canarytokens fetch method so that it hits the paginate endpoint --- canarytools/models/canarytokens.py | 36 +++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/canarytools/models/canarytokens.py b/canarytools/models/canarytokens.py index 5a67f90..1ff3b9d 100644 --- a/canarytools/models/canarytokens.py +++ b/canarytools/models/canarytokens.py @@ -6,6 +6,8 @@ import logging logger = logging.getLogger('canarytools') +CANARYTOKENS_FETCH_PAGE_SIZE_LIMIT = 5000 + class CanaryTokens(object): def __init__(self, console): """Initialize CanaryToken @@ -125,7 +127,7 @@ def get_token(self, canarytoken): return self.console.get('canarytoken/fetch', params, self.parse) def all(self, include_endpoints=True): - """Fetch all Canarytokens + """Fetch all Canarytokens using the paginate endpoint :return: A list of Canarytoken objects :rtype: List of :class:`CanaryToken ` objects @@ -137,8 +139,36 @@ def all(self, include_endpoints=True): >>> import canarytools >>> tokens = console.tokens.all() """ - params = {'include_endpoints': str(include_endpoints)} - return self.console.get('canarytokens/fetch', params, self.parse) + cursor = None + all_tokens = [] + + while True: + params = { + 'limit': CANARYTOKENS_FETCH_PAGE_SIZE_LIMIT, + 'render_transients': str(include_endpoints), + } + if cursor: + params['cursor'] = cursor + + page_data = self.console.get('canarytokens/paginate', params, parser=lambda data: data) + page_tokens = page_data.get('canarytokens', []) + + for token in page_tokens: + all_tokens.append(CanaryToken.parse(self.console, token)) + + if len(page_tokens) < CANARYTOKENS_FETCH_PAGE_SIZE_LIMIT: + break + + cursor_data = page_data.get('cursor') + if isinstance(cursor_data, dict): + cursor = cursor_data.get('next') + else: + cursor = cursor_data + + if not cursor: + break + + return all_tokens def parse(self, data): """Parse JSON data From 0771cc281da4ed8f24e7ea103bd2e76c50f60fe9 Mon Sep 17 00:00:00 2001 From: Tom Windell Date: Fri, 3 Jul 2026 15:27:56 +0200 Subject: [PATCH 2/3] Simplify code --- canarytools/models/canarytokens.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/canarytools/models/canarytokens.py b/canarytools/models/canarytokens.py index 1ff3b9d..04ebf21 100644 --- a/canarytools/models/canarytokens.py +++ b/canarytools/models/canarytokens.py @@ -6,7 +6,7 @@ import logging logger = logging.getLogger('canarytools') -CANARYTOKENS_FETCH_PAGE_SIZE_LIMIT = 5000 +CANARYTOKENS_PAGE_SIZE_LIMIT = 2500 class CanaryTokens(object): def __init__(self, console): @@ -127,7 +127,7 @@ def get_token(self, canarytoken): return self.console.get('canarytoken/fetch', params, self.parse) def all(self, include_endpoints=True): - """Fetch all Canarytokens using the paginate endpoint + """Fetch all Canarytokens in batches using the paginate endpoint :return: A list of Canarytoken objects :rtype: List of :class:`CanaryToken ` objects @@ -144,27 +144,21 @@ def all(self, include_endpoints=True): while True: params = { - 'limit': CANARYTOKENS_FETCH_PAGE_SIZE_LIMIT, + 'limit': CANARYTOKENS_PAGE_SIZE_LIMIT, 'render_transients': str(include_endpoints), } + if cursor: params['cursor'] = cursor - page_data = self.console.get('canarytokens/paginate', params, parser=lambda data: data) - page_tokens = page_data.get('canarytokens', []) - - for token in page_tokens: - all_tokens.append(CanaryToken.parse(self.console, token)) - - if len(page_tokens) < CANARYTOKENS_FETCH_PAGE_SIZE_LIMIT: - break + response = self.console.get('canarytokens/paginate', params, parser=dict) - cursor_data = page_data.get('cursor') - if isinstance(cursor_data, dict): - cursor = cursor_data.get('next') - else: - cursor = cursor_data + all_tokens.extend( + CanaryToken.parse(self.console, token) + for token in response.get('canarytokens', []) + ) + cursor = (response.get('cursor') or {}).get('next') if not cursor: break From 7cf1016643dac2c6e9420b5c87f2ff7eb991f4db Mon Sep 17 00:00:00 2001 From: Tom Windell Date: Mon, 6 Jul 2026 14:41:24 +0200 Subject: [PATCH 3/3] PR comments --- canarytools/models/canarytokens.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/canarytools/models/canarytokens.py b/canarytools/models/canarytokens.py index 04ebf21..4424e04 100644 --- a/canarytools/models/canarytokens.py +++ b/canarytools/models/canarytokens.py @@ -127,7 +127,7 @@ def get_token(self, canarytoken): return self.console.get('canarytoken/fetch', params, self.parse) def all(self, include_endpoints=True): - """Fetch all Canarytokens in batches using the paginate endpoint + """Fetch all Canarytokens :return: A list of Canarytoken objects :rtype: List of :class:`CanaryToken ` objects @@ -158,7 +158,7 @@ def all(self, include_endpoints=True): for token in response.get('canarytokens', []) ) - cursor = (response.get('cursor') or {}).get('next') + cursor = response.get('cursor', {}) if not cursor: break