From 7bd5695ec7d45a9e094c3ea814e3b55670de05c5 Mon Sep 17 00:00:00 2001 From: "Graciliano M. P." Date: Sat, 1 Aug 2026 18:11:25 -0300 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20EntityPagination=20=E2=80=94=20lazi?= =?UTF-8?q?ly=20loaded=20paginated=20view=20over=20a=20select?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A handler that keeps the pages it has already loaded, over a select that does not know its total length upfront. - Pages are 1-based (matching the `page` parameter of the `select*` methods); entry indexes are 0-based (matching a Dart `List`). - Pages can be loaded out of order, leaving gaps: `getAt(45)` with a limit of 20 loads only page 3. - Synchronous access (`operator []`, `loadedEntities`) never fetches; only the `FutureOr` methods do. `operator []` returns null for a gap, an unloaded page or out of range alike. - Deliberately NOT a `List`/`Iterable`: both require a `length`, which is exactly what a paginated select can't answer until it reaches the end. A lying `length` would silently break every `for`/`map`/`toList`. Since every page but the last holds exactly `limit` entries, identifying the final page yields the total even with gaps: `totalLength == (finalPage - 1) * limit + entries(finalPage)`. The end resolves when a page comes back short, when an empty page has a loaded full predecessor, or when page 1 is empty. An empty page *without* a loaded predecessor does NOT resolve it -- jumping to page 50 of a 3-page result only proves the end is somewhere before 50 -- but it is still recorded to avoid re-fetching that page or any after it. Concurrent requests for the same page share one fetch, and a failed load is evicted so a retry actually retries. Tests: 38 unit tests over a fake page loader, covering the full state machine -- sparse gaps, all three end-resolution rules, the jump-past-the-end case, dedupe, failure eviction, reset/refresh. Co-Authored-By: Claude Opus 5 (1M context) --- lib/bones_api.dart | 1 + lib/src/bones_api_entity_pagination.dart | 462 +++++++++++++++ test/bones_api_entity_pagination_test.dart | 631 +++++++++++++++++++++ 3 files changed, 1094 insertions(+) create mode 100644 lib/src/bones_api_entity_pagination.dart create mode 100644 test/bones_api_entity_pagination_test.dart diff --git a/lib/bones_api.dart b/lib/bones_api.dart index 4cb4553..c8f48f0 100644 --- a/lib/bones_api.dart +++ b/lib/bones_api.dart @@ -325,6 +325,7 @@ export 'src/bones_api_entity_db_memory.dart'; export 'src/bones_api_entity_db_object_memory.dart'; export 'src/bones_api_entity_db_relational.dart'; export 'src/bones_api_entity_db_sql.dart'; +export 'src/bones_api_entity_pagination.dart'; export 'src/bones_api_entity_reference.dart'; export 'src/bones_api_entity_rules.dart'; export 'src/bones_api_entity_sql.dart'; diff --git a/lib/src/bones_api_entity_pagination.dart b/lib/src/bones_api_entity_pagination.dart new file mode 100644 index 0000000..5f55c7f --- /dev/null +++ b/lib/src/bones_api_entity_pagination.dart @@ -0,0 +1,462 @@ +import 'package:async_extension/async_extension.dart'; + +/// Loads the entries of [page] (1-based) of an [EntityPagination], +/// with at most [limit] entries. +typedef EntityPageLoader = + FutureOr> Function(int page, int limit); + +/// A lazily loaded, paginated view over a select operation. +/// +/// It keeps the pages it has already loaded and never fetches implicitly: +/// the synchronous accessors ([operator []], [loadedEntities]) only ever see +/// what is loaded, and only the `FutureOr` methods ([getAt], [getPage], +/// [loadNextPage], [loadAll], [stream]) fetch. +/// +/// Pages are **1-based** (matching the `page` parameter of the `select*` +/// methods) and entry indexes are **0-based** (matching a Dart `List`): +/// see [indexOfPage] and [pageOfIndex]. +/// +/// Pages can be loaded out of order, leaving gaps: `getAt(45)` with a [limit] +/// of 20 loads only page 3, so [loadedPages] is `[3]` while indexes `0..39` +/// stay unloaded. +/// +/// ## The total length is not known upfront +/// +/// A paginated select doesn't count the matching entries, so [totalLength] is +/// `null` until the final page is identified. Since every page except the last +/// one holds exactly [limit] entries, identifying the final page is enough to +/// compute the total — even when there are gaps: +/// +/// ``` +/// totalLength == (finalPage - 1) * limit + +/// ``` +/// +/// See [isFinalPageResolved] for when that happens. +/// +/// ## Not a `List` +/// +/// This is deliberately not a `List` or an `Iterable`: both require a `length`, +/// which is exactly what a paginated select can't answer until it reaches the +/// end. Use [operator []] for what is loaded, and [loadAll] when a full list is +/// really needed. +/// +/// ## Consistency +/// +/// Each page is an independent select, without a shared `Transaction`. Entries +/// inserted or deleted between two page loads shift the offsets, so a page +/// loaded later can repeat or skip entries. This is inherent to offset-based +/// pagination; ordering by ID (the default of the `paginate*` methods) makes it +/// as stable as it can be. +class EntityPagination { + /// The page size: the maximum number of entries of a page. + /// + /// Every page except the final one holds exactly [limit] entries. + final int limit; + + /// Loads a page. See [EntityPageLoader]. + final EntityPageLoader pageLoader; + + /// An optional description of the paginated query, for [toString]. + final String? query; + + EntityPagination({ + required this.limit, + required this.pageLoader, + this.query, + }) { + if (limit <= 0) { + throw ArgumentError.value(limit, 'limit', 'The page size must be > 0'); + } + } + + /// The loaded pages, by page number. + final Map> _pages = >{}; + + /// The in-flight page loads, so 2 concurrent requests for the same page + /// share a single fetch. + final Map>> _loadingPages = >>{}; + + int? _finalPage; + + /// The lowest page known to be empty. + /// + /// An empty page only tells that the final page is *before* it, not which + /// one it is, so it can't resolve [finalPage] by itself. It is still enough + /// to avoid fetching that page (or any page after it) again. + int? _minEmptyPage; + + /// The index of the 1st entry of [page]. + int indexOfPage(int page) => (page - 1) * limit; + + /// The page holding the entry at [index]. + int pageOfIndex(int index) => (index ~/ limit) + 1; + + // --------------------------------------------------------------------- + // What is loaded: + // --------------------------------------------------------------------- + + /// Returns `true` if [page] was already loaded (even if it came back empty). + bool isPageLoaded(int page) => _pages.containsKey(page); + + /// Returns `true` if the entry at [index] is loaded. + bool isIndexLoaded(int index) => this[index] != null; + + /// The loaded page numbers, ascending. Can have gaps. + List get loadedPages => _pages.keys.toList()..sort(); + + /// The number of loaded pages. + int get loadedPagesLength => _pages.length; + + /// The highest loaded page, or `null` if nothing was loaded yet. + /// + /// Counts a page that came back empty; see [maxKnownPage] for the highest + /// page known to actually hold entries. + int? get maxLoadedPage { + int? max; + for (var page in _pages.keys) { + if (max == null || page > max) max = page; + } + return max; + } + + /// The highest entry index that is loaded, or `null` if none is. + int? get maxLoadedIndex { + int? max; + for (var e in _pages.entries) { + var entries = e.value; + if (entries.isEmpty) continue; + var last = indexOfPage(e.key) + entries.length - 1; + if (max == null || last > max) max = last; + } + return max; + } + + /// The number of loaded entries. Entries in a gap are not counted, so this + /// can be less than `maxLoadedIndex + 1`. + int get loadedEntitiesLength => + _pages.values.fold(0, (total, entries) => total + entries.length); + + /// The loaded entries, in index order. Gaps are skipped. + List get loadedEntities => [ + for (var page in loadedPages) ..._pages[page]!, + ]; + + /// The loaded entries of [page], or `null` if it is not loaded. + List? loadedPageEntities(int page) => _pages[page]; + + // --------------------------------------------------------------------- + // What is known: + // --------------------------------------------------------------------- + + /// The highest page known to exist: the highest loaded page that came back + /// with entries. `null` if no page with entries was loaded yet. + int? get maxKnownPage { + int? max; + for (var e in _pages.entries) { + if (e.value.isEmpty) continue; + if (max == null || e.key > max) max = e.key; + } + return max; + } + + /// Whether the final page has been identified. + /// + /// Resolved when: + /// - a page comes back with `1..limit-1` entries: it is the final page; + /// - a page comes back empty and its previous page is loaded and full: + /// the previous page is the final one; + /// - page 1 comes back empty: the select matches nothing. + /// + /// A page that comes back empty *without* a loaded, full predecessor does + /// not resolve it: it only says the final page is somewhere before. + bool get isFinalPageResolved => _finalPage != null; + + /// The final page, or `null` while unresolved. See [isFinalPageResolved]. + int? get finalPage => _finalPage; + + /// The total number of entries, or `null` while [isFinalPageResolved] + /// is `false`. + int? get totalLength { + var finalPage = _finalPage; + if (finalPage == null) return null; + + var entries = _pages[finalPage]; + if (entries == null) return null; + + return indexOfPage(finalPage) + entries.length; + } + + /// Whether the select is known to match no entry at all. + bool get isKnownEmpty => totalLength == 0; + + /// Whether [index] is known to be out of range. + /// + /// Always `false` while [isFinalPageResolved] is `false`, since the end is + /// not known yet. + bool isIndexKnownOutOfRange(int index) { + if (index < 0) return true; + var total = totalLength; + return total != null && index >= total; + } + + // --------------------------------------------------------------------- + // Synchronous access: never fetches. + // --------------------------------------------------------------------- + + /// The loaded entry at [index], or `null` if it is not loaded (in a gap, in + /// a page not loaded yet, or out of range). + /// + /// Never fetches: use [getAt] to load on demand, and [isPageLoaded] to tell + /// "not loaded" from "out of range". + O? operator [](int index) { + if (index < 0) return null; + + var page = pageOfIndex(index); + var entries = _pages[page]; + if (entries == null) return null; + + var offset = index - indexOfPage(page); + if (offset < 0 || offset >= entries.length) return null; + + return entries[offset]; + } + + // --------------------------------------------------------------------- + // Asynchronous access: loads what it needs. + // --------------------------------------------------------------------- + + /// The entry at [index], loading its page if needed. + /// Returns `null` if [index] is out of range. + FutureOr getAt(int index) { + if (index < 0) return null; + + var cached = this[index]; + if (cached != null) return cached; + + var page = pageOfIndex(index); + // Loaded, but shorter than `index`: out of range. + if (isPageLoaded(page)) return null; + + return loadPage(page).resolveMapped((_) => this[index]); + } + + /// The entries of [page], loading it if needed. + FutureOr> getPage(int page) => loadPage(page); + + /// The entries from [startIndex] (inclusive) to [endIndex] (exclusive), + /// loading every page spanning the range, including pages inside a gap. + FutureOr> getRange(int startIndex, int endIndex) { + if (startIndex < 0) { + throw ArgumentError.value(startIndex, 'startIndex', 'Must be >= 0'); + } + + if (endIndex <= startIndex) return []; + + var firstPage = pageOfIndex(startIndex); + var lastPage = pageOfIndex(endIndex - 1); + + var loads = [ + for (var page = firstPage; page <= lastPage; ++page) loadPage(page), + ]; + + return loads.resolveAll().resolveMapped((_) { + var range = []; + for (var i = startIndex; i < endIndex; ++i) { + var o = this[i]; + if (o != null) range.add(o); + } + return range; + }); + } + + /// Loads [page] (1-based), or returns it if already loaded. + /// + /// Concurrent calls for the same page share a single fetch. + FutureOr> loadPage(int page) { + if (page < 1) { + throw ArgumentError.value( + page, + 'page', + 'Pages are 1-based, must be >= 1', + ); + } + + var loaded = _pages[page]; + if (loaded != null) return loaded; + + // Already known to be past the end, no need to fetch: + if (_isPageKnownEmpty(page)) return []; + + var loading = _loadingPages[page]; + if (loading != null) return loading; + + var ret = pageLoader(page, limit); + + if (ret is! Future>) { + _setPage(page, ret); + return _pages[page]!; + } + + var future = ret + .then((entries) { + _loadingPages.remove(page); + _setPage(page, entries); + return _pages[page]!; + }) + .onError((e, s) { + _loadingPages.remove(page); + throw e; + }); + + _loadingPages[page] = future; + return future; + } + + bool _isPageKnownEmpty(int page) { + var finalPage = _finalPage; + if (finalPage != null && page > finalPage) return true; + + var minEmptyPage = _minEmptyPage; + return minEmptyPage != null && page >= minEmptyPage; + } + + /// Loads the page after [maxLoadedPage] (or page 1 when nothing is loaded). + /// + /// Returns `null` when there is nothing more to load. + FutureOr?> loadNextPage() { + var next = (maxLoadedPage ?? 0) + 1; + + if (_isPageKnownEmpty(next)) return null; + + return loadPage( + next, + ).resolveMapped((entries) => entries.isEmpty ? null : entries); + } + + /// Loads pages, from the one after [maxLoadedPage], until the final page is + /// resolved or [maxPages] pages have been loaded. + /// + /// Returns [totalLength], which is `null` if [maxPages] stopped it before + /// the end was reached. + FutureOr loadAll({int? maxPages}) => _loadAllImpl(maxPages, 0); + + FutureOr _loadAllImpl(int? maxPages, int loadedCount) { + if (isFinalPageResolved) return totalLength; + if (maxPages != null && loadedCount >= maxPages) return totalLength; + + return loadNextPage().resolveMapped((entries) { + if (entries == null) return totalLength; + return _loadAllImpl(maxPages, loadedCount + 1); + }); + } + + /// Streams the entries from [fromPage], loading the pages as it goes. + Stream stream({int fromPage = 1}) async* { + var page = fromPage; + + while (true) { + var entries = await loadPage(page); + if (entries.isEmpty) break; + + for (var o in entries) { + yield o; + } + + var finalPage = _finalPage; + if (finalPage != null && page >= finalPage) break; + + ++page; + } + } + + // --------------------------------------------------------------------- + // Control: + // --------------------------------------------------------------------- + + void _setPage(int page, List entries) { + var list = List.unmodifiable(entries); + _pages[page] = list; + _resolveFinalPage(page, list); + } + + void _resolveFinalPage(int page, List entries) { + if (_finalPage != null) return; + + if (entries.isEmpty) { + var minEmptyPage = _minEmptyPage; + if (minEmptyPage == null || page < minEmptyPage) { + _minEmptyPage = page; + } + + // Nothing at all: + if (page == 1) { + _finalPage = 1; + return; + } + } else if (entries.length < limit) { + // A short page can only be the last one: + _finalPage = page; + return; + } + + // An empty page whose previous page is loaded and full pins the end: + var minEmptyPage = _minEmptyPage; + if (minEmptyPage != null && minEmptyPage > 1) { + var previous = _pages[minEmptyPage - 1]; + if (previous != null && previous.length == limit) { + _finalPage = minEmptyPage - 1; + } + } + } + + /// Discards every loaded page and everything known about the end, + /// keeping the query. + void reset() { + _pages.clear(); + _loadingPages.clear(); + _finalPage = null; + _minEmptyPage = null; + } + + /// Re-fetches the currently loaded pages, discarding what was known about + /// the end (it may have moved). + FutureOr refresh() { + var pages = loadedPages; + reset(); + + if (pages.isEmpty) return null; + + return pages + .map((page) => loadPage(page)) + .resolveAll() + .resolveMapped((_) => null); + } + + Map information({bool extended = false}) => { + 'limit': limit, + 'loadedPages': loadedPages, + 'loadedPagesLength': loadedPagesLength, + 'loadedEntitiesLength': loadedEntitiesLength, + if (maxLoadedPage != null) 'maxLoadedPage': maxLoadedPage, + if (maxLoadedIndex != null) 'maxLoadedIndex': maxLoadedIndex, + if (maxKnownPage != null) 'maxKnownPage': maxKnownPage, + 'isFinalPageResolved': isFinalPageResolved, + if (finalPage != null) 'finalPage': finalPage, + if (totalLength != null) 'totalLength': totalLength, + if (extended && query != null) 'query': query, + }; + + @override + String toString() { + var total = totalLength; + return 'EntityPagination<$O>{' + 'limit: $limit, ' + 'loadedPages: ${loadedPages.length}, ' + 'loadedEntities: $loadedEntitiesLength, ' + 'maxLoadedIndex: $maxLoadedIndex, ' + 'maxKnownPage: $maxKnownPage, ' + '${total != null ? 'totalLength: $total' : 'totalLength: '}' + '${query != null ? ' ; query: $query' : ''}' + '}'; + } +} diff --git a/test/bones_api_entity_pagination_test.dart b/test/bones_api_entity_pagination_test.dart new file mode 100644 index 0000000..8b82bc3 --- /dev/null +++ b/test/bones_api_entity_pagination_test.dart @@ -0,0 +1,631 @@ +@Tags(['entities']) +// ignore_for_file: discarded_futures +import 'package:bones_api/bones_api.dart'; +import 'package:test/test.dart'; + +/// A trivial entity, so the pagination state machine can be exercised without +/// a DB. +class _Item { + final int id; + + _Item(this.id); + + @override + String toString() => 'Item($id)'; +} + +/// A fake backing store of [total] items, recording every page fetch. +class _FakeSource { + final int total; + final bool async; + + /// Every page requested, in order. Duplicates here mean a page was fetched + /// more than once. + final List fetches = []; + + _FakeSource(this.total, {this.async = true}); + + FutureOr> load(int page, int limit) { + fetches.add(page); + + var start = (page - 1) * limit; + var entries = <_Item>[ + for (var i = start; i < start + limit && i < total; ++i) _Item(i), + ]; + + return async ? Future.value(entries) : entries; + } + + EntityPagination<_Item> pagination({int limit = 10}) => + EntityPagination<_Item>(limit: limit, pageLoader: load, query: 'fake'); +} + +List _ids(Iterable<_Item> items) => items.map((e) => e.id).toList(); + +void main() { + group('EntityPagination: construction', () { + test('rejects a non-positive limit', () { + var source = _FakeSource(10); + + expect( + () => EntityPagination<_Item>(limit: 0, pageLoader: source.load), + throwsArgumentError, + ); + expect( + () => EntityPagination<_Item>(limit: -1, pageLoader: source.load), + throwsArgumentError, + ); + }); + + test('starts empty and loads nothing', () { + var source = _FakeSource(100); + var p = source.pagination(); + + expect(source.fetches, isEmpty, reason: 'Must not fetch eagerly'); + + expect(p.loadedPages, isEmpty); + expect(p.loadedPagesLength, equals(0)); + expect(p.loadedEntities, isEmpty); + expect(p.loadedEntitiesLength, equals(0)); + expect(p.maxLoadedPage, isNull); + expect(p.maxLoadedIndex, isNull); + expect(p.maxKnownPage, isNull); + expect(p.isFinalPageResolved, isFalse); + expect(p.finalPage, isNull); + expect(p.totalLength, isNull); + expect(p.isKnownEmpty, isFalse); + expect(p[0], isNull); + }); + }); + + group('EntityPagination: page/index mapping', () { + test('pages are 1-based and indexes 0-based', () { + var p = _FakeSource(100).pagination(limit: 20); + + expect(p.indexOfPage(1), equals(0)); + expect(p.indexOfPage(2), equals(20)); + expect(p.indexOfPage(3), equals(40)); + + expect(p.pageOfIndex(0), equals(1)); + expect(p.pageOfIndex(19), equals(1)); + expect(p.pageOfIndex(20), equals(2)); + expect(p.pageOfIndex(45), equals(3)); + }); + + test('loadPage rejects a page below 1', () { + var p = _FakeSource(100).pagination(); + + expect(() => p.loadPage(0), throwsArgumentError); + expect(() => p.loadPage(-1), throwsArgumentError); + }); + }); + + group('EntityPagination: sequential loading', () { + test('loadNextPage walks the pages', () async { + var source = _FakeSource(25); + var p = source.pagination(limit: 10); + + expect( + _ids((await p.loadNextPage())!), + equals([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), + ); + expect(p.maxLoadedPage, equals(1)); + expect(p.maxLoadedIndex, equals(9)); + expect(p.maxKnownPage, equals(1)); + expect( + p.isFinalPageResolved, + isFalse, + reason: 'A full page is not final', + ); + expect(p.totalLength, isNull); + + expect( + _ids((await p.loadNextPage())!), + equals([10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), + ); + expect(p.isFinalPageResolved, isFalse); + + // Page 3 is short -> it is the final page: + expect(_ids((await p.loadNextPage())!), equals([20, 21, 22, 23, 24])); + expect(p.isFinalPageResolved, isTrue); + expect(p.finalPage, equals(3)); + expect(p.totalLength, equals(25)); + expect(p.maxLoadedIndex, equals(24)); + + // Nothing more: + expect(await p.loadNextPage(), isNull); + expect(source.fetches, equals([1, 2, 3]), reason: 'No extra fetch'); + }); + + test('sync page loader works without awaiting', () { + var source = _FakeSource(5, async: false); + var p = source.pagination(limit: 10); + + var entries = p.loadPage(1); + expect(entries, isA>()); + expect(_ids(entries as List<_Item>), equals([0, 1, 2, 3, 4])); + expect(p.totalLength, equals(5)); + }); + }); + + group('EntityPagination: final page resolution', () { + test('a short page is the final page', () async { + var p = _FakeSource(7).pagination(limit: 10); + + await p.loadPage(1); + expect(p.finalPage, equals(1)); + expect(p.totalLength, equals(7)); + expect(p.isKnownEmpty, isFalse); + }); + + test('an exactly-full last page needs the next page to resolve', () async { + var source = _FakeSource(20); + var p = source.pagination(limit: 10); + + await p.loadPage(1); + await p.loadPage(2); + expect( + p.isFinalPageResolved, + isFalse, + reason: 'Page 2 is full, page 3 might exist', + ); + + // Page 3 is empty and page 2 is loaded and full -> page 2 is final: + expect(await p.loadPage(3), isEmpty); + expect(p.isFinalPageResolved, isTrue); + expect(p.finalPage, equals(2)); + expect(p.totalLength, equals(20)); + }); + + test('an empty page 1 means the select matches nothing', () async { + var p = _FakeSource(0).pagination(limit: 10); + + expect(await p.loadPage(1), isEmpty); + expect(p.isFinalPageResolved, isTrue); + expect(p.finalPage, equals(1)); + expect(p.totalLength, equals(0)); + expect(p.isKnownEmpty, isTrue); + expect(p.maxKnownPage, isNull, reason: 'No page holds entries'); + expect(p.maxLoadedIndex, isNull); + }); + + test( + 'an empty page far past the end does NOT resolve the final page', + () async { + var source = _FakeSource(25); + var p = source.pagination(limit: 10); + + // Jump way past the end: only tells that the end is before page 50. + expect(await p.loadPage(50), isEmpty); + expect(p.isFinalPageResolved, isFalse); + expect(p.finalPage, isNull); + expect(p.totalLength, isNull); + expect(p.maxKnownPage, isNull); + expect( + p.maxLoadedPage, + equals(50), + reason: 'It was loaded, though empty', + ); + + // But it is enough to skip fetching that page, and any page after it: + source.fetches.clear(); + expect(await p.loadPage(50), isEmpty); + expect(await p.loadPage(60), isEmpty); + expect(source.fetches, isEmpty, reason: 'Known to be past the end'); + + // Reaching the short page still resolves it: + await p.loadPage(3); + expect(p.finalPage, equals(3)); + expect(p.totalLength, equals(25)); + }, + ); + + test( + 'a later empty page resolves once its predecessor is loaded full', + () async { + var p = _FakeSource(20).pagination(limit: 10); + + // Page 3 is empty, but page 2 is not loaded yet: unresolved. + expect(await p.loadPage(3), isEmpty); + expect(p.isFinalPageResolved, isFalse); + + // Loading page 2 (full) now pins the end at page 2: + await p.loadPage(2); + expect(p.isFinalPageResolved, isTrue); + expect(p.finalPage, equals(2)); + expect(p.totalLength, equals(20)); + }, + ); + }); + + group('EntityPagination: sparse loading (gaps)', () { + test('getAt loads only the page it needs', () async { + var source = _FakeSource(100); + var p = source.pagination(limit: 20); + + var item = await p.getAt(45); + expect(item?.id, equals(45)); + + expect(source.fetches, equals([3]), reason: 'Only page 3'); + expect(p.loadedPages, equals([3])); + expect(p.maxLoadedPage, equals(3)); + expect(p.maxLoadedIndex, equals(59)); + expect(p.maxKnownPage, equals(3)); + expect(p.loadedEntitiesLength, equals(20)); + + // Everything before page 3 is a gap: + expect(p[0], isNull); + expect(p[25], isNull); + expect(p.isIndexLoaded(25), isFalse); + expect(p[40], isNotNull); + expect(p.isIndexLoaded(40), isTrue); + }); + + test('loadedEntities is in index order, gaps skipped', () async { + var p = _FakeSource(100).pagination(limit: 10); + + // Deliberately out of order: + await p.loadPage(3); + await p.loadPage(1); + + expect(p.loadedPages, equals([1, 3])); + expect( + _ids(p.loadedEntities), + equals([ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + ]), + ); + expect(p.loadedEntitiesLength, equals(20)); + expect( + p.maxLoadedIndex, + equals(29), + reason: 'maxLoadedIndex counts the gap, loadedEntitiesLength does not', + ); + }); + + test('loadNextPage continues after the highest loaded page', () async { + var source = _FakeSource(100); + var p = source.pagination(limit: 10); + + await p.loadPage(5); + source.fetches.clear(); + + var next = await p.loadNextPage(); + expect(_ids(next!).first, equals(50)); + expect(source.fetches, equals([6])); + }); + }); + + group('EntityPagination: access', () { + test('operator [] never fetches', () async { + var source = _FakeSource(100); + var p = source.pagination(limit: 10); + + expect(p[5], isNull); + expect(source.fetches, isEmpty); + + await p.loadPage(1); + source.fetches.clear(); + + expect(p[5]?.id, equals(5)); + expect(p[50], isNull); + expect(source.fetches, isEmpty, reason: 'Sync access must not fetch'); + }); + + test('getAt returns null out of range without fetching twice', () async { + var source = _FakeSource(5); + var p = source.pagination(limit: 10); + + expect((await p.getAt(3))?.id, equals(3)); + expect(await p.getAt(7), isNull, reason: 'Page loaded but shorter'); + + expect(source.fetches, equals([1])); + }); + + test('getAt with a negative index', () async { + var p = _FakeSource(10).pagination(); + expect(await p.getAt(-1), isNull); + expect(p[-1], isNull); + }); + + test('getRange loads every page spanning the range', () async { + var source = _FakeSource(100); + var p = source.pagination(limit: 10); + + var range = await p.getRange(5, 25); + expect(_ids(range), equals([for (var i = 5; i < 25; ++i) i])); + expect(source.fetches, equals([1, 2, 3])); + }); + + test('getRange fills a gap', () async { + var source = _FakeSource(100); + var p = source.pagination(limit: 10); + + await p.loadPage(1); + await p.loadPage(3); + source.fetches.clear(); + + var range = await p.getRange(0, 30); + expect(_ids(range), equals([for (var i = 0; i < 30; ++i) i])); + expect(source.fetches, equals([2]), reason: 'Only the missing page'); + }); + + test('getRange edge cases', () async { + var p = _FakeSource(100).pagination(limit: 10); + + expect(await p.getRange(5, 5), isEmpty); + expect(await p.getRange(5, 1), isEmpty); + expect(() => p.getRange(-1, 5), throwsArgumentError); + }); + + test('getRange past the end returns what exists', () async { + var p = _FakeSource(12).pagination(limit: 10); + + var range = await p.getRange(8, 30); + expect(_ids(range), equals([8, 9, 10, 11])); + expect(p.totalLength, equals(12)); + }); + }); + + group('EntityPagination: loadAll and stream', () { + test('loadAll resolves the total', () async { + var source = _FakeSource(25); + var p = source.pagination(limit: 10); + + expect(await p.loadAll(), equals(25)); + expect(p.isFinalPageResolved, isTrue); + expect(p.finalPage, equals(3)); + expect(_ids(p.loadedEntities), equals([for (var i = 0; i < 25; ++i) i])); + expect(source.fetches, equals([1, 2, 3])); + }); + + test('loadAll on an empty result', () async { + var p = _FakeSource(0).pagination(limit: 10); + + expect(await p.loadAll(), equals(0)); + expect(p.isKnownEmpty, isTrue); + }); + + test('loadAll on an exact multiple of the page size', () async { + var source = _FakeSource(20); + var p = source.pagination(limit: 10); + + expect(await p.loadAll(), equals(20)); + expect(p.finalPage, equals(2)); + expect( + source.fetches, + equals([1, 2, 3]), + reason: 'Needs the empty page 3 to know page 2 was the last', + ); + }); + + test( + 'loadAll with maxPages stops early and leaves it unresolved', + () async { + var source = _FakeSource(100); + var p = source.pagination(limit: 10); + + expect(await p.loadAll(maxPages: 2), isNull); + expect(p.isFinalPageResolved, isFalse); + expect(p.loadedPages, equals([1, 2])); + expect(source.fetches, equals([1, 2])); + + // Resuming continues from where it stopped: + expect(await p.loadAll(), equals(100)); + expect(p.finalPage, equals(10)); + }, + ); + + test('stream walks every entry', () async { + var source = _FakeSource(25); + var p = source.pagination(limit: 10); + + expect( + _ids(await p.stream().toList()), + equals([for (var i = 0; i < 25; ++i) i]), + ); + expect(p.totalLength, equals(25)); + }); + + test('stream from a given page', () async { + var p = _FakeSource(25).pagination(limit: 10); + + expect( + _ids(await p.stream(fromPage: 3).toList()), + equals([20, 21, 22, 23, 24]), + ); + }); + + test('stream of an empty result', () async { + var p = _FakeSource(0).pagination(limit: 10); + expect(await p.stream().toList(), isEmpty); + }); + }); + + group('EntityPagination: caching and dedupe', () { + test('a loaded page is not fetched again', () async { + var source = _FakeSource(100); + var p = source.pagination(limit: 10); + + await p.loadPage(2); + await p.loadPage(2); + await p.getPage(2); + await p.getAt(15); + + expect(source.fetches, equals([2])); + }); + + test( + 'concurrent requests for the same page share a single fetch', + () async { + var source = _FakeSource(100); + var p = source.pagination(limit: 10); + + // Started together, before any completes: + var results = await Future.wait([ + Future.value(p.getAt(3)), + Future.value(p.getAt(7)), + Future.value(p.loadPage(1)), + ]); + + expect((results[0] as _Item?)?.id, equals(3)); + expect((results[1] as _Item?)?.id, equals(7)); + expect( + source.fetches, + equals([1]), + reason: 'Page 1 must be fetched once, not 3 times', + ); + }, + ); + + test( + 'a failing page load does not leave a stuck in-flight entry', + () async { + var attempts = 0; + + var p = EntityPagination<_Item>( + limit: 10, + pageLoader: (page, limit) async { + ++attempts; + if (attempts == 1) throw StateError('boom'); + return [_Item(0)]; + }, + ); + + await expectLater(p.loadPage(1), throwsA(isA())); + expect(p.isPageLoaded(1), isFalse); + + // Retrying must actually retry, not replay the failed future: + var entries = await p.loadPage(1); + expect(_ids(entries), equals([0])); + expect(attempts, equals(2)); + }, + ); + }); + + group('EntityPagination: control', () { + test('reset drops everything and keeps the query', () async { + var source = _FakeSource(25); + var p = source.pagination(limit: 10); + + await p.loadAll(); + expect(p.totalLength, equals(25)); + + p.reset(); + + expect(p.loadedPages, isEmpty); + expect(p.loadedEntitiesLength, equals(0)); + expect(p.isFinalPageResolved, isFalse); + expect(p.totalLength, isNull); + expect(p.maxLoadedIndex, isNull); + + // Still usable: + source.fetches.clear(); + expect(_ids((await p.loadNextPage())!).first, equals(0)); + expect(source.fetches, equals([1])); + }); + + test('refresh re-fetches exactly the loaded pages', () async { + var source = _FakeSource(100); + var p = source.pagination(limit: 10); + + await p.loadPage(1); + await p.loadPage(4); + source.fetches.clear(); + + await p.refresh(); + + expect(source.fetches..sort(), equals([1, 4])); + expect(p.loadedPages, equals([1, 4])); + expect(p.loadedEntitiesLength, equals(20)); + }); + + test('refresh on an untouched pagination is a no-op', () async { + var source = _FakeSource(100); + var p = source.pagination(); + + await p.refresh(); + expect(source.fetches, isEmpty); + }); + + test('refresh discards a resolved end', () async { + var source = _FakeSource(25); + var p = source.pagination(limit: 10); + + await p.loadAll(); + expect(p.finalPage, equals(3)); + + await p.refresh(); + + // Page 3 is short, so reloading it resolves the end again: + expect(p.finalPage, equals(3)); + expect(p.totalLength, equals(25)); + }); + }); + + group('EntityPagination: information', () { + test('reports the state', () async { + var p = _FakeSource(25).pagination(limit: 10); + + var before = p.information(); + expect(before['limit'], equals(10)); + expect(before['loadedPages'], isEmpty); + expect(before['isFinalPageResolved'], isFalse); + expect(before.containsKey('totalLength'), isFalse); + + await p.loadAll(); + + var after = p.information(); + expect(after['loadedPages'], equals([1, 2, 3])); + expect(after['loadedEntitiesLength'], equals(25)); + expect(after['maxLoadedIndex'], equals(24)); + expect(after['maxKnownPage'], equals(3)); + expect(after['isFinalPageResolved'], isTrue); + expect(after['finalPage'], equals(3)); + expect(after['totalLength'], equals(25)); + + expect(p.information(extended: true)['query'], equals('fake')); + expect(p.toString(), contains('totalLength: 25')); + }); + + test('toString marks an unresolved total', () { + var p = _FakeSource(25).pagination(); + expect(p.toString(), contains('')); + }); + + test('isIndexKnownOutOfRange', () async { + var p = _FakeSource(25).pagination(limit: 10); + + expect(p.isIndexKnownOutOfRange(-1), isTrue); + expect( + p.isIndexKnownOutOfRange(1000), + isFalse, + reason: 'The end is not known yet', + ); + + await p.loadAll(); + + expect(p.isIndexKnownOutOfRange(24), isFalse); + expect(p.isIndexKnownOutOfRange(25), isTrue); + }); + }); +} From b13a8facc7805484adfe93faf3786ad87f117fe8 Mon Sep 17 00:00:00 2001 From: "Graciliano M. P." Date: Sat, 1 Aug 2026 18:20:29 -0300 Subject: [PATCH 2/3] feat: paginateByQuery / paginate / paginateAll entry points Builds an `EntityPagination` over the existing select path, on `EntitySource`, `EntityRepository` (adding `resolutionRules`, mirroring how `selectByQuery` is split) and the `APIRepository` facade. Each page is `selectByQuery(..., limit: limit, page: page)`, so it rides entirely on the pagination shipped in 1.11.0. `orderByID` defaults to `true` rather than following the `offset != null` rule: a paginated read is only meaningful over a stable order, so it should not be opt-in here. Also fixes `loadAll()`, caught by the integration test: it returned as soon as the end was resolved, so a sparse `getAt` that had already resolved the final page left its gap unfilled and `loadedEntities` was missing entries. It now walks from page 1, skipping already-loaded pages without re-fetching them (and without spending the `maxPages` budget), so afterwards the result is complete and gap-free. Tests: an `EntityPagination: lazy paged access` test in the shared adapter template -- lazy start, page 1, a jump to page 3 leaving a gap, end resolution from a short page, `loadAll` filling the gap, `getRange`, descending, streaming, and an empty query. Runs against the in-memory, PostgreSQL, MySQL and object-directory adapters; PostgreSQL and MySQL verified on real containers. Co-Authored-By: Claude Opus 5 (1M context) --- lib/src/bones_api_entity.dart | 171 ++++++++++++++++++ lib/src/bones_api_entity_pagination.dart | 33 +++- lib/src/bones_api_repository.dart | 62 +++++++ test/bones_api_entity_db_sql_select_test.dart | 89 +++++++++ test/bones_api_entity_db_tests_base.dart | 92 ++++++++++ 5 files changed, 438 insertions(+), 9 deletions(-) diff --git a/lib/src/bones_api_entity.dart b/lib/src/bones_api_entity.dart index 72a9d6c..5eaf6ac 100644 --- a/lib/src/bones_api_entity.dart +++ b/lib/src/bones_api_entity.dart @@ -19,6 +19,7 @@ import 'package:swiss_knife/swiss_knife.dart' show EventStream, DataURLBase64; import 'bones_api_base.dart'; import 'bones_api_condition.dart'; import 'bones_api_entity_annotation.dart'; +import 'bones_api_entity_pagination.dart'; import 'bones_api_entity_reference.dart'; import 'bones_api_entity_rules.dart'; import 'bones_api_error_zone.dart'; @@ -3632,6 +3633,88 @@ abstract class EntitySource extends EntityAccessor { EntityResolutionRules? resolutionRules, }); + /// {@template bones_api.paginate} + /// Returns an [EntityPagination] over this select, loading nothing yet: + /// pages are fetched only when asked for. See [EntityPagination]. + /// + /// - [limit]: the page size. Required, and must be `> 0`. + /// - [orderByID]: defaults to `true`. A paginated read is only meaningful + /// over a stable order, so unlike the `select*` methods this is on by + /// default instead of being implied by an offset. + /// - [orderDirection]: the [OrderDirection] of the ordering. + /// {@endtemplate} + EntityPagination paginateByQuery( + String query, { + Object? parameters, + List? positionalParameters, + Map? namedParameters, + required int limit, + bool? orderByID, + OrderDirection? orderDirection, + Transaction? transaction, + }) => EntityPagination( + limit: limit, + query: query, + pageLoader: + (page, limit) => selectByQuery( + query, + parameters: parameters, + positionalParameters: positionalParameters, + namedParameters: namedParameters, + transaction: transaction, + limit: limit, + page: page, + orderByID: orderByID ?? true, + orderDirection: orderDirection, + ).resolveMapped((os) => os.toList()), + ); + + /// {@macro bones_api.paginate} + EntityPagination paginate( + EntityMatcher matcher, { + Object? parameters, + List? positionalParameters, + Map? namedParameters, + required int limit, + bool? orderByID, + OrderDirection? orderDirection, + Transaction? transaction, + }) => EntityPagination( + limit: limit, + query: '$matcher', + pageLoader: + (page, limit) => select( + matcher, + parameters: parameters, + positionalParameters: positionalParameters, + namedParameters: namedParameters, + transaction: transaction, + limit: limit, + page: page, + orderByID: orderByID ?? true, + orderDirection: orderDirection, + ).resolveMapped((os) => os.toList()), + ); + + /// {@macro bones_api.paginate} + EntityPagination paginateAll({ + required int limit, + bool? orderByID, + OrderDirection? orderDirection, + Transaction? transaction, + }) => EntityPagination( + limit: limit, + query: 'ALL', + pageLoader: + (page, limit) => selectAll( + transaction: transaction, + limit: limit, + page: page, + orderByID: orderByID ?? true, + orderDirection: orderDirection, + ).resolveMapped((os) => os.toList()), + ); + FutureOr> selectRelationship( O? o, String field, { @@ -5725,6 +5808,94 @@ abstract class EntityRepository extends EntityAccessor ); } + /// {@macro bones_api.paginate} + /// + /// - [resolutionRules]: applied to every loaded page. + @override + EntityPagination paginateByQuery( + String query, { + Object? parameters, + List? positionalParameters, + Map? namedParameters, + required int limit, + bool? orderByID, + OrderDirection? orderDirection, + Transaction? transaction, + EntityResolutionRules? resolutionRules, + }) => EntityPagination( + limit: limit, + query: query, + pageLoader: + (page, limit) => selectByQuery( + query, + parameters: parameters, + positionalParameters: positionalParameters, + namedParameters: namedParameters, + transaction: transaction, + limit: limit, + page: page, + orderByID: orderByID ?? true, + orderDirection: orderDirection, + resolutionRules: resolutionRules, + ).resolveMapped((os) => os.toList()), + ); + + /// {@macro bones_api.paginate} + /// + /// - [resolutionRules]: applied to every loaded page. + @override + EntityPagination paginate( + EntityMatcher matcher, { + Object? parameters, + List? positionalParameters, + Map? namedParameters, + required int limit, + bool? orderByID, + OrderDirection? orderDirection, + Transaction? transaction, + EntityResolutionRules? resolutionRules, + }) => EntityPagination( + limit: limit, + query: '$matcher', + pageLoader: + (page, limit) => select( + matcher, + parameters: parameters, + positionalParameters: positionalParameters, + namedParameters: namedParameters, + transaction: transaction, + limit: limit, + page: page, + orderByID: orderByID ?? true, + orderDirection: orderDirection, + resolutionRules: resolutionRules, + ).resolveMapped((os) => os.toList()), + ); + + /// {@macro bones_api.paginate} + /// + /// - [resolutionRules]: applied to every loaded page. + @override + EntityPagination paginateAll({ + required int limit, + bool? orderByID, + OrderDirection? orderDirection, + Transaction? transaction, + EntityResolutionRules? resolutionRules, + }) => EntityPagination( + limit: limit, + query: 'ALL', + pageLoader: + (page, limit) => selectAll( + transaction: transaction, + limit: limit, + page: page, + orderByID: orderByID ?? true, + orderDirection: orderDirection, + resolutionRules: resolutionRules, + ).resolveMapped((os) => os.toList()), + ); + @override FutureOr deleteEntity(O o, {Transaction? transaction}) => deleteByID(getEntityID(o), transaction: transaction); diff --git a/lib/src/bones_api_entity_pagination.dart b/lib/src/bones_api_entity_pagination.dart index 5f55c7f..e689679 100644 --- a/lib/src/bones_api_entity_pagination.dart +++ b/lib/src/bones_api_entity_pagination.dart @@ -333,20 +333,35 @@ class EntityPagination { ).resolveMapped((entries) => entries.isEmpty ? null : entries); } - /// Loads pages, from the one after [maxLoadedPage], until the final page is - /// resolved or [maxPages] pages have been loaded. + /// Loads every page from page 1 until the end, so that afterwards + /// [loadedEntities] holds the complete result with no gap. + /// + /// Pages already loaded are skipped without re-fetching (and without + /// counting against [maxPages]), so this also fills the gaps left by a + /// sparse [getAt]. /// /// Returns [totalLength], which is `null` if [maxPages] stopped it before /// the end was reached. - FutureOr loadAll({int? maxPages}) => _loadAllImpl(maxPages, 0); + FutureOr loadAll({int? maxPages}) => _loadAllImpl(maxPages, 0, 1); + + FutureOr _loadAllImpl(int? maxPages, int fetchCount, int page) { + // Skip what is already loaded: no fetch, no budget spent. + while (isPageLoaded(page)) { + var finalPage = _finalPage; + if (finalPage != null && page >= finalPage) return totalLength; + ++page; + } + + var finalPage = _finalPage; + if (finalPage != null && page > finalPage) return totalLength; + + if (_isPageKnownEmpty(page)) return totalLength; - FutureOr _loadAllImpl(int? maxPages, int loadedCount) { - if (isFinalPageResolved) return totalLength; - if (maxPages != null && loadedCount >= maxPages) return totalLength; + if (maxPages != null && fetchCount >= maxPages) return totalLength; - return loadNextPage().resolveMapped((entries) { - if (entries == null) return totalLength; - return _loadAllImpl(maxPages, loadedCount + 1); + return loadPage(page).resolveMapped((entries) { + if (entries.isEmpty) return totalLength; + return _loadAllImpl(maxPages, fetchCount + 1, page + 1); }); } diff --git a/lib/src/bones_api_repository.dart b/lib/src/bones_api_repository.dart index 4e45ef6..64ff90e 100644 --- a/lib/src/bones_api_repository.dart +++ b/lib/src/bones_api_repository.dart @@ -3,6 +3,7 @@ import 'package:swiss_knife/swiss_knife.dart'; import 'bones_api_condition.dart'; import 'bones_api_entity.dart'; +import 'bones_api_entity_pagination.dart'; import 'bones_api_entity_rules.dart'; import 'bones_api_initializable.dart'; import 'bones_api_types.dart'; @@ -274,6 +275,67 @@ abstract class APIRepository with Initializable { resolutionRules: resolutionRules, ); + /// {@macro bones_api.paginate} + EntityPagination paginateByQuery( + String query, { + Object? parameters, + List? positionalParameters, + Map? namedParameters, + required int limit, + bool? orderByID, + OrderDirection? orderDirection, + Transaction? transaction, + EntityResolutionRules? resolutionRules, + }) => entityRepository.paginateByQuery( + query, + parameters: parameters, + positionalParameters: positionalParameters, + namedParameters: namedParameters, + limit: limit, + orderByID: orderByID, + orderDirection: orderDirection, + transaction: transaction, + resolutionRules: resolutionRules, + ); + + /// {@macro bones_api.paginate} + EntityPagination paginate( + EntityMatcher matcher, { + Object? parameters, + List? positionalParameters, + Map? namedParameters, + required int limit, + bool? orderByID, + OrderDirection? orderDirection, + Transaction? transaction, + EntityResolutionRules? resolutionRules, + }) => entityRepository.paginate( + matcher, + parameters: parameters, + positionalParameters: positionalParameters, + namedParameters: namedParameters, + limit: limit, + orderByID: orderByID, + orderDirection: orderDirection, + transaction: transaction, + resolutionRules: resolutionRules, + ); + + /// {@macro bones_api.paginate} + EntityPagination paginateAll({ + required int limit, + bool? orderByID, + OrderDirection? orderDirection, + Transaction? transaction, + EntityResolutionRules? resolutionRules, + }) => entityRepository.paginateAll( + limit: limit, + orderByID: orderByID, + orderDirection: orderDirection, + transaction: transaction, + resolutionRules: resolutionRules, + ); + FutureOr> deleteByQuery( String query, { Object? parameters, diff --git a/test/bones_api_entity_db_sql_select_test.dart b/test/bones_api_entity_db_sql_select_test.dart index a5eb530..d68da19 100644 --- a/test/bones_api_entity_db_sql_select_test.dart +++ b/test/bones_api_entity_db_sql_select_test.dart @@ -616,6 +616,95 @@ void main() { ); }); + test('paginateByQuery over the real select path', () async { + var p = roleRepository.paginateByQuery( + ' id >= ? ', + parameters: [ids.first], + limit: 2, + ); + + // Nothing is loaded until asked: + expect(p.loadedPages, isEmpty); + expect(p.totalLength, isNull); + expect(p[0], isNull); + + expect((await p.loadNextPage())!.map((e) => e.id), equals(ids.take(2))); + expect(p[0]?.id, equals(ids[0])); + expect(p[1]?.id, equals(ids[1])); + expect(p.maxLoadedIndex, equals(1)); + expect(p.maxKnownPage, equals(1)); + expect(p.isFinalPageResolved, isFalse); + + // Jump ahead, leaving page 2 as a gap: + expect((await p.getAt(4))?.id, equals(ids[4])); + expect(p.loadedPages, equals([1, 3])); + expect(p[2], isNull, reason: 'Page 2 is a gap'); + expect(p.loadedEntitiesLength, equals(3)); + expect(p.maxLoadedIndex, equals(4)); + + // Page 3 is short (5 roles, limit 2) -> the end is resolved: + expect(p.isFinalPageResolved, isTrue); + expect(p.finalPage, equals(3)); + expect(p.totalLength, equals(5)); + + // Filling the gap gives the full ordered set: + await p.loadAll(); + expect(p.loadedEntities.map((e) => e.id).toList(), equals(ids)); + }); + + test('paginateByQuery: loadAll, stream and an empty query', () async { + var p = roleRepository.paginateByQuery( + ' id >= ? ', + parameters: [ids.first], + limit: 2, + ); + + expect(await p.loadAll(), equals(5)); + expect(p.loadedEntities.map((e) => e.id).toList(), equals(ids)); + + var streamed = + await roleRepository + .paginateByQuery(' id >= ? ', parameters: [ids.first], limit: 2) + .stream() + .toList(); + expect(streamed.map((e) => e.id).toList(), equals(ids)); + + // A query matching nothing resolves as empty: + var none = roleRepository.paginateByQuery( + ' id > ? ', + parameters: [ids.last], + limit: 2, + ); + expect(await none.loadAll(), equals(0)); + expect(none.isKnownEmpty, isTrue); + expect(none.finalPage, equals(1)); + expect(none[0], isNull); + }); + + test('paginateByQuery: descending, paginate() and paginateAll()', () async { + var desc = roleRepository.paginateByQuery( + ' id >= ? ', + parameters: [ids.first], + limit: 2, + orderDirection: OrderDirection.descending, + ); + await desc.loadAll(); + expect( + desc.loadedEntities.map((e) => e.id).toList(), + equals(ids.reversed.toList()), + ); + + var byMatcher = roleRepository.paginate(ConditionANY(), limit: 3); + expect( + (await byMatcher.getPage(1)).map((e) => e.id), + equals(ids.take(3)), + ); + + var all = roleRepository.paginateAll(limit: 4); + expect(await all.loadAll(), equals(5)); + expect(all.loadedEntities.map((e) => e.id).toList(), equals(ids)); + }); + test('count is not affected by the ordering', () async { expect(await roleRepository.length(), equals(ids.length)); expect( diff --git a/test/bones_api_entity_db_tests_base.dart b/test/bones_api_entity_db_tests_base.dart index 042b94a..82b76f5 100644 --- a/test/bones_api_entity_db_tests_base.dart +++ b/test/bones_api_entity_db_tests_base.dart @@ -2681,6 +2681,98 @@ Future runAdapterTests( expect(await selectNone(offset: 5, limit: 2), isEmpty); }); + test('EntityPagination: lazy paged access', () async { + final campaignRepo = entityRepositoryProvider.campaignAPIRepository; + + var ids = []; + for (var i = 1; i <= 7; ++i) { + var id = await campaignRepo.store(Campaign('PAGINATION-0$i')); + ids.add(id as int); + } + expect(ids, orderedEquals([...ids]..sort())); + + var p = campaignRepo.paginateByQuery( + ' id >= ? ', + parameters: [ids.first], + limit: 3, + ); + + // Nothing is fetched until asked: + expect(p.loadedPages, isEmpty); + expect(p.totalLength, isNull); + expect(p[0], isNull); + + // Page 1: + var page1 = await p.loadNextPage(); + expect(page1!.map((e) => e.id).toList(), equals(ids.take(3).toList())); + expect(p[0]?.id, equals(ids[0])); + expect(p[2]?.id, equals(ids[2])); + expect(p.maxLoadedIndex, equals(2)); + expect(p.maxKnownPage, equals(1)); + expect( + p.isFinalPageResolved, + isFalse, + reason: 'A full page does not resolve the end', + ); + expect(p.totalLength, isNull); + + // Jump to index 6 (page 3), leaving page 2 as a gap: + expect((await p.getAt(6))?.id, equals(ids[6])); + expect(p.loadedPages, equals([1, 3])); + expect(p[3], isNull, reason: 'Page 2 was never loaded'); + expect(p.isIndexLoaded(3), isFalse); + expect(p.loadedEntitiesLength, equals(4)); + expect(p.maxLoadedIndex, equals(6)); + + // Page 3 holds 1 of 3 entries -> it is the final page: + expect(p.isFinalPageResolved, isTrue); + expect(p.finalPage, equals(3)); + expect(p.totalLength, equals(7)); + expect(p.isIndexKnownOutOfRange(7), isTrue); + expect(await p.getAt(7), isNull); + + // `loadAll` fills the gap left by the jump: + expect(await p.loadAll(), equals(7)); + expect(p.loadedPages, equals([1, 2, 3])); + expect(p.loadedEntities.map((e) => e.id).toList(), equals(ids)); + + // `getRange` across the whole set: + var range = await p.getRange(2, 5); + expect(range.map((e) => e.id).toList(), equals(ids.sublist(2, 5))); + + // Descending pagination: + var desc = campaignRepo.paginateByQuery( + ' id >= ? ', + parameters: [ids.first], + limit: 3, + orderDirection: OrderDirection.descending, + ); + expect(await desc.loadAll(), equals(7)); + expect( + desc.loadedEntities.map((e) => e.id).toList(), + equals(ids.reversed.toList()), + ); + + // Streaming walks every entry, in order: + var streamed = + await campaignRepo + .paginateByQuery(' id >= ? ', parameters: [ids.first], limit: 2) + .stream() + .toList(); + expect(streamed.map((e) => e.id).toList(), equals(ids)); + + // A query matching nothing resolves as empty: + var none = campaignRepo.paginateByQuery( + ' name == ? ', + parameters: ['PAGINATION-NO-SUCH-CAMPAIGN'], + limit: 3, + ); + expect(await none.loadAll(), equals(0)); + expect(none.isKnownEmpty, isTrue); + expect(none.finalPage, equals(1)); + expect(none[0], isNull); + }); + test('Pagination [objectAdapter]: orderByID / offset / limit', () async { final photoRepo = entityRepositoryProvider2.photoAPIRepository; From 11b801c7832f0b7271e7663ec35b451b40aeaa10 Mon Sep 17 00:00:00 2001 From: "Graciliano M. P." Date: Sat, 1 Aug 2026 18:21:42 -0300 Subject: [PATCH 3/3] chore: v1.12.0 (CHANGELOG, version, README) Documents `EntityPagination` and the `paginate*` entry points: the 1-based page / 0-based index split, the sparse gap behaviour, why synchronous access never fetches, why it is deliberately not a `List`, how the end (and therefore the total) is resolved, and the consistency caveat of offset-based pagination across independent page loads. Minor bump: purely additive. No existing signature or behaviour changes. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 57 +++++++++++++++++++++++++++++++++++++ README.md | 36 +++++++++++++++++++++++ lib/src/bones_api_base.dart | 2 +- pubspec.yaml | 2 +- 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddf9adc..e42689f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,60 @@ +## 1.12.0 + +- New `EntityPagination`: a lazily loaded, paginated view over a select, + for reading a result page by page without knowing its total length upfront. + + ```dart + var p = userRepository.paginateByQuery(' state == ? ', + parameters: ['NY'], limit: 20); + + await p.loadNextPage(); // page 1 + p[0]; // sync, already loaded + await p.getAt(45); // loads page 3 on demand, leaving page 2 a gap + await p.loadAll(); // fills the gaps and resolves the total + ``` + + - Pages are **1-based** (matching the `page` parameter of the `select*` + methods); entry indexes are **0-based** (matching a Dart `List`). See + `indexOfPage` / `pageOfIndex`. + - Pages can be loaded out of order, leaving gaps: `getAt(45)` with a `limit` + of 20 loads only page 3. + - Synchronous access (`operator []`, `loadedEntities`) **never fetches**; + only the `FutureOr` methods (`getAt`, `getPage`, `getRange`, + `loadNextPage`, `loadPage`, `loadAll`, `stream`) do. `operator []` returns + `null` for a gap, an unloaded page or an out-of-range index alike; use + `isPageLoaded` / `isIndexKnownOutOfRange` to tell them apart. + - It is deliberately **not** a `List` or an `Iterable`: both require a + `length`, which is exactly what a paginated select can't answer until it + reaches the end. Use `loadAll` when a complete list is really needed. + + What it knows: `loadedPages`, `loadedPagesLength`, `loadedEntities`, + `loadedEntitiesLength`, `maxLoadedPage`, `maxLoadedIndex`, `maxKnownPage`, + `isFinalPageResolved`, `finalPage`, `totalLength`, `isKnownEmpty`, + and `information()`. + + Since every page except the last holds exactly `limit` entries, identifying + the final page yields the total even with gaps: + `totalLength == (finalPage - 1) * limit + entries(finalPage)`. + The end resolves when a page comes back short, when an empty page has a + loaded and full predecessor, or when page 1 comes back empty. An empty page + *without* a loaded predecessor does **not** resolve it — jumping to page 50 + of a 3-page result only proves the end is somewhere before page 50 — but it + is still recorded, to avoid re-fetching that page or any page after it. + + Concurrent requests for the same page share a single fetch, and a failed + load is evicted so a retry actually retries. + +- New `paginateByQuery`, `paginate` and `paginateAll` on `EntitySource`, + `EntityRepository` (with `resolutionRules`) and `APIRepository`. They return + immediately without loading anything. `orderByID` defaults to `true` there, + rather than following the `offset != null` rule of the `select*` methods: + a paginated read is only meaningful over a stable order. + +- Note: each page is an independent select, without a shared `Transaction`. + Entries inserted or deleted between two page loads shift the offsets, so a + page loaded later can repeat or skip entries. This is inherent to + offset-based pagination; ordering by ID makes it as stable as it can be. + ## 1.11.0 - `selectByQuery` and its siblings gained 4 optional parameters, for pagination diff --git a/README.md b/README.md index c960e79..8c334c7 100644 --- a/README.md +++ b/README.md @@ -459,6 +459,42 @@ The `select*` methods accept `limit`, `offset`, `page`, `orderByID` and combined with an `offset`, if there is no positive `limit` to page by, or if it is below 1. +### Paginated reads: `EntityPagination` + +For reading a result page by page, `paginateByQuery` (and `paginate` / +`paginateAll`) returns an `EntityPagination`, which keeps the pages it has +already loaded: + +```dart +var page = accountRepository.paginateByQuery( + ' address.state == ? ', + parameters: ['NY'], + limit: 20, +); + +await page.loadNextPage(); // loads page 1 +page[0]; // synchronous: already loaded +page[45]; // null: not loaded (never fetches) + +await page.getAt(45); // loads page 3 on demand +page.loadedPages; // [1, 3] — page 2 is a gap +page.maxLoadedIndex; // 59 +page.totalLength; // null: the end is not known yet + +await page.loadAll(); // fills the gaps and resolves the end +page.totalLength; // 57 +page.finalPage; // 3 +``` + +Pages are 1-based and entry indexes 0-based. Synchronous access +(`operator []`, `loadedEntities`) never fetches — only the `FutureOr` methods +(`getAt`, `getPage`, `getRange`, `loadNextPage`, `loadAll`, `stream`) do. + +It is deliberately not a `List`: a paginated select does not know its length +until it reaches the end, so `totalLength` is `null` until the final page is +identified (`isFinalPageResolved`). Until then you still know +`maxLoadedIndex`, `maxKnownPage` and which pages are loaded. + The config file used above: File: `api-local.yaml` diff --git a/lib/src/bones_api_base.dart b/lib/src/bones_api_base.dart index 9953b6c..eb514fe 100644 --- a/lib/src/bones_api_base.dart +++ b/lib/src/bones_api_base.dart @@ -48,7 +48,7 @@ typedef APILogger = /// Bones API Library class. class BonesAPI { // ignore: constant_identifier_names - static const String VERSION = '1.11.0'; + static const String VERSION = '1.12.0'; static bool _boot = false; diff --git a/pubspec.yaml b/pubspec.yaml index f5c9c65..0ad23c7 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: bones_api description: Bones_API - A powerful API backend framework for Dart. It comes with a built-in HTTP Server, route handler, entity handler, SQL translator, and DB adapters. -version: 1.11.0 +version: 1.12.0 homepage: https://github.com/Colossus-Services/bones_api environment: