Skip to content

Commit d637725

Browse files
author
Samson Gebre
committed
feat: enhance SQL pagination handling with warnings for infinite loops and update tests
1 parent 3282b69 commit d637725

File tree

3 files changed

+31
-7
lines changed

3 files changed

+31
-7
lines changed

CHANGELOG.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
### Added
11+
- Batch API: `client.batch` namespace for deferred-execution batch operations that pack multiple Dataverse Web API calls into a single `POST $batch` HTTP request (#129)
12+
- Batch DataFrame integration: `client.batch.dataframe` namespace with pandas DataFrame wrappers for batch operations (#129)
13+
- `client.records.upsert()` and `client.batch.records.upsert()` backed by the `UpsertMultiple` bound action with alternate-key support (#129)
14+
- QueryBuilder: `client.query.builder("table")` with a fluent API, 20+ chainable methods (`select`, `filter_eq`, `filter_contains`, `order_by`, `expand`, etc.), and composable filter expressions using Python operators (`&`, `|`, `~`) (#118)
15+
- Memo/multiline column type support: `"memo"` (or `"multiline"`) can now be passed as a column type in `client.tables.create()` and `client.tables.add_columns()` (#155)
16+
17+
### Changed
18+
- Picklist label-to-integer resolution now uses a single bulk `PicklistAttributeMetadata` API call for the entire table instead of per-attribute requests, with a 1-hour TTL cache (#154)
19+
1020
### Fixed
1121
- `client.query.sql()` silently truncated results at 5,000 rows. The method now follows `@odata.nextLink` pagination and returns all matching rows (#157).
22+
- Alternate key fields were incorrectly merged into the `UpsertMultiple` request body, causing `400 Bad Request` on the create path (#129)
23+
- Docstring type annotations corrected for Microsoft Learn API reference compatibility (#153)
1224

1325
## [0.1.0b7] - 2026-03-17
1426

@@ -96,6 +108,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
96108
- Comprehensive error handling with specific exception types (`DataverseError`, `AuthenticationError`, etc.) (#22, #24)
97109
- HTTP retry logic with exponential backoff for resilient operations (#72)
98110

111+
[Unreleased]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b7...HEAD
99112
[0.1.0b7]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b6...v0.1.0b7
100113
[0.1.0b6]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b5...v0.1.0b6
101114
[0.1.0b5]: https://github.com/microsoft/PowerPlatform-DataverseClient-Python/compare/v0.1.0b4...v0.1.0b5

src/PowerPlatform/Dataverse/data/_odata.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -825,6 +825,15 @@ def _query_sql(self, sql: str) -> list[dict[str, Any]]:
825825
while next_link:
826826
# Guard 1: exact URL cycle (same next_link returned twice)
827827
if next_link in visited:
828+
warnings.warn(
829+
f"SQL pagination stopped after {len(results)} rows — "
830+
"the Dataverse server returned the same nextLink URL twice, "
831+
"indicating an infinite pagination cycle. "
832+
"Returning the rows collected so far. "
833+
"To avoid pagination entirely, add a TOP clause to your query.",
834+
RuntimeWarning,
835+
stacklevel=4,
836+
)
828837
break
829838
visited.add(next_link)
830839
# Guard 2: server-side bug where pagingcookie does not advance between

tests/unit/data/test_sql_parse.py

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -225,8 +225,9 @@ def test_query_sql_mid_pagination_error_warns_and_returns_partial():
225225
assert result == [{"id": 1}]
226226

227227

228-
def test_query_sql_repeated_next_link_stops_pagination():
229-
"""If the server keeps returning the same @odata.nextLink the loop must not run forever."""
228+
def test_query_sql_repeated_next_link_warns_and_stops():
229+
"""If the server keeps returning the same @odata.nextLink a RuntimeWarning is emitted and
230+
the loop stops without running forever."""
230231
client = _query_sql_client()
231232
# Both pages return the same next_link — simulates a server that re-executes the SQL
232233
repeating_body = {"value": [{"id": 1}], "@odata.nextLink": "https://org.example/page2"}
@@ -241,7 +242,8 @@ def test_query_sql_repeated_next_link_stops_pagination():
241242
patch.object(client, "_build_sql", return_value=MagicMock()),
242243
patch.object(client, "_request", return_value=resp2) as mock_req,
243244
):
244-
result = client._query_sql("SELECT id FROM account")
245+
with pytest.warns(RuntimeWarning, match="pagination stopped"):
246+
result = client._query_sql("SELECT id FROM account")
245247

246248
# fetched page2 once, then detected the cycle and stopped
247249
mock_req.assert_called_once_with("get", "https://org.example/page2")
@@ -343,11 +345,11 @@ def _make_next_link(pagingcookie_inner: str, pagenumber: int = 2) -> str:
343345
from urllib.parse import quote as _url_quote
344346

345347
skiptoken_xml = (
346-
f'<cookie pagenumber="{pagenumber}" '
347-
f'pagingcookie="{pagingcookie_inner}" '
348-
f'istracking="False" />'
348+
f'<cookie pagenumber="{pagenumber}" ' f'pagingcookie="{pagingcookie_inner}" ' f'istracking="False" />'
349+
)
350+
return (
351+
f"https://org.example/api/data/v9.2?$sql=SELECT%20name%20FROM%20account&$skiptoken={_url_quote(skiptoken_xml)}"
349352
)
350-
return f"https://org.example/api/data/v9.2?$sql=SELECT%20name%20FROM%20account&$skiptoken={_url_quote(skiptoken_xml)}"
351353

352354

353355
def test_extract_pagingcookie_returns_cookie_value():

0 commit comments

Comments
 (0)