Skip to content

feat: retry rate-limited requests and share pagination logic #218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pchuri merged 2 commits into from
Jul 19, 2026
Merged

feat: retry rate-limited requests and share pagination logic #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
61a14c8
feat: retry rate-limited requests and share pagination logic
pchuri Jul 19, 2026
9a422e0
fix: retry 503 responses only for read methods
pchuri Jul 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 106 additions & 89 deletions lib/confluence-client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ const { htmlToMarkdown, NAMED_ENTITIES } = require('./html-to-markdown');

const WRITE_FORMATS = ['auto', 'storage', 'html', 'markdown'];
const PAGE_LINK_LOOKUP_CONCURRENCY = 10;
const RETRY_SAFE_METHODS = new Set(['get', 'head', 'options']);
const MAX_RETRY_DELAY_MS = 60000;

const escapeXmlText = (value) => String(value)
.replace(/&/g, '&')
Expand Down Expand Up @@ -132,9 +134,32 @@ class ConfluenceClient {

this.client = axios.create(clientOptions);

this.maxRetries = 3;
this.retryBaseDelayMs = 1000;

this.client.interceptors.response.use(
response => response,
error => {
async error => {
const requestConfig = error.config;
const status = error.response?.status;
// 429 means the server rejected the request before processing, so
// any method is safe to replay. A 503 can arrive after a proxy or
// backend already applied a write, so only retry read methods.
const retryable = status === 429 ||
(status === 503 && RETRY_SAFE_METHODS.has((requestConfig?.method || '').toLowerCase()));
if (
requestConfig &&
retryable &&
// A stream body is consumed on first send and cannot be replayed
typeof requestConfig.data?.pipe !== 'function'
) {
const attempt = requestConfig.__retryCount || 0;
if (attempt < this.maxRetries) {
requestConfig.__retryCount = attempt + 1;
await this.sleep(this.retryDelayMs(error.response, attempt));
return this.client.request(requestConfig);
}
}
if (error.response?.status === 401) {
const hints = ['Authentication failed (401 Unauthorized).'];
if (this.isScopedToken()) {
Expand Down Expand Up @@ -562,26 +587,14 @@ class ConfluenceClient {
const cap = maxResults === null || maxResults === undefined
? null
: this.parsePositiveInt(maxResults, 500);
const pageSize = this.parsePositiveInt(options.pageSize, 500);
let start = this.parsePositiveInt(options.start, 0);
const spaces = [];

let hasNext = true;
while (hasNext) {
if (cap !== null && spaces.length >= cap) break;
const requestLimit = cap === null
? pageSize
: Math.min(pageSize, cap - spaces.length);
const page = await this.listSpaces({ limit: requestLimit, start });
spaces.push(...page.results);

hasNext = page.nextStart !== null && page.nextStart !== undefined;
if (hasNext) {
start = page.nextStart;
return this.collectPaginated(
({ limit, start }) => this.listSpaces({ limit, start }),
{
maxResults: cap,
pageSize: this.parsePositiveInt(options.pageSize, 500),
start: this.parsePositiveInt(options.start, 0)
}
}

return cap !== null ? spaces.slice(0, cap) : spaces;
);
}

/**
Expand Down Expand Up @@ -895,34 +908,21 @@ class ConfluenceClient {
* Fetch all comments for a page, honoring an optional maxResults cap
*/
async getAllComments(pageIdOrUrl, options = {}) {
const pageSize = this.parsePositiveInt(options.pageSize || options.limit, 25);
const maxResults = this.parsePositiveInt(options.maxResults, null);
let start = this.parsePositiveInt(options.start, 0);
const comments = [];

let hasNext = true;
while (hasNext) {
const page = await this.listComments(pageIdOrUrl, {
limit: pageSize,
return this.collectPaginated(
({ limit, start }) => this.listComments(pageIdOrUrl, {
limit,
start,
expand: options.expand,
location: options.location,
depth: options.depth,
parentVersion: options.parentVersion
});
comments.push(...page.results);

if (maxResults && comments.length >= maxResults) {
return comments.slice(0, maxResults);
}

hasNext = page.nextStart !== null && page.nextStart !== undefined;
if (hasNext) {
start = page.nextStart;
}),
{
maxResults: this.parsePositiveInt(options.maxResults, null) || null,
pageSize: this.parsePositiveInt(options.pageSize || options.limit, 25),
start: this.parsePositiveInt(options.start, 0)
}
}

return comments;
);
}

normalizeComment(raw) {
Expand Down Expand Up @@ -1111,32 +1111,18 @@ class ConfluenceClient {
* Fetch all attachments for a page, honoring an optional maxResults cap
*/
async getAllAttachments(pageIdOrUrl, options = {}) {
const pageSize = this.parsePositiveInt(options.pageSize || options.limit, 50);
const maxResults = this.parsePositiveInt(options.maxResults, null);
const filename = options.filename;
let start = this.parsePositiveInt(options.start, 0);
const attachments = [];

let hasNext = true;
while (hasNext) {
const page = await this.listAttachments(pageIdOrUrl, {
limit: pageSize,
return this.collectPaginated(
({ limit, start }) => this.listAttachments(pageIdOrUrl, {
limit,
start,
filename
});
attachments.push(...page.results);

if (maxResults && attachments.length >= maxResults) {
return attachments.slice(0, maxResults);
}

hasNext = page.nextStart !== null && page.nextStart !== undefined;
if (hasNext) {
start = page.nextStart;
filename: options.filename
}),
{
maxResults: this.parsePositiveInt(options.maxResults, null) || null,
pageSize: this.parsePositiveInt(options.pageSize || options.limit, 50),
start: this.parsePositiveInt(options.start, 0)
}
}

return attachments;
);
}

/**
Expand Down Expand Up @@ -1287,30 +1273,14 @@ class ConfluenceClient {
* Fetch all content properties for a page, honoring an optional maxResults cap
*/
async getAllProperties(pageIdOrUrl, options = {}) {
const pageSize = this.parsePositiveInt(options.pageSize || options.limit, 25);
const maxResults = this.parsePositiveInt(options.maxResults, null);
let start = this.parsePositiveInt(options.start, 0);
const properties = [];

let hasNext = true;
while (hasNext) {
const page = await this.listProperties(pageIdOrUrl, {
limit: pageSize,
start
});
properties.push(...page.results);

if (maxResults && properties.length >= maxResults) {
return properties.slice(0, maxResults);
}

hasNext = page.nextStart !== null && page.nextStart !== undefined;
if (hasNext) {
start = page.nextStart;
return this.collectPaginated(
({ limit, start }) => this.listProperties(pageIdOrUrl, { limit, start }),
{
maxResults: this.parsePositiveInt(options.maxResults, null) || null,
pageSize: this.parsePositiveInt(options.pageSize || options.limit, 25),
start: this.parsePositiveInt(options.start, 0)
}
}

return properties;
);
}

/**
Expand Down Expand Up @@ -2228,6 +2198,53 @@ class ConfluenceClient {
return parsed;
}

sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}

/**
* Delay before retrying a throttled request: honor Retry-After
* (seconds or HTTP-date), otherwise exponential backoff.
*/
retryDelayMs(response, attempt) {
const retryAfter = response?.headers?.['retry-after'];
if (retryAfter !== undefined && retryAfter !== null && retryAfter !== '') {
const seconds = Number(retryAfter);
if (Number.isFinite(seconds) && seconds >= 0) {
return Math.min(seconds * 1000, MAX_RETRY_DELAY_MS);
}
const dateMs = Date.parse(retryAfter);
if (!Number.isNaN(dateMs)) {
return Math.min(Math.max(dateMs - Date.now(), 0), MAX_RETRY_DELAY_MS);
}
}
return Math.min(this.retryBaseDelayMs * 2 ** attempt, MAX_RETRY_DELAY_MS);
}

/**
* Accumulate results across start/limit pages until the server stops
* returning a next link or `maxResults` is reached (null = unlimited).
* `fetchPage({ limit, start })` must resolve to `{ results, nextStart }`.
*/
async collectPaginated(fetchPage, { maxResults = null, pageSize, start = 0 }) {
const items = [];
let hasNext = true;
while (hasNext) {
if (maxResults !== null && items.length >= maxResults) break;
const limit = maxResults === null
? pageSize
: Math.min(pageSize, maxResults - items.length);
const page = await fetchPage({ limit, start });
items.push(...page.results);

hasNext = page.nextStart !== null && page.nextStart !== undefined;
if (hasNext) {
start = page.nextStart;
}
}
return maxResults !== null ? items.slice(0, maxResults) : items;
}

async rawRequest(method, endpoint, options = {}) {
const { headers = {}, params, data } = options;

Expand Down
117 changes: 117 additions & 0 deletions tests/confluence-client.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,123 @@ describe('ConfluenceClient', () => {
});
});

describe('rate-limit retry (429/503)', () => {
const makeClient = () => {
const retryClient = new ConfluenceClient({
domain: 'confluence.company.com',
token: 'test-token',
apiPath: '/rest/api'
});
retryClient.retryBaseDelayMs = 1;
return retryClient;
};

test('retries a 429 response and succeeds', async () => {
const retryClient = makeClient();
const mock = new MockAdapter(retryClient.client);
mock
.onGet(/\/content\/123/).replyOnce(429, {}, { 'retry-after': '0' })
.onGet(/\/content\/123/).replyOnce(200, {
body: { storage: { value: '<p>hi</p>' } }
});

await expect(retryClient.readPage('123', 'storage')).resolves.toBe('<p>hi</p>');
expect(mock.history.get.length).toBe(2);
mock.restore();
});

test('retries a 503 response and succeeds', async () => {
const retryClient = makeClient();
const mock = new MockAdapter(retryClient.client);
mock
.onGet(/\/content\/123/).replyOnce(503)
.onGet(/\/content\/123/).replyOnce(200, {
body: { storage: { value: '<p>hi</p>' } }
});

await expect(retryClient.readPage('123', 'storage')).resolves.toBe('<p>hi</p>');
expect(mock.history.get.length).toBe(2);
mock.restore();
});

test('retries a 429 write request (rejected before processing)', async () => {
const retryClient = makeClient();
const mock = new MockAdapter(retryClient.client);
mock
.onPost('/content').replyOnce(429, {}, { 'retry-after': '0' })
.onPost('/content').replyOnce(200, { id: '456' });

const response = await retryClient.client.post('/content', { title: 'New page' });
expect(response.data.id).toBe('456');
expect(mock.history.post.length).toBe(2);
mock.restore();
});

test('does not retry a 503 write request (may already be applied)', async () => {
const retryClient = makeClient();
const mock = new MockAdapter(retryClient.client);
mock.onPost('/content').reply(503);

await expect(
retryClient.client.post('/content', { title: 'New page' })
).rejects.toThrow(/503/);
expect(mock.history.post.length).toBe(1);
mock.restore();
});

test('gives up after maxRetries attempts', async () => {
const retryClient = makeClient();
retryClient.maxRetries = 2;
const mock = new MockAdapter(retryClient.client);
mock.onGet(/\/content\/123/).reply(429);

await expect(retryClient.readPage('123')).rejects.toThrow(/429/);
expect(mock.history.get.length).toBe(3);
mock.restore();
});

test('does not retry non-retryable errors', async () => {
const retryClient = makeClient();
const mock = new MockAdapter(retryClient.client);
mock.onGet(/\/content\/123/).reply(400);

await expect(retryClient.readPage('123')).rejects.toThrow(/400/);
expect(mock.history.get.length).toBe(1);
mock.restore();
});

test('does not retry requests with a stream body', async () => {
const retryClient = makeClient();
const mock = new MockAdapter(retryClient.client);
mock.onPost('/upload').reply(429);

const streamLike = { pipe: () => {} };
await expect(
retryClient.client.request({ url: '/upload', method: 'post', data: streamLike })
).rejects.toThrow(/429/);
expect(mock.history.post.length).toBe(1);
mock.restore();
});

test('retryDelayMs honors Retry-After seconds, dates, and backoff', () => {
const retryClient = makeClient();
retryClient.retryBaseDelayMs = 1000;

expect(retryClient.retryDelayMs({ headers: { 'retry-after': '2' } }, 0)).toBe(2000);
expect(retryClient.retryDelayMs({ headers: { 'retry-after': '120' } }, 0)).toBe(60000);

const soon = new Date(Date.now() + 5000).toUTCString();
const dateDelay = retryClient.retryDelayMs({ headers: { 'retry-after': soon } }, 0);
expect(dateDelay).toBeGreaterThan(0);
expect(dateDelay).toBeLessThanOrEqual(5000);

expect(retryClient.retryDelayMs({ headers: {} }, 0)).toBe(1000);
expect(retryClient.retryDelayMs({ headers: {} }, 1)).toBe(2000);
expect(retryClient.retryDelayMs({ headers: {} }, 2)).toBe(4000);
expect(retryClient.retryDelayMs(undefined, 0)).toBe(1000);
});
});

describe('401 error handling (cookie auth)', () => {
test('provides cookie-specific hints for cookie auth', async () => {
const cookieClient = new ConfluenceClient({
Expand Down