From 68fde82a92534f60d9f4b63bda66cbcab838985a Mon Sep 17 00:00:00 2001 From: BART! Date: Thu, 9 Jul 2026 13:05:04 +0200 Subject: [PATCH 1/6] chore(yarn): update yarn --- .yarnrc.yml | 2 ++ package.json | 2 +- yarn.lock | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.yarnrc.yml b/.yarnrc.yml index dc61281..063ed60 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -4,3 +4,5 @@ approvedGitRepositories: enableScripts: true nodeLinker: node-modules + +npmMinimalAgeGate: 0 diff --git a/package.json b/package.json index bfdaa5d..23cf078 100644 --- a/package.json +++ b/package.json @@ -126,5 +126,5 @@ "node-csfd-api": "./dist/cli.js" }, "sideEffects": false, - "packageManager": "yarn@4.14.1" + "packageManager": "yarn@4.17.1" } diff --git a/yarn.lock b/yarn.lock index 2cc93c2..fa1b47f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # Manual changes might be lost - proceed with caution! __metadata: - version: 9 + version: 10 cacheKey: 10c0 "@babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": From 4c470b54fb317cb4fba43e43b57e745cf9189820 Mon Sep 17 00:00:00 2001 From: BART! Date: Mon, 3 Aug 2026 19:11:48 +0200 Subject: [PATCH 2/6] fix(anubis): resolve challenge --- src/anubis/challenge.ts | 159 ++++++++++++++++++ src/anubis/client.ts | 81 ++++++++++ src/anubis/index.ts | 14 ++ src/anubis/proof-of-work.ts | 68 ++++++++ src/anubis/sha256.ts | 122 ++++++++++++++ src/fetchers/index.ts | 82 +++++++--- src/index.ts | 1 + tests/anubis.test.ts | 314 ++++++++++++++++++++++++++++++++++++ 8 files changed, 823 insertions(+), 18 deletions(-) create mode 100644 src/anubis/challenge.ts create mode 100644 src/anubis/client.ts create mode 100644 src/anubis/index.ts create mode 100644 src/anubis/proof-of-work.ts create mode 100644 src/anubis/sha256.ts create mode 100644 tests/anubis.test.ts diff --git a/src/anubis/challenge.ts b/src/anubis/challenge.ts new file mode 100644 index 0000000..03b4eb7 --- /dev/null +++ b/src/anubis/challenge.ts @@ -0,0 +1,159 @@ +import { DEFAULT_TIME_BUDGET_MS, solveProofOfWork } from './proof-of-work'; + +// Anubis (BotStopper by Techaro) is a proof-of-work anti-bot interstitial: +// instead of the page it serves an HTML challenge that a browser solves in +// JavaScript. This module replicates the protocol so a plain `fetch` can earn +// the auth cookie. See: https://github.com/TecharoHQ/anubis + +const AUTH_COOKIE_NAME = 'techaro.lol-anubis-auth'; +const VERIFY_COOKIE_NAME = 'techaro.lol-anubis-cookie-verification'; +const PASS_CHALLENGE_PATH = '/.within.website/x/cmd/anubis/api/pass-challenge'; + +// Anubis also ships non-hashing challenge methods (metarefresh, preact). Only +// its SHA-256 ones are replicated here; anything else must fail immediately +// rather than burn the whole time budget computing a hash nobody asked for. +const SUPPORTED_ALGORITHMS = ['fast', 'slow']; + +// Structural markers, deliberately not the localised body text: the +// interstitial is translated, so matching prose would both miss locales and +// risk false positives on user-generated content that quotes it. +const CHALLENGE_MARKERS = ['id="anubis_challenge"', '/.within.website/x/cmd/anubis/']; + +export type FetchLike = (input: string, init?: RequestInit) => Promise; + +interface ParsedChallenge { + rules: { algorithm: string; difficulty: number }; + challenge: { id: string; randomData: string }; +} + +export interface ChallengeResult { + /** Cookie to replay, or `null` when the runtime's own cookie jar holds it. */ + cookie: string | null; + /** True when Set-Cookie was hidden and the runtime now owns the cookie. */ + platformCookieJar: boolean; +} + +export interface PassChallengeParams { + /** Body of the interstitial page that was served instead of the content. */ + html: string; + /** Response headers that came with it, carrying the verification cookie. */ + headers: Headers; + /** The URL that was blocked; used as the redirect target after passing. */ + url: string; + /** Headers to reuse, so the exchange looks like the original request. */ + requestHeaders?: Headers; + fetch: FetchLike; + timeBudgetMs?: number; +} + +export const isAnubisChallenge = (html: string): boolean => + CHALLENGE_MARKERS.some((marker) => html.includes(marker)); + +const parseChallenge = (html: string): ParsedChallenge | null => { + const match = html.match( + /`; + + // A readable Set-Cookie keeps us on the manual (Node) path, so these bail + // before any request — a fetch that throws proves the network is never hit. + const nodeHeaders = () => + new Headers({ 'set-cookie': 'techaro.lol-anubis-cookie-verification=test-id; Path=/' }); + const forbiddenFetch = () => Promise.reject(new Error('must not reach the network')); + + const attempt = (html: string) => + passChallenge({ + html, + headers: nodeHeaders(), + url: 'https://www.csfd.cz/film/1/', + fetch: forbiddenFetch + }); + + test('refuses a challenge method it cannot compute', async () => { + await expect(attempt(challengePage('metarefresh'))).resolves.toBeNull(); + }); + + test('accepts the methods it does implement', async () => { + // Reaching the network means the guard let it through to the exchange. + await expect(attempt(challengePage('fast'))).rejects.toThrow('must not reach the network'); + }); + + test('refuses an unparseable challenge', async () => { + await expect( + attempt('') + ).resolves.toBeNull(); + }); +}); + +// The live site only challenges a fraction of requests, so a green integration +// run can miss the solver entirely. This exercises the whole exchange against a +// stand-in server that validates the proof exactly as Anubis does, offline. +describe('Anubis: challenge exchange', () => { + const CHALLENGE_ID = '019f8462-2dd6-7755-9d82-de2c14c2d59e'; + const RANDOM_DATA = 'a3f1c908'.repeat(16); + const DIFFICULTY = 2; + const PAGE_URL = 'https://example.test/protected/'; + const PASS_PATH = '/.within.website/x/cmd/anubis/api/pass-challenge'; + const AUTH_COOKIE = 'techaro.lol-anubis-auth=header.payload.signature'; + const VERIFY_COOKIE = `techaro.lol-anubis-cookie-verification=${CHALLENGE_ID}`; + + const interstitial = (algorithm = 'fast') => + ``; + + const withSetCookie = (...cookies: string[]): Headers => { + const headers = new Headers(); + cookies.forEach((cookie) => headers.append('set-cookie', cookie)); + return headers; + }; + + /** + * Stands in for Anubis. `hidesSetCookie` mimics a browser or React Native, + * where Set-Cookie is stripped from what scripts may read. + */ + const fakeAnubis = ({ hidesSetCookie = false } = {}) => { + const calls: { url: URL; init?: RequestInit }[] = []; + + const fetch = async (input: string, init?: RequestInit): Promise => { + const url = new URL(input); + calls.push({ url, init }); + + if (url.pathname !== PASS_PATH) { + return new Response(interstitial(), { + headers: hidesSetCookie ? new Headers() : withSetCookie(VERIFY_COOKIE) + }); + } + + // Validate the submitted proof the way the real server would. + const nonce = url.searchParams.get('nonce') ?? ''; + const claimed = url.searchParams.get('response') ?? ''; + const digest = createHash('sha256').update(RANDOM_DATA + nonce).digest('hex'); + const provenWork = claimed === digest && digest.startsWith('0'.repeat(DIFFICULTY)); + const rightChallenge = url.searchParams.get('id') === CHALLENGE_ID; + const provenCookies = + hidesSetCookie || new Headers(init?.headers).get('Cookie') === VERIFY_COOKIE; + + if (!provenWork || !rightChallenge || !provenCookies) { + return new Response('challenge failed', { status: 403 }); + } + return new Response(null, { + status: 302, + headers: hidesSetCookie ? new Headers({ location: PAGE_URL }) : withSetCookie(AUTH_COOKIE) + }); + }; + + return { fetch, calls, passCalls: () => calls.filter((c) => c.url.pathname === PASS_PATH) }; + }; + + test('earns and caches a cookie the server accepts', async () => { + const server = fakeAnubis(); + const client = createAnubisClient({ fetch: server.fetch }); + + // The server only issues the cookie if the proof verifies, so this passing + // means our hash construction still matches Anubis' own. + expect(await client.pass(interstitial(), withSetCookie(VERIFY_COOKIE), PAGE_URL)).toBe(true); + expect(client.getCookie()).toBe(AUTH_COOKIE); + expect(client.usesPlatformCookieJar()).toBe(false); + }); + + test('submits the parameters Anubis expects', async () => { + const server = fakeAnubis(); + const client = createAnubisClient({ fetch: server.fetch }); + await client.pass(interstitial(), withSetCookie(VERIFY_COOKIE), PAGE_URL); + + const [call] = server.passCalls(); + expect(call.url.searchParams.get('id')).toBe(CHALLENGE_ID); + expect(call.url.searchParams.get('redir')).toBe(PAGE_URL); + expect(Number(call.url.searchParams.get('elapsedTime'))).toBeGreaterThanOrEqual(0); + expect(new Headers(call.init?.headers).get('Cookie')).toBe(VERIFY_COOKIE); + // Following the 302 would discard the Set-Cookie we came for. + expect(call.init?.redirect).toBe('manual'); + }); + + test('keeps no cookie when the server rejects the proof', async () => { + const client = createAnubisClient({ + fetch: async () => new Response('challenge failed', { status: 403 }) + }); + + expect(await client.pass(interstitial(), withSetCookie(VERIFY_COOKIE), PAGE_URL)).toBe(false); + expect(client.getCookie()).toBeNull(); + }); + + test('refuses a challenge method it cannot compute without asking the server', async () => { + const server = fakeAnubis(); + const client = createAnubisClient({ fetch: server.fetch }); + + expect( + await client.pass(interstitial('metarefresh'), withSetCookie(VERIFY_COOKIE), PAGE_URL) + ).toBe(false); + expect(server.calls).toHaveLength(0); + }); + + test('concurrent callers share a single proof-of-work', async () => { + const server = fakeAnubis(); + const client = createAnubisClient({ fetch: server.fetch }); + + const results = await Promise.all( + Array.from({ length: 4 }, () => + client.pass(interstitial(), withSetCookie(VERIFY_COOKIE), PAGE_URL) + ) + ); + + expect(results).toEqual([true, true, true, true]); + expect(server.passCalls()).toHaveLength(1); + }); + + test('hands the cookie to the runtime jar when Set-Cookie is hidden', async () => { + const server = fakeAnubis({ hidesSetCookie: true }); + const client = createAnubisClient({ fetch: server.fetch }); + + expect(await client.pass(interstitial(), new Headers(), PAGE_URL)).toBe(true); + expect(client.getCookie()).toBeNull(); + expect(client.usesPlatformCookieJar()).toBe(true); + + // The challenge is re-requested with credentials so the jar can store it. + const reissue = server.calls.find((call) => call.url.pathname !== PASS_PATH); + expect(reissue?.init?.credentials).toBe('include'); + }); +}); + +describe('Anubis: client', () => { + test('holds and clears its cookie independently per instance', () => { + const a = createAnubisClient(); + const b = createAnubisClient(); + + a.setCookie('techaro.lol-anubis-auth=token-a'); + expect(a.getCookie()).toBe('techaro.lol-anubis-auth=token-a'); + expect(b.getCookie()).toBeNull(); + + a.reset(); + expect(a.getCookie()).toBeNull(); + }); + + test('assumes manual cookie handling until told otherwise', () => { + expect(createAnubisClient().usesPlatformCookieJar()).toBe(false); + }); +}); + +describe('Anubis: isAnubisChallenge', () => { + test('detects the embedded challenge script', () => { + expect( + isAnubisChallenge('') + ).toBe(true); + }); + + test('detects the interstitial by its asset paths', () => { + expect( + isAnubisChallenge('') + ).toBe(true); + }); + + // The library serves cs/en/sk, so detection must not hinge on localised prose. + test.each([ + ['Czech', 'Ujišťujeme se, že nejste robot!'], + ['English', "Making sure you're not a bot!"] + ])('detects the %s interstitial via its structure', (_locale, title) => { + const page = `${title}`; + expect(isAnubisChallenge(page)).toBe(true); + }); + + test('returns false for a normal page', () => { + expect(isAnubisChallenge('

BART!

')).toBe(false); + }); + + // A review quoting the interstitial must not be mistaken for a block. + test('does not false-positive on user content mentioning the challenge', () => { + expect(isAnubisChallenge('

Hláška "Ujišťujeme se, že nejste robot!" mě dostala

')).toBe( + false + ); + }); +}); From ec1fc093c9da9a1fb28738a4a3adc368f7a12d7d Mon Sep 17 00:00:00 2001 From: BART! Date: Mon, 3 Aug 2026 19:25:49 +0200 Subject: [PATCH 3/6] chore(deps): update yarn --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 23cf078..92e1d53 100644 --- a/package.json +++ b/package.json @@ -126,5 +126,5 @@ "node-csfd-api": "./dist/cli.js" }, "sideEffects": false, - "packageManager": "yarn@4.17.1" + "packageManager": "yarn@4.18.0" } From d6c8cba80570a6030fdf3cd4f8692f5113faf33b Mon Sep 17 00:00:00 2001 From: BART! Date: Mon, 3 Aug 2026 20:08:46 +0200 Subject: [PATCH 4/6] feat(anubis): verification --- src/anubis/challenge.ts | 150 +++++++++++++++++--- tests/anubis.test.ts | 299 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 405 insertions(+), 44 deletions(-) diff --git a/src/anubis/challenge.ts b/src/anubis/challenge.ts index 03b4eb7..a142b8d 100644 --- a/src/anubis/challenge.ts +++ b/src/anubis/challenge.ts @@ -9,10 +9,17 @@ const AUTH_COOKIE_NAME = 'techaro.lol-anubis-auth'; const VERIFY_COOKIE_NAME = 'techaro.lol-anubis-cookie-verification'; const PASS_CHALLENGE_PATH = '/.within.website/x/cmd/anubis/api/pass-challenge'; -// Anubis also ships non-hashing challenge methods (metarefresh, preact). Only -// its SHA-256 ones are replicated here; anything else must fail immediately -// rather than burn the whole time budget computing a hash nobody asked for. -const SUPPORTED_ALGORITHMS = ['fast', 'slow']; +// Anubis picks a challenge method per request. The SHA-256 ones make the client +// burn CPU; `metarefresh` instead makes it sit out a declared delay. Anything +// else (e.g. `preact`) needs a real JS runtime and must fail immediately rather +// than burn the whole time budget computing a hash nobody asked for. +const PROOF_OF_WORK_ALGORITHMS = ['fast', 'slow']; +const METAREFRESH_ALGORITHM = 'metarefresh'; + +// Anubis states the wait in a `Refresh` header or its `` twin. This is +// only the fallback for a page that omits both, where the exchange URL has to +// be rebuilt from the challenge anyway. +const DEFAULT_METAREFRESH_DELAY_MS = 2000; // Structural markers, deliberately not the localised body text: the // interstitial is translated, so matching prose would both miss locales and @@ -80,6 +87,88 @@ const readCookie = (headers: Headers, name: string): string | null => { const hidesSetCookie = (headers: Headers): boolean => typeof headers.getSetCookie !== 'function' || headers.getSetCookie().length === 0; +const REFRESH_HEADER_DIRECTIVE = /^\s*(\d+)\s*;\s*url=(.+)$/i; +const REFRESH_META_DIRECTIVE = + /]+http-equiv=["']?refresh["']?[^>]*content=["'](\d+)[^;]*;\s*url=([^"'>]+)/i; + +interface RefreshDirective { + delayMs: number; + url: string; +} + +/** + * The `; url=` directive Anubis serves with a metarefresh + * challenge. It arrives as a `Refresh` header on some responses and as its + * `` equivalent on others, so both are read. + */ +const readRefreshDirective = (html: string, headers: Headers): RefreshDirective | null => { + const directive = + headers.get('refresh')?.match(REFRESH_HEADER_DIRECTIVE) ?? html.match(REFRESH_META_DIRECTIVE); + if (!directive) { + return null; + } + return { + delayMs: Number(directive[1]) * 1000, + // Inside a meta attribute the query separators arrive HTML-escaped. + url: directive[2].trim().replace(/&/g, '&') + }; +}; + +const wait = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +const proofOfWorkPassUrl = async ( + url: string, + { id, randomData }: ParsedChallenge['challenge'], + difficulty: number, + timeBudgetMs: number +): Promise => { + const startedAt = Date.now(); + const solution = await solveProofOfWork(randomData, difficulty, timeBudgetMs); + if (!solution) { + return null; + } + + const passUrl = new URL(PASS_CHALLENGE_PATH, url); + passUrl.searchParams.set('id', id); + passUrl.searchParams.set('response', solution.hash); + passUrl.searchParams.set('nonce', String(solution.nonce)); + passUrl.searchParams.set('redir', url); + passUrl.searchParams.set('elapsedTime', String(Date.now() - startedAt)); + return passUrl.toString(); +}; + +/** + * Metarefresh asks for patience rather than hashes: Anubis hands over the + * exchange URL up front but answers it with 403 until the delay it declared has + * actually elapsed, so the wait is the whole proof. + */ +const metarefreshPassUrl = async ( + url: string, + { id, randomData }: ParsedChallenge['challenge'], + directive: RefreshDirective | null, + timeBudgetMs: number +): Promise => { + const delayMs = directive?.delayMs ?? DEFAULT_METAREFRESH_DELAY_MS; + // Waiting longer than the caller allowed is worse than not passing at all. + if (delayMs > timeBudgetMs) { + return null; + } + + // Anubis' own URL is authoritative, so it is preferred over rebuilding one. + let passUrl: URL; + if (directive) { + passUrl = new URL(directive.url, url); + } else { + passUrl = new URL(PASS_CHALLENGE_PATH, url); + passUrl.searchParams.set('challenge', randomData); + passUrl.searchParams.set('id', id); + passUrl.searchParams.set('redir', url); + } + + await wait(delayMs); + return passUrl.toString(); +}; + /** * Solve the challenge on an interstitial page and exchange it for an Anubis * auth cookie. Returns `null` if the challenge could not be passed. @@ -111,24 +200,26 @@ export const passChallenge = async ({ } const parsed = parseChallenge(html); - if (!parsed || !SUPPORTED_ALGORITHMS.includes(parsed.rules.algorithm)) { + if (!parsed) { return null; } const { challenge, rules } = parsed; - const startedAt = Date.now(); - const solution = await solveProofOfWork(challenge.randomData, rules.difficulty, timeBudgetMs); - if (!solution) { + let passUrl: string | null = null; + if (PROOF_OF_WORK_ALGORITHMS.includes(rules.algorithm)) { + passUrl = await proofOfWorkPassUrl(url, challenge, rules.difficulty, timeBudgetMs); + } else if (rules.algorithm === METAREFRESH_ALGORITHM) { + passUrl = await metarefreshPassUrl( + url, + challenge, + readRefreshDirective(html, headers), + timeBudgetMs + ); + } + if (!passUrl) { return null; } - const passUrl = new URL(PASS_CHALLENGE_PATH, url); - passUrl.searchParams.set('id', challenge.id); - passUrl.searchParams.set('response', solution.hash); - passUrl.searchParams.set('nonce', String(solution.nonce)); - passUrl.searchParams.set('redir', url); - passUrl.searchParams.set('elapsedTime', String(Date.now() - startedAt)); - // Anubis requires the verification cookie it set on the interstitial as proof // that cookies work; on a jar runtime the runtime itself attaches it. const passHeaders = new Headers(requestHeaders); @@ -139,7 +230,7 @@ export const passChallenge = async ({ // `redirect: 'manual'` stops fetch from following the 302 to `redir`, which // would discard the Set-Cookie we need to read off this very response. - const response = await fetch(passUrl.toString(), { + const response = await fetch(passUrl, { method: 'GET', credentials: platformCookieJar ? 'include' : 'omit', redirect: 'manual', @@ -151,9 +242,26 @@ export const passChallenge = async ({ return { cookie: authCookie, platformCookieJar }; } - // Nothing to read: either the exchange failed, or we are on a runtime that - // hides Set-Cookie and has already stored the cookie itself. A 302 to `redir` - // (status 0 / opaqueredirect in browsers) is Anubis' success signal. - const passed = response.status === 302 || response.type === 'opaqueredirect'; - return passed && !verifyCookie ? { cookie: null, platformCookieJar } : null; + // No cookie in hand. A runtime that lets us read Set-Cookie would have shown + // it, so this is a failed exchange; only a jar runtime can have passed while + // keeping the cookie to itself. + if (!platformCookieJar) { + return null; + } + + // A 302 to `redir` is Anubis' success signal, and the jar has just stored the + // cookie off it. Browsers report the unfollowed redirect as `opaqueredirect`. + if (response.status === 302 || response.type === 'opaqueredirect') { + return { cookie: null, platformCookieJar }; + } + + // React Native ignores `redirect: 'manual'` and follows the 302 itself, so + // what we hold is the page we were after — proof enough, unless Anubis is + // still challenging us. + if (response.ok) { + const body = await response.text(); + return body && !isAnubisChallenge(body) ? { cookie: null, platformCookieJar } : null; + } + + return null; }; diff --git a/tests/anubis.test.ts b/tests/anubis.test.ts index 51acb87..78b5fd7 100644 --- a/tests/anubis.test.ts +++ b/tests/anubis.test.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; import { createAnubisClient, isAnubisChallenge, @@ -9,6 +9,13 @@ import { toHex } from '../src/anubis'; +/** Node exposes Set-Cookie; browsers and React Native hide it behind their jar. */ +const withSetCookie = (...cookies: string[]): Headers => { + const headers = new Headers(); + cookies.forEach((cookie) => headers.append('set-cookie', cookie)); + return headers; +}; + // The pure-JS SHA-256 is what makes the solver portable (Node/browser/RN). Pin // it to the NIST vectors and cross-check it against node:crypto (available here // as an independent oracle, though never imported by the library itself). @@ -24,7 +31,14 @@ describe('sha256 (portable pure-JS)', () => { test('matches node:crypto across block boundaries', () => { // 55/56/64 bytes exercise the padding edge cases (one vs two blocks). - for (const input of ['', 'a', 'a'.repeat(55), 'a'.repeat(56), 'a'.repeat(64), 'x'.repeat(200)]) { + for (const input of [ + '', + 'a', + 'a'.repeat(55), + 'a'.repeat(56), + 'a'.repeat(64), + 'x'.repeat(200) + ]) { expect(toHex(sha256(input))).toBe(createHash('sha256').update(input).digest('hex')); } }); @@ -45,10 +59,26 @@ describe('Anubis: solveProofOfWork', () => { // Precomputed vectors: the solver iterates nonce from 0 upward, so the first // valid nonce for a given (data, difficulty) is deterministic. const vectors = [ - { difficulty: 1, nonce: 8, hash: '0ca5e234ebe9ed5341e35a02c4ab2d44860cf7c8084c76a0e2b9496536b26554' }, - { difficulty: 2, nonce: 1048, hash: '006fe24fe34772b34e103be62bf4d75718b57c5d87f2e1b83f61709eae986c00' }, - { difficulty: 3, nonce: 8623, hash: '0003be1161d05bf661f70b314a4241b7b80fe851d274db557fec72ad56cfe35e' }, - { difficulty: 4, nonce: 10148, hash: '0000b951efd5c69a4d9fca8263c0c59a7612fc3db11b5a618ab498f5d0be4d3c' } + { + difficulty: 1, + nonce: 8, + hash: '0ca5e234ebe9ed5341e35a02c4ab2d44860cf7c8084c76a0e2b9496536b26554' + }, + { + difficulty: 2, + nonce: 1048, + hash: '006fe24fe34772b34e103be62bf4d75718b57c5d87f2e1b83f61709eae986c00' + }, + { + difficulty: 3, + nonce: 8623, + hash: '0003be1161d05bf661f70b314a4241b7b80fe851d274db557fec72ad56cfe35e' + }, + { + difficulty: 4, + nonce: 10148, + hash: '0000b951efd5c69a4d9fca8263c0c59a7612fc3db11b5a618ab498f5d0be4d3c' + } ]; test.each(vectors)( @@ -58,12 +88,19 @@ describe('Anubis: solveProofOfWork', () => { } ); - test.each(vectors)('the digest actually satisfies difficulty $difficulty', ({ difficulty, nonce, hash }) => { - // difficulty N leading zero nibbles == N leading '0' hex characters - expect(hash.startsWith('0'.repeat(difficulty))).toBe(true); - // and the hash is genuinely sha256(data + nonce) - expect(createHash('sha256').update(DATA + nonce).digest('hex')).toBe(hash); - }); + test.each(vectors)( + 'the digest actually satisfies difficulty $difficulty', + ({ difficulty, nonce, hash }) => { + // difficulty N leading zero nibbles == N leading '0' hex characters + expect(hash.startsWith('0'.repeat(difficulty))).toBe(true); + // and the hash is genuinely sha256(data + nonce) + expect( + createHash('sha256') + .update(DATA + nonce) + .digest('hex') + ).toBe(hash); + } + ); test('difficulty 0 is solved immediately by nonce 0', async () => { const result = await solveProofOfWork(DATA, 0); @@ -111,8 +148,9 @@ describe('Anubis: passChallenge guards', () => { fetch: forbiddenFetch }); + // `preact` runs Anubis' own UI code, which needs a real JS runtime. test('refuses a challenge method it cannot compute', async () => { - await expect(attempt(challengePage('metarefresh'))).resolves.toBeNull(); + await expect(attempt(challengePage('preact'))).resolves.toBeNull(); }); test('accepts the methods it does implement', async () => { @@ -145,12 +183,6 @@ describe('Anubis: challenge exchange', () => { challenge: { id: CHALLENGE_ID, randomData: RANDOM_DATA, method: algorithm } })}`; - const withSetCookie = (...cookies: string[]): Headers => { - const headers = new Headers(); - cookies.forEach((cookie) => headers.append('set-cookie', cookie)); - return headers; - }; - /** * Stands in for Anubis. `hidesSetCookie` mimics a browser or React Native, * where Set-Cookie is stripped from what scripts may read. @@ -171,7 +203,9 @@ describe('Anubis: challenge exchange', () => { // Validate the submitted proof the way the real server would. const nonce = url.searchParams.get('nonce') ?? ''; const claimed = url.searchParams.get('response') ?? ''; - const digest = createHash('sha256').update(RANDOM_DATA + nonce).digest('hex'); + const digest = createHash('sha256') + .update(RANDOM_DATA + nonce) + .digest('hex'); const provenWork = claimed === digest && digest.startsWith('0'.repeat(DIFFICULTY)); const rightChallenge = url.searchParams.get('id') === CHALLENGE_ID; const provenCookies = @@ -227,12 +261,27 @@ describe('Anubis: challenge exchange', () => { const server = fakeAnubis(); const client = createAnubisClient({ fetch: server.fetch }); - expect( - await client.pass(interstitial('metarefresh'), withSetCookie(VERIFY_COOKIE), PAGE_URL) - ).toBe(false); + expect(await client.pass(interstitial('preact'), withSetCookie(VERIFY_COOKIE), PAGE_URL)).toBe( + false + ); expect(server.calls).toHaveLength(0); }); + // A runtime that lets us read Set-Cookie would have shown the auth cookie, so + // a bare redirect is a failed exchange — not a cookie quietly stashed in a jar + // this runtime does not even have. Claiming success here would send the retry + // out with `credentials: 'include'` and no cookie at all. + test('does not claim success when a readable response redirects without a cookie', async () => { + const client = createAnubisClient({ + fetch: async () => new Response(null, { status: 302, headers: withSetCookie('_nss=1') }) + }); + + // Set-Cookie is readable, it just never carried the verification cookie. + expect(await client.pass(interstitial(), withSetCookie('_nss=1'), PAGE_URL)).toBe(false); + expect(client.getCookie()).toBeNull(); + expect(client.usesPlatformCookieJar()).toBe(false); + }); + test('concurrent callers share a single proof-of-work', async () => { const server = fakeAnubis(); const client = createAnubisClient({ fetch: server.fetch }); @@ -261,6 +310,210 @@ describe('Anubis: challenge exchange', () => { }); }); +// `metarefresh` is Anubis' patience challenge: no hashing, but the exchange is +// refused until the delay it declared has elapsed. ČSFD serves this one to +// requests that already look browser-like, so it is the variant most likely to +// be met in practice. +describe('Anubis: metarefresh challenge', () => { + const CHALLENGE_ID = '019fc8c0-0951-771c-8569-4305e0bdccea'; + const RANDOM_DATA = 'b737eccc'.repeat(16); + const PAGE_URL = 'https://example.test/protected/'; + const PASS_PATH = '/.within.website/x/cmd/anubis/api/pass-challenge'; + const AUTH_COOKIE = 'techaro.lol-anubis-auth=header.payload.signature'; + const VERIFY_COOKIE = `techaro.lol-anubis-cookie-verification=${CHALLENGE_ID}`; + + /** How Anubis handed over the exchange URL — or `none` when it served neither. */ + type Directive = 'header' | 'meta' | 'none'; + + const passQuery = `challenge=${RANDOM_DATA}&id=${CHALLENGE_ID}&redir=%2Fprotected%2F`; + const directive = (delaySeconds: number) => `${delaySeconds}; url=${PASS_PATH}?${passQuery}`; + + const interstitial = (via: Directive, delaySeconds: number): string => { + // Inside a meta attribute the query separators have to be escaped. + const meta = + via === 'meta' + ? `` + : ''; + const challenge = JSON.stringify({ + rules: { algorithm: 'metarefresh', difficulty: 1 }, + challenge: { id: CHALLENGE_ID, method: 'metarefresh', randomData: RANDOM_DATA } + }); + return `${meta}`; + }; + + const challengeHeaders = (via: Directive, delaySeconds: number): Headers => { + const headers = withSetCookie(`${VERIFY_COOKIE}; Path=/`); + if (via === 'header') { + headers.set('refresh', directive(delaySeconds)); + } + return headers; + }; + + /** + * Stands in for Anubis: refuses the exchange until `enforcedDelaySeconds` have + * passed and insists on the verification cookie, as the live server does. + */ + const fakeAnubis = (enforcedDelaySeconds: number) => { + const calls: { url: URL; init?: RequestInit }[] = []; + let waited = enforcedDelaySeconds === 0; + if (!waited) { + setTimeout(() => (waited = true), enforcedDelaySeconds * 1000); + } + + const fetch = async (input: string, init?: RequestInit): Promise => { + const url = new URL(input); + calls.push({ url, init }); + + if (!waited) { + return new Response('Oh noes!', { status: 403 }); + } + if (new Headers(init?.headers).get('Cookie') !== VERIFY_COOKIE) { + return new Response('Oh noes!', { status: 500 }); + } + if ( + url.searchParams.get('id') !== CHALLENGE_ID || + url.searchParams.get('challenge') !== RANDOM_DATA + ) { + return new Response('Oh noes!', { status: 403 }); + } + return new Response(null, { status: 302, headers: withSetCookie(AUTH_COOKIE) }); + }; + + return { fetch, calls }; + }; + + test('passes without hashing when Anubis sends the Refresh header', async () => { + const server = fakeAnubis(0); + const client = createAnubisClient({ fetch: server.fetch }); + + expect( + await client.pass(interstitial('header', 0), challengeHeaders('header', 0), PAGE_URL) + ).toBe(true); + expect(client.getCookie()).toBe(AUTH_COOKIE); + expect(server.calls).toHaveLength(1); + expect(server.calls[0].url.pathname).toBe(PASS_PATH); + }); + + test('reads the exchange URL out of the meta tag as well, unescaped', async () => { + const server = fakeAnubis(0); + const client = createAnubisClient({ fetch: server.fetch }); + + expect(await client.pass(interstitial('meta', 0), challengeHeaders('meta', 0), PAGE_URL)).toBe( + true + ); + // A stray `&` would fold the whole query into a single parameter. + const [call] = server.calls; + expect(call.url.searchParams.get('id')).toBe(CHALLENGE_ID); + expect(call.url.searchParams.get('challenge')).toBe(RANDOM_DATA); + expect(call.url.searchParams.get('redir')).toBe('/protected/'); + }); + + test('sits out the declared delay instead of being refused', async () => { + vi.useFakeTimers(); + try { + const server = fakeAnubis(2); + const client = createAnubisClient({ fetch: server.fetch }); + const passing = client.pass( + interstitial('header', 2), + challengeHeaders('header', 2), + PAGE_URL + ); + + await vi.advanceTimersByTimeAsync(1999); + expect(server.calls).toHaveLength(0); + + await vi.advanceTimersByTimeAsync(1); + expect(await passing).toBe(true); + expect(client.getCookie()).toBe(AUTH_COOKIE); + } finally { + vi.useRealTimers(); + } + }); + + test('rebuilds the exchange URL when Anubis serves no directive at all', async () => { + vi.useFakeTimers(); + try { + const server = fakeAnubis(2); + const client = createAnubisClient({ fetch: server.fetch }); + const passing = client.pass(interstitial('none', 2), challengeHeaders('none', 2), PAGE_URL); + + await vi.advanceTimersByTimeAsync(2000); + expect(await passing).toBe(true); + + const [call] = server.calls; + expect(call.url.pathname).toBe(PASS_PATH); + expect(call.url.searchParams.get('challenge')).toBe(RANDOM_DATA); + expect(call.url.searchParams.get('id')).toBe(CHALLENGE_ID); + expect(call.url.searchParams.get('redir')).toBe(PAGE_URL); + } finally { + vi.useRealTimers(); + } + }); + + test('gives up rather than wait longer than the caller allows', async () => { + const server = fakeAnubis(0); + const client = createAnubisClient({ fetch: server.fetch, timeBudgetMs: 500 }); + + expect( + await client.pass(interstitial('header', 30), challengeHeaders('header', 30), PAGE_URL) + ).toBe(false); + expect(server.calls).toHaveLength(0); + }); +}); + +// React Native's fetch is an XHR polyfill: `Headers` has no `getSetCookie` and +// `redirect: 'manual'` is ignored, so the 302 after the exchange is followed and +// the auth cookie only ever lands in the platform's own jar. +describe('Anubis: runtimes that hide Set-Cookie', () => { + const CHALLENGE_ID = '019f8462-2dd6-7755-9d82-de2c14c2d59e'; + const RANDOM_DATA = 'a3f1c908'.repeat(16); + const PAGE_URL = 'https://example.test/protected/'; + const PASS_PATH = '/.within.website/x/cmd/anubis/api/pass-challenge'; + const PROTECTED_PAGE = '

Vykoupení z věznice Shawshank

'; + + const interstitial = ``; + + /** + * A React Native-shaped fetch: nothing readable in Set-Cookie, and the + * redirect is followed, so the exchange returns whatever `redir` served. + */ + const reactNativeFetch = (destination: string) => { + const calls: { url: URL; init?: RequestInit }[] = []; + const fetch = async (input: string, init?: RequestInit): Promise => { + const url = new URL(input); + calls.push({ url, init }); + const body = url.pathname === PASS_PATH ? destination : interstitial; + return new Response(body, { status: 200, headers: new Headers() }); + }; + return { fetch, calls }; + }; + + test('takes the followed redirect as proof and leaves the cookie to the jar', async () => { + const runtime = reactNativeFetch(PROTECTED_PAGE); + const client = createAnubisClient({ fetch: runtime.fetch }); + + expect(await client.pass(interstitial, new Headers(), PAGE_URL)).toBe(true); + expect(client.getCookie()).toBeNull(); + expect(client.usesPlatformCookieJar()).toBe(true); + + // Every request must opt into the jar, or the cookie never rides along. + expect(runtime.calls.every((call) => call.init?.credentials === 'include')).toBe(true); + }); + + test('reports failure when the exchange lands back on the interstitial', async () => { + const runtime = reactNativeFetch(interstitial); + const client = createAnubisClient({ fetch: runtime.fetch }); + + expect(await client.pass(interstitial, new Headers(), PAGE_URL)).toBe(false); + expect(client.getCookie()).toBeNull(); + }); +}); + describe('Anubis: client', () => { test('holds and clears its cookie independently per instance', () => { const a = createAnubisClient(); From d1a718a20f086d2be92df74ab5978ce6a88df574 Mon Sep 17 00:00:00 2001 From: BART! Date: Mon, 3 Aug 2026 20:21:14 +0200 Subject: [PATCH 5/6] feat(fetch): error reasons --- src/errors.ts | 36 +++++++++++++++++ src/fetchers/index.ts | 89 +++++++++++++++++++++++++----------------- src/index.ts | 2 + tests/fetchers.test.ts | 29 ++++++-------- 4 files changed, 103 insertions(+), 53 deletions(-) create mode 100644 src/errors.ts diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..11fd4af --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,36 @@ +import { LIB_PREFIX } from './vars'; + +export type CsfdErrorReason = + /** An anti-bot challenge stood in the way and could not be passed. */ + | 'blocked' + /** ČSFD answered 404 — the movie, creator or user does not exist. */ + | 'not-found' + /** Any other unsuccessful HTTP status. */ + | 'http' + /** The request never completed: offline, DNS, TLS, timeout. */ + | 'network'; + +interface CsfdErrorOptions { + status?: number; + cause?: unknown; +} + +/** + * Raised when a page cannot be retrieved. `reason` says why, so callers can + * tell "this movie does not exist" from "ČSFD is refusing us right now" and + * react differently instead of guessing from a message. + */ +export class CsfdError extends Error { + readonly reason: CsfdErrorReason; + readonly url: string; + /** HTTP status, when the request got far enough to have one. */ + readonly status?: number; + + constructor(reason: CsfdErrorReason, url: string, message: string, options?: CsfdErrorOptions) { + super(`${LIB_PREFIX} ${message}`, { cause: options?.cause }); + this.name = 'CsfdError'; + this.reason = reason; + this.url = url; + this.status = options?.status; + } +} diff --git a/src/fetchers/index.ts b/src/fetchers/index.ts index 0378b27..d3ce617 100644 --- a/src/fetchers/index.ts +++ b/src/fetchers/index.ts @@ -1,5 +1,5 @@ import { createAnubisClient } from '../anubis'; -import { LIB_PREFIX } from '../vars'; +import { CsfdError } from '../errors'; import { fetchSafe } from './fetch.polyfill'; interface BrowserProfile { @@ -103,55 +103,74 @@ const buildHeaders = (optionsRequest?: RequestInit): Headers => { return mergedHeaders; }; +/** + * Fetch a ČSFD page, passing an anti-bot challenge if one is served. + * + * @throws {CsfdError} when the page cannot be retrieved. Never returns a + * placeholder body: letting one reach the parsers turns a plain failure into an + * unrelated crash deep inside them. + */ export const fetchPage = async (url: string, optionsRequest?: RequestInit): Promise => { - try { - const { headers: _, ...restOptions } = optionsRequest || {}; - // Stay credential-less by default so no ambient session rides along; only - // runtimes that hide Set-Cookie need their jar, and only once detected. - const doFetch = () => - fetchSafe(url, { + const { headers: _, ...restOptions } = optionsRequest || {}; + + const doFetch = async (): Promise => { + let response: Response; + try { + // Stay credential-less by default so no ambient session rides along; only + // runtimes that hide Set-Cookie need their jar, and only once detected. + response = await fetchSafe(url, { credentials: anubis.usesPlatformCookieJar() ? 'include' : 'omit', ...restOptions, headers: buildHeaders(optionsRequest) }); + } catch (e: unknown) { + throw new CsfdError('network', url, `Request failed for url: ${url}`, { cause: e }); + } - let response = await doFetch(); if (!response.ok) { - throw new Error(`node-csfd-api: Bad response ${response.status} for url: ${url}`); + throw new CsfdError( + response.status === 404 ? 'not-found' : 'http', + url, + `Bad response ${response.status} for url: ${url}`, + { status: response.status } + ); } - - let html = await response.text(); - - if (anubis.isChallenge(html)) { - const passed = await anubis.pass( + return response; + }; + + let response = await doFetch(); + let html = await response.text(); + + if (anubis.isChallenge(html)) { + // A failure inside the exchange is just another way of not getting through, + // so it is reported as `blocked` with the original error kept as the cause. + let passed = false; + let exchangeError: unknown; + try { + passed = await anubis.pass( html, response.headers, url, new Headers({ ...baseHeaders, ...randomProfile() }) ); - if (passed) { - response = await doFetch(); - if (!response.ok) { - throw new Error(`node-csfd-api: Bad response ${response.status} for url: ${url}`); - } - html = await response.text(); - } - // Fail loudly rather than let the interstitial reach the parsers, where - // it would silently look like a page with no results. - if (anubis.isChallenge(html)) { - throw new Error( - `node-csfd-api: Anubis challenge could not be solved for url: ${url}. You may be rate-limited or blocked by ČSFD.` - ); - } + } catch (e: unknown) { + exchangeError = e; } - return html; - } catch (e: unknown) { - if (e instanceof Error) { - console.error(LIB_PREFIX, e.message); - } else { - console.error(LIB_PREFIX, String(e)); + if (passed) { + response = await doFetch(); + html = await response.text(); + } + + if (anubis.isChallenge(html)) { + throw new CsfdError( + 'blocked', + url, + `Anti-bot challenge could not be passed for url: ${url}. You may be rate-limited or blocked by ČSFD.`, + { cause: exchangeError } + ); } - return 'Error'; } + + return html; }; diff --git a/src/index.ts b/src/index.ts index d320c70..d0741a8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -95,6 +95,8 @@ export const csfd = new Csfd( cinemaScraper ); +export { CsfdError } from './errors'; +export type { CsfdErrorReason } from './errors'; export { getAnubisCookie, resetAnubisCookie, setAnubisCookie } from './fetchers'; export type * from './dto'; diff --git a/tests/fetchers.test.ts b/tests/fetchers.test.ts index dd8458e..53c7b29 100644 --- a/tests/fetchers.test.ts +++ b/tests/fetchers.test.ts @@ -1,5 +1,5 @@ import { beforeAll, describe, expect, test } from 'vitest'; -import { csfd, CSFDUserRatings } from '../src'; +import { csfd, CsfdError, CSFDUserRatings } from '../src'; import { CSFDCinema } from '../src/dto/cinema'; import { CSFDCreator, CSFDCreatorScreening } from '../src/dto/creator'; import { CSFDColorRating, CSFDFilmTypes } from '../src/dto/global'; @@ -34,7 +34,6 @@ describe('Fetch generic page', () => { test('Fetch main page and check html logic', async () => { const html = await fetchPage('https://www.csfd.cz/'); expect(html).toContain('csfd'); - expect(html).not.toEqual('Error'); }); }); @@ -370,26 +369,20 @@ describe('Live: User Reviews page', () => { // Edge cases describe('User page 404', () => { - test('Fetch error URL', async () => { - try { - const url = userRatingsUrl(badId); - const html = await fetchPage(url); - expect(html).toBe('Error'); - } catch (e) { - expect(e).toContain(Error); - } + test('Rejects with a typed error instead of a placeholder body', async () => { + const error = await fetchPage(userRatingsUrl(badId)).catch((e: unknown) => e); + expect(error).toBeInstanceOf(CsfdError); + expect((error as CsfdError).reason).toBe('not-found'); + expect((error as CsfdError).status).toBe(404); + expect((error as CsfdError).url).toContain(String(badId)); }); }); describe('Movie page 404', () => { - test('Fetch error URL', async () => { - try { - const url = movieUrl(badId, {}); - const html = await fetchPage(url); - expect(html).toBe('Error'); - } catch (e) { - expect(e).toThrow(Error); - } + test('Rejects with a typed error instead of a placeholder body', async () => { + const error = await fetchPage(movieUrl(badId, {})).catch((e: unknown) => e); + expect(error).toBeInstanceOf(CsfdError); + expect((error as CsfdError).reason).toBe('not-found'); }); }); From b19cb8e4765bd2f5a6ab92ca5536ea73e54922af Mon Sep 17 00:00:00 2001 From: BART! Date: Mon, 3 Aug 2026 20:39:24 +0200 Subject: [PATCH 6/6] fix(anubis): challenge for rn --- src/anubis/challenge.ts | 11 ++++++++- tests/anubis.test.ts | 52 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/anubis/challenge.ts b/src/anubis/challenge.ts index a142b8d..0f8307e 100644 --- a/src/anubis/challenge.ts +++ b/src/anubis/challenge.ts @@ -195,7 +195,16 @@ export const passChallenge = async ({ redirect: 'manual', headers: requestHeaders }); - html = await reissued.text(); + const reissuedHtml = await reissued.text(); + + // Credentials change the answer: the jar may already hold a valid auth + // cookie, in which case this sails straight past Anubis. There is then no + // challenge left to solve — only a request worth retrying with the jar. + if (!isAnubisChallenge(reissuedHtml)) { + return { cookie: null, platformCookieJar }; + } + + html = reissuedHtml; headers = reissued.headers; } diff --git a/tests/anubis.test.ts b/tests/anubis.test.ts index 78b5fd7..5d2c3a7 100644 --- a/tests/anubis.test.ts +++ b/tests/anubis.test.ts @@ -512,6 +512,58 @@ describe('Anubis: runtimes that hide Set-Cookie', () => { expect(await client.pass(interstitial, new Headers(), PAGE_URL)).toBe(false); expect(client.getCookie()).toBeNull(); }); + + // Observed on a device: the credential-less first request is challenged even + // though the jar already holds a valid auth cookie, and the reissue that does + // send it sails through to ČSFD's own canonical redirect. There is nothing + // left to solve, so the challenge must not be treated as unsolvable. + test('treats a reissue that gets through as passed, not as a dead end', async () => { + const calls: URL[] = []; + const client = createAnubisClient({ + fetch: async (input) => { + calls.push(new URL(input)); + return new Response('Redirecting', { + status: 302, + headers: new Headers({ location: '/film/1822825-pet-svestek/prehled/' }) + }); + } + }); + + expect(await client.pass(interstitial, new Headers(), PAGE_URL)).toBe(true); + expect(client.usesPlatformCookieJar()).toBe(true); + expect(client.getCookie()).toBeNull(); + + // Only the reissue: solving anything would have needed a second request. + expect(calls).toHaveLength(1); + expect(calls[0].pathname).toBe('/protected/'); + }); + + // The interstitial we were handed is worthless on a jar runtime — its + // verification cookie was never stored — so the reissued one has to be used. + test('solves the challenge the reissue returns, not the one it was handed', async () => { + const staleInterstitial = ``; + + const calls: URL[] = []; + const client = createAnubisClient({ + fetch: async (input) => { + const url = new URL(input); + calls.push(url); + if (url.pathname === PASS_PATH) { + return new Response(PROTECTED_PAGE, { status: 200, headers: new Headers() }); + } + return new Response(interstitial, { status: 200, headers: new Headers() }); + } + }); + + expect(await client.pass(staleInterstitial, new Headers(), PAGE_URL)).toBe(true); + const exchange = calls.find((url) => url.pathname === PASS_PATH); + expect(exchange?.searchParams.get('id')).toBe(CHALLENGE_ID); + }); }); describe('Anubis: client', () => {