diff --git a/.yarnrc.yml b/.yarnrc.yml index dc612810..063ed601 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -4,3 +4,5 @@ approvedGitRepositories: enableScripts: true nodeLinker: node-modules + +npmMinimalAgeGate: 0 diff --git a/package.json b/package.json index bfdaa5d1..92e1d53c 100644 --- a/package.json +++ b/package.json @@ -126,5 +126,5 @@ "node-csfd-api": "./dist/cli.js" }, "sideEffects": false, - "packageManager": "yarn@4.14.1" + "packageManager": "yarn@4.18.0" } diff --git a/src/anubis/challenge.ts b/src/anubis/challenge.ts new file mode 100644 index 00000000..0f8307ec --- /dev/null +++ b/src/anubis/challenge.ts @@ -0,0 +1,276 @@ +import { DEFAULT_TIME_BUDGET_MS, solveProofOfWork } from './proof-of-work'; + +// Anubis (BotStopper by Techaro) is a proof-of-work anti-bot interstitial: +// instead of the page it serves an HTML challenge that a browser solves in +// JavaScript. This module replicates the protocol so a plain `fetch` can earn +// the auth cookie. See: https://github.com/TecharoHQ/anubis + +const AUTH_COOKIE_NAME = 'techaro.lol-anubis-auth'; +const VERIFY_COOKIE_NAME = 'techaro.lol-anubis-cookie-verification'; +const PASS_CHALLENGE_PATH = '/.within.website/x/cmd/anubis/api/pass-challenge'; + +// Anubis picks a challenge method per request. The SHA-256 ones make the client +// burn CPU; `metarefresh` instead makes it sit out a declared delay. Anything +// else (e.g. `preact`) needs a real JS runtime and must fail immediately rather +// than burn the whole time budget computing a hash nobody asked for. +const PROOF_OF_WORK_ALGORITHMS = ['fast', 'slow']; +const METAREFRESH_ALGORITHM = 'metarefresh'; + +// Anubis states the wait in a `Refresh` header or its `` twin. This is +// only the fallback for a page that omits both, where the exchange URL has to +// be rebuilt from the challenge anyway. +const DEFAULT_METAREFRESH_DELAY_MS = 2000; + +// Structural markers, deliberately not the localised body text: the +// interstitial is translated, so matching prose would both miss locales and +// risk false positives on user-generated content that quotes it. +const CHALLENGE_MARKERS = ['id="anubis_challenge"', '/.within.website/x/cmd/anubis/']; + +export type FetchLike = (input: string, init?: RequestInit) => Promise; + +interface ParsedChallenge { + rules: { algorithm: string; difficulty: number }; + challenge: { id: string; randomData: string }; +} + +export interface ChallengeResult { + /** Cookie to replay, or `null` when the runtime's own cookie jar holds it. */ + cookie: string | null; + /** True when Set-Cookie was hidden and the runtime now owns the cookie. */ + platformCookieJar: boolean; +} + +export interface PassChallengeParams { + /** Body of the interstitial page that was served instead of the content. */ + html: string; + /** Response headers that came with it, carrying the verification cookie. */ + headers: Headers; + /** The URL that was blocked; used as the redirect target after passing. */ + url: string; + /** Headers to reuse, so the exchange looks like the original request. */ + requestHeaders?: Headers; + fetch: FetchLike; + timeBudgetMs?: number; +} + +export const isAnubisChallenge = (html: string): boolean => + CHALLENGE_MARKERS.some((marker) => html.includes(marker)); + +const parseChallenge = (html: string): ParsedChallenge | null => { + const match = html.match( + /`; + + // A readable Set-Cookie keeps us on the manual (Node) path, so these bail + // before any request — a fetch that throws proves the network is never hit. + const nodeHeaders = () => + new Headers({ 'set-cookie': 'techaro.lol-anubis-cookie-verification=test-id; Path=/' }); + const forbiddenFetch = () => Promise.reject(new Error('must not reach the network')); + + const attempt = (html: string) => + passChallenge({ + html, + headers: nodeHeaders(), + url: 'https://www.csfd.cz/film/1/', + fetch: forbiddenFetch + }); + + // `preact` runs Anubis' own UI code, which needs a real JS runtime. + test('refuses a challenge method it cannot compute', async () => { + await expect(attempt(challengePage('preact'))).resolves.toBeNull(); + }); + + test('accepts the methods it does implement', async () => { + // Reaching the network means the guard let it through to the exchange. + await expect(attempt(challengePage('fast'))).rejects.toThrow('must not reach the network'); + }); + + test('refuses an unparseable challenge', async () => { + await expect( + attempt('') + ).resolves.toBeNull(); + }); +}); + +// The live site only challenges a fraction of requests, so a green integration +// run can miss the solver entirely. This exercises the whole exchange against a +// stand-in server that validates the proof exactly as Anubis does, offline. +describe('Anubis: challenge exchange', () => { + const CHALLENGE_ID = '019f8462-2dd6-7755-9d82-de2c14c2d59e'; + const RANDOM_DATA = 'a3f1c908'.repeat(16); + const DIFFICULTY = 2; + const PAGE_URL = 'https://example.test/protected/'; + const PASS_PATH = '/.within.website/x/cmd/anubis/api/pass-challenge'; + const AUTH_COOKIE = 'techaro.lol-anubis-auth=header.payload.signature'; + const VERIFY_COOKIE = `techaro.lol-anubis-cookie-verification=${CHALLENGE_ID}`; + + const interstitial = (algorithm = 'fast') => + ``; + + /** + * Stands in for Anubis. `hidesSetCookie` mimics a browser or React Native, + * where Set-Cookie is stripped from what scripts may read. + */ + const fakeAnubis = ({ hidesSetCookie = false } = {}) => { + const calls: { url: URL; init?: RequestInit }[] = []; + + const fetch = async (input: string, init?: RequestInit): Promise => { + const url = new URL(input); + calls.push({ url, init }); + + if (url.pathname !== PASS_PATH) { + return new Response(interstitial(), { + headers: hidesSetCookie ? new Headers() : withSetCookie(VERIFY_COOKIE) + }); + } + + // Validate the submitted proof the way the real server would. + const nonce = url.searchParams.get('nonce') ?? ''; + const claimed = url.searchParams.get('response') ?? ''; + const digest = createHash('sha256') + .update(RANDOM_DATA + nonce) + .digest('hex'); + const provenWork = claimed === digest && digest.startsWith('0'.repeat(DIFFICULTY)); + const rightChallenge = url.searchParams.get('id') === CHALLENGE_ID; + const provenCookies = + hidesSetCookie || new Headers(init?.headers).get('Cookie') === VERIFY_COOKIE; + + if (!provenWork || !rightChallenge || !provenCookies) { + return new Response('challenge failed', { status: 403 }); + } + return new Response(null, { + status: 302, + headers: hidesSetCookie ? new Headers({ location: PAGE_URL }) : withSetCookie(AUTH_COOKIE) + }); + }; + + return { fetch, calls, passCalls: () => calls.filter((c) => c.url.pathname === PASS_PATH) }; + }; + + test('earns and caches a cookie the server accepts', async () => { + const server = fakeAnubis(); + const client = createAnubisClient({ fetch: server.fetch }); + + // The server only issues the cookie if the proof verifies, so this passing + // means our hash construction still matches Anubis' own. + expect(await client.pass(interstitial(), withSetCookie(VERIFY_COOKIE), PAGE_URL)).toBe(true); + expect(client.getCookie()).toBe(AUTH_COOKIE); + expect(client.usesPlatformCookieJar()).toBe(false); + }); + + test('submits the parameters Anubis expects', async () => { + const server = fakeAnubis(); + const client = createAnubisClient({ fetch: server.fetch }); + await client.pass(interstitial(), withSetCookie(VERIFY_COOKIE), PAGE_URL); + + const [call] = server.passCalls(); + expect(call.url.searchParams.get('id')).toBe(CHALLENGE_ID); + expect(call.url.searchParams.get('redir')).toBe(PAGE_URL); + expect(Number(call.url.searchParams.get('elapsedTime'))).toBeGreaterThanOrEqual(0); + expect(new Headers(call.init?.headers).get('Cookie')).toBe(VERIFY_COOKIE); + // Following the 302 would discard the Set-Cookie we came for. + expect(call.init?.redirect).toBe('manual'); + }); + + test('keeps no cookie when the server rejects the proof', async () => { + const client = createAnubisClient({ + fetch: async () => new Response('challenge failed', { status: 403 }) + }); + + expect(await client.pass(interstitial(), withSetCookie(VERIFY_COOKIE), PAGE_URL)).toBe(false); + expect(client.getCookie()).toBeNull(); + }); + + test('refuses a challenge method it cannot compute without asking the server', async () => { + const server = fakeAnubis(); + const client = createAnubisClient({ fetch: server.fetch }); + + expect(await client.pass(interstitial('preact'), withSetCookie(VERIFY_COOKIE), PAGE_URL)).toBe( + false + ); + expect(server.calls).toHaveLength(0); + }); + + // A runtime that lets us read Set-Cookie would have shown the auth cookie, so + // a bare redirect is a failed exchange — not a cookie quietly stashed in a jar + // this runtime does not even have. Claiming success here would send the retry + // out with `credentials: 'include'` and no cookie at all. + test('does not claim success when a readable response redirects without a cookie', async () => { + const client = createAnubisClient({ + fetch: async () => new Response(null, { status: 302, headers: withSetCookie('_nss=1') }) + }); + + // Set-Cookie is readable, it just never carried the verification cookie. + expect(await client.pass(interstitial(), withSetCookie('_nss=1'), PAGE_URL)).toBe(false); + expect(client.getCookie()).toBeNull(); + expect(client.usesPlatformCookieJar()).toBe(false); + }); + + test('concurrent callers share a single proof-of-work', async () => { + const server = fakeAnubis(); + const client = createAnubisClient({ fetch: server.fetch }); + + const results = await Promise.all( + Array.from({ length: 4 }, () => + client.pass(interstitial(), withSetCookie(VERIFY_COOKIE), PAGE_URL) + ) + ); + + expect(results).toEqual([true, true, true, true]); + expect(server.passCalls()).toHaveLength(1); + }); + + test('hands the cookie to the runtime jar when Set-Cookie is hidden', async () => { + const server = fakeAnubis({ hidesSetCookie: true }); + const client = createAnubisClient({ fetch: server.fetch }); + + expect(await client.pass(interstitial(), new Headers(), PAGE_URL)).toBe(true); + expect(client.getCookie()).toBeNull(); + expect(client.usesPlatformCookieJar()).toBe(true); + + // The challenge is re-requested with credentials so the jar can store it. + const reissue = server.calls.find((call) => call.url.pathname !== PASS_PATH); + expect(reissue?.init?.credentials).toBe('include'); + }); +}); + +// `metarefresh` is Anubis' patience challenge: no hashing, but the exchange is +// refused until the delay it declared has elapsed. ČSFD serves this one to +// requests that already look browser-like, so it is the variant most likely to +// be met in practice. +describe('Anubis: metarefresh challenge', () => { + const CHALLENGE_ID = '019fc8c0-0951-771c-8569-4305e0bdccea'; + const RANDOM_DATA = 'b737eccc'.repeat(16); + const PAGE_URL = 'https://example.test/protected/'; + const PASS_PATH = '/.within.website/x/cmd/anubis/api/pass-challenge'; + const AUTH_COOKIE = 'techaro.lol-anubis-auth=header.payload.signature'; + const VERIFY_COOKIE = `techaro.lol-anubis-cookie-verification=${CHALLENGE_ID}`; + + /** How Anubis handed over the exchange URL — or `none` when it served neither. */ + type Directive = 'header' | 'meta' | 'none'; + + const passQuery = `challenge=${RANDOM_DATA}&id=${CHALLENGE_ID}&redir=%2Fprotected%2F`; + const directive = (delaySeconds: number) => `${delaySeconds}; url=${PASS_PATH}?${passQuery}`; + + const interstitial = (via: Directive, delaySeconds: number): string => { + // Inside a meta attribute the query separators have to be escaped. + const meta = + via === 'meta' + ? `` + : ''; + const challenge = JSON.stringify({ + rules: { algorithm: 'metarefresh', difficulty: 1 }, + challenge: { id: CHALLENGE_ID, method: 'metarefresh', randomData: RANDOM_DATA } + }); + return `${meta}`; + }; + + const challengeHeaders = (via: Directive, delaySeconds: number): Headers => { + const headers = withSetCookie(`${VERIFY_COOKIE}; Path=/`); + if (via === 'header') { + headers.set('refresh', directive(delaySeconds)); + } + return headers; + }; + + /** + * Stands in for Anubis: refuses the exchange until `enforcedDelaySeconds` have + * passed and insists on the verification cookie, as the live server does. + */ + const fakeAnubis = (enforcedDelaySeconds: number) => { + const calls: { url: URL; init?: RequestInit }[] = []; + let waited = enforcedDelaySeconds === 0; + if (!waited) { + setTimeout(() => (waited = true), enforcedDelaySeconds * 1000); + } + + const fetch = async (input: string, init?: RequestInit): Promise => { + const url = new URL(input); + calls.push({ url, init }); + + if (!waited) { + return new Response('Oh noes!', { status: 403 }); + } + if (new Headers(init?.headers).get('Cookie') !== VERIFY_COOKIE) { + return new Response('Oh noes!', { status: 500 }); + } + if ( + url.searchParams.get('id') !== CHALLENGE_ID || + url.searchParams.get('challenge') !== RANDOM_DATA + ) { + return new Response('Oh noes!', { status: 403 }); + } + return new Response(null, { status: 302, headers: withSetCookie(AUTH_COOKIE) }); + }; + + return { fetch, calls }; + }; + + test('passes without hashing when Anubis sends the Refresh header', async () => { + const server = fakeAnubis(0); + const client = createAnubisClient({ fetch: server.fetch }); + + expect( + await client.pass(interstitial('header', 0), challengeHeaders('header', 0), PAGE_URL) + ).toBe(true); + expect(client.getCookie()).toBe(AUTH_COOKIE); + expect(server.calls).toHaveLength(1); + expect(server.calls[0].url.pathname).toBe(PASS_PATH); + }); + + test('reads the exchange URL out of the meta tag as well, unescaped', async () => { + const server = fakeAnubis(0); + const client = createAnubisClient({ fetch: server.fetch }); + + expect(await client.pass(interstitial('meta', 0), challengeHeaders('meta', 0), PAGE_URL)).toBe( + true + ); + // A stray `&` would fold the whole query into a single parameter. + const [call] = server.calls; + expect(call.url.searchParams.get('id')).toBe(CHALLENGE_ID); + expect(call.url.searchParams.get('challenge')).toBe(RANDOM_DATA); + expect(call.url.searchParams.get('redir')).toBe('/protected/'); + }); + + test('sits out the declared delay instead of being refused', async () => { + vi.useFakeTimers(); + try { + const server = fakeAnubis(2); + const client = createAnubisClient({ fetch: server.fetch }); + const passing = client.pass( + interstitial('header', 2), + challengeHeaders('header', 2), + PAGE_URL + ); + + await vi.advanceTimersByTimeAsync(1999); + expect(server.calls).toHaveLength(0); + + await vi.advanceTimersByTimeAsync(1); + expect(await passing).toBe(true); + expect(client.getCookie()).toBe(AUTH_COOKIE); + } finally { + vi.useRealTimers(); + } + }); + + test('rebuilds the exchange URL when Anubis serves no directive at all', async () => { + vi.useFakeTimers(); + try { + const server = fakeAnubis(2); + const client = createAnubisClient({ fetch: server.fetch }); + const passing = client.pass(interstitial('none', 2), challengeHeaders('none', 2), PAGE_URL); + + await vi.advanceTimersByTimeAsync(2000); + expect(await passing).toBe(true); + + const [call] = server.calls; + expect(call.url.pathname).toBe(PASS_PATH); + expect(call.url.searchParams.get('challenge')).toBe(RANDOM_DATA); + expect(call.url.searchParams.get('id')).toBe(CHALLENGE_ID); + expect(call.url.searchParams.get('redir')).toBe(PAGE_URL); + } finally { + vi.useRealTimers(); + } + }); + + test('gives up rather than wait longer than the caller allows', async () => { + const server = fakeAnubis(0); + const client = createAnubisClient({ fetch: server.fetch, timeBudgetMs: 500 }); + + expect( + await client.pass(interstitial('header', 30), challengeHeaders('header', 30), PAGE_URL) + ).toBe(false); + expect(server.calls).toHaveLength(0); + }); +}); + +// React Native's fetch is an XHR polyfill: `Headers` has no `getSetCookie` and +// `redirect: 'manual'` is ignored, so the 302 after the exchange is followed and +// the auth cookie only ever lands in the platform's own jar. +describe('Anubis: runtimes that hide Set-Cookie', () => { + const CHALLENGE_ID = '019f8462-2dd6-7755-9d82-de2c14c2d59e'; + const RANDOM_DATA = 'a3f1c908'.repeat(16); + const PAGE_URL = 'https://example.test/protected/'; + const PASS_PATH = '/.within.website/x/cmd/anubis/api/pass-challenge'; + const PROTECTED_PAGE = '

Vykoupení z věznice Shawshank

'; + + const interstitial = ``; + + /** + * A React Native-shaped fetch: nothing readable in Set-Cookie, and the + * redirect is followed, so the exchange returns whatever `redir` served. + */ + const reactNativeFetch = (destination: string) => { + const calls: { url: URL; init?: RequestInit }[] = []; + const fetch = async (input: string, init?: RequestInit): Promise => { + const url = new URL(input); + calls.push({ url, init }); + const body = url.pathname === PASS_PATH ? destination : interstitial; + return new Response(body, { status: 200, headers: new Headers() }); + }; + return { fetch, calls }; + }; + + test('takes the followed redirect as proof and leaves the cookie to the jar', async () => { + const runtime = reactNativeFetch(PROTECTED_PAGE); + const client = createAnubisClient({ fetch: runtime.fetch }); + + expect(await client.pass(interstitial, new Headers(), PAGE_URL)).toBe(true); + expect(client.getCookie()).toBeNull(); + expect(client.usesPlatformCookieJar()).toBe(true); + + // Every request must opt into the jar, or the cookie never rides along. + expect(runtime.calls.every((call) => call.init?.credentials === 'include')).toBe(true); + }); + + test('reports failure when the exchange lands back on the interstitial', async () => { + const runtime = reactNativeFetch(interstitial); + const client = createAnubisClient({ fetch: runtime.fetch }); + + expect(await client.pass(interstitial, new Headers(), PAGE_URL)).toBe(false); + expect(client.getCookie()).toBeNull(); + }); + + // Observed on a device: the credential-less first request is challenged even + // though the jar already holds a valid auth cookie, and the reissue that does + // send it sails through to ČSFD's own canonical redirect. There is nothing + // left to solve, so the challenge must not be treated as unsolvable. + test('treats a reissue that gets through as passed, not as a dead end', async () => { + const calls: URL[] = []; + const client = createAnubisClient({ + fetch: async (input) => { + calls.push(new URL(input)); + return new Response('Redirecting', { + status: 302, + headers: new Headers({ location: '/film/1822825-pet-svestek/prehled/' }) + }); + } + }); + + expect(await client.pass(interstitial, new Headers(), PAGE_URL)).toBe(true); + expect(client.usesPlatformCookieJar()).toBe(true); + expect(client.getCookie()).toBeNull(); + + // Only the reissue: solving anything would have needed a second request. + expect(calls).toHaveLength(1); + expect(calls[0].pathname).toBe('/protected/'); + }); + + // The interstitial we were handed is worthless on a jar runtime — its + // verification cookie was never stored — so the reissued one has to be used. + test('solves the challenge the reissue returns, not the one it was handed', async () => { + const staleInterstitial = ``; + + const calls: URL[] = []; + const client = createAnubisClient({ + fetch: async (input) => { + const url = new URL(input); + calls.push(url); + if (url.pathname === PASS_PATH) { + return new Response(PROTECTED_PAGE, { status: 200, headers: new Headers() }); + } + return new Response(interstitial, { status: 200, headers: new Headers() }); + } + }); + + expect(await client.pass(staleInterstitial, new Headers(), PAGE_URL)).toBe(true); + const exchange = calls.find((url) => url.pathname === PASS_PATH); + expect(exchange?.searchParams.get('id')).toBe(CHALLENGE_ID); + }); +}); + +describe('Anubis: client', () => { + test('holds and clears its cookie independently per instance', () => { + const a = createAnubisClient(); + const b = createAnubisClient(); + + a.setCookie('techaro.lol-anubis-auth=token-a'); + expect(a.getCookie()).toBe('techaro.lol-anubis-auth=token-a'); + expect(b.getCookie()).toBeNull(); + + a.reset(); + expect(a.getCookie()).toBeNull(); + }); + + test('assumes manual cookie handling until told otherwise', () => { + expect(createAnubisClient().usesPlatformCookieJar()).toBe(false); + }); +}); + +describe('Anubis: isAnubisChallenge', () => { + test('detects the embedded challenge script', () => { + expect( + isAnubisChallenge('') + ).toBe(true); + }); + + test('detects the interstitial by its asset paths', () => { + expect( + isAnubisChallenge('') + ).toBe(true); + }); + + // The library serves cs/en/sk, so detection must not hinge on localised prose. + test.each([ + ['Czech', 'Ujišťujeme se, že nejste robot!'], + ['English', "Making sure you're not a bot!"] + ])('detects the %s interstitial via its structure', (_locale, title) => { + const page = `${title}`; + expect(isAnubisChallenge(page)).toBe(true); + }); + + test('returns false for a normal page', () => { + expect(isAnubisChallenge('

BART!

')).toBe(false); + }); + + // A review quoting the interstitial must not be mistaken for a block. + test('does not false-positive on user content mentioning the challenge', () => { + expect(isAnubisChallenge('

Hláška "Ujišťujeme se, že nejste robot!" mě dostala

')).toBe( + false + ); + }); +}); diff --git a/tests/fetchers.test.ts b/tests/fetchers.test.ts index dd8458ef..53c7b291 100644 --- a/tests/fetchers.test.ts +++ b/tests/fetchers.test.ts @@ -1,5 +1,5 @@ import { beforeAll, describe, expect, test } from 'vitest'; -import { csfd, CSFDUserRatings } from '../src'; +import { csfd, CsfdError, CSFDUserRatings } from '../src'; import { CSFDCinema } from '../src/dto/cinema'; import { CSFDCreator, CSFDCreatorScreening } from '../src/dto/creator'; import { CSFDColorRating, CSFDFilmTypes } from '../src/dto/global'; @@ -34,7 +34,6 @@ describe('Fetch generic page', () => { test('Fetch main page and check html logic', async () => { const html = await fetchPage('https://www.csfd.cz/'); expect(html).toContain('csfd'); - expect(html).not.toEqual('Error'); }); }); @@ -370,26 +369,20 @@ describe('Live: User Reviews page', () => { // Edge cases describe('User page 404', () => { - test('Fetch error URL', async () => { - try { - const url = userRatingsUrl(badId); - const html = await fetchPage(url); - expect(html).toBe('Error'); - } catch (e) { - expect(e).toContain(Error); - } + test('Rejects with a typed error instead of a placeholder body', async () => { + const error = await fetchPage(userRatingsUrl(badId)).catch((e: unknown) => e); + expect(error).toBeInstanceOf(CsfdError); + expect((error as CsfdError).reason).toBe('not-found'); + expect((error as CsfdError).status).toBe(404); + expect((error as CsfdError).url).toContain(String(badId)); }); }); describe('Movie page 404', () => { - test('Fetch error URL', async () => { - try { - const url = movieUrl(badId, {}); - const html = await fetchPage(url); - expect(html).toBe('Error'); - } catch (e) { - expect(e).toThrow(Error); - } + test('Rejects with a typed error instead of a placeholder body', async () => { + const error = await fetchPage(movieUrl(badId, {})).catch((e: unknown) => e); + expect(error).toBeInstanceOf(CsfdError); + expect((error as CsfdError).reason).toBe('not-found'); }); }); diff --git a/yarn.lock b/yarn.lock index 2cc93c24..fa1b47f7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # Manual changes might be lost - proceed with caution! __metadata: - version: 9 + version: 10 cacheKey: 10c0 "@babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0":