diff --git a/lib/addons/botDetection.test.ts b/lib/addons/botDetection.test.ts new file mode 100644 index 0000000..51bd784 --- /dev/null +++ b/lib/addons/botDetection.test.ts @@ -0,0 +1,42 @@ +import { isBot } from "./botDetection"; + +describe("isBot", () => { + it.each([ + "Googlebot/2.1 (+http://www.google.com/bot.html)", + "Mozilla/5.0 (compatible; bingbot/2.0)", + "some-crawler/1.0", + "spider", + "Mozilla/5.0 (X11; Linux x86_64) HeadlessChrome/120.0.0.0", + "PhantomJS/2.1.1", + "curl/8.4.0", + "python-requests/2.31.0", + "axios/1.6.0", + "PostmanRuntime/7.36.0", + "GoogleOther", + ])("returns true for bot user agent %j", (ua) => { + expect(isBot(ua)).toBe(true); + }); + + it.each([ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Gecko/20100101 Firefox/121.0", + ])("returns false for real browser user agent %j", (ua) => { + expect(isBot(ua)).toBe(false); + }); + + it("is case insensitive", () => { + expect(isBot("GOOGLEBOT")).toBe(true); + expect(isBot("CuRl/1.0")).toBe(true); + }); + + it("returns false for an empty user agent", () => { + expect(isBot("")).toBe(false); + }); + + it("falls back to navigator.userAgent when no argument is provided", () => { + const spy = jest.spyOn(navigator, "userAgent", "get").mockReturnValue("Googlebot/2.1"); + expect(isBot()).toBe(true); + spy.mockRestore(); + }); +}); diff --git a/lib/addons/botDetection.ts b/lib/addons/botDetection.ts new file mode 100644 index 0000000..0fcc86a --- /dev/null +++ b/lib/addons/botDetection.ts @@ -0,0 +1,33 @@ +const BOT_PATTERN = new RegExp( + [ + "bot", + "crawler", + "spider", + "scraper", + "headless", + "phantomjs", + "selenium", + "webdriver", + "curl", + "wget", + "python", + "java", + "perl", + "ruby", + "go-http-client", + "okhttp", + "axios", + "fetch", + "postman", + "insomnia", + "googleother", + "google-extended", + "google-inspectiontool", + ].join("|"), + "i" +); + +/** True if the current user agent looks like a known bot/crawler. */ +export function isBot(userAgent: string = navigator.userAgent): boolean { + return BOT_PATTERN.test(userAgent || ""); +} diff --git a/lib/edge/targeting.test.js b/lib/edge/targeting.test.js index 2c82961..8d6356b 100644 --- a/lib/edge/targeting.test.js +++ b/lib/edge/targeting.test.js @@ -1,4 +1,4 @@ -import { PrebidORTB2, TargetingKeyValues, Targeting } from "./targeting"; +import { PrebidORTB2, TargetingKeyValues, Targeting, SkipTargetingForBots } from "./targeting"; describe("PrebidORTB2", () => { test("returns empty array on empty input", () => { @@ -45,6 +45,31 @@ describe("TargetingKeyValues", () => { }); }); +describe("SkipTargetingForBots", () => { + const TARGETING_DONE_KEY = "OPTABLE_TARGETING_DONE"; + + beforeEach(() => { + sessionStorage.clear(); + jest.restoreAllMocks(); + }); + + it("marks targeting as done and returns true for a bot", () => { + jest.spyOn(navigator, "userAgent", "get").mockReturnValue("Googlebot/2.1"); + expect(SkipTargetingForBots()).toBe(true); + expect(sessionStorage.getItem(TARGETING_DONE_KEY)).toBe("1"); + }); + + it("is a no-op and returns false for a real user", () => { + jest + .spyOn(navigator, "userAgent", "get") + .mockReturnValue( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + ); + expect(SkipTargetingForBots()).toBe(false); + expect(sessionStorage.getItem(TARGETING_DONE_KEY)).toBeNull(); + }); +}); + describe("determineABTest", () => { // Mock Math.random to control the random bucket selection let originalMathRandom; diff --git a/lib/edge/targeting.ts b/lib/edge/targeting.ts index 29cc10f..9fe95ee 100644 --- a/lib/edge/targeting.ts +++ b/lib/edge/targeting.ts @@ -1,6 +1,7 @@ import type { ResolvedConfig, ABTestConfig, MatcherOverride } from "../config"; import { fetch } from "../core/network"; import { LocalStorage } from "../core/storage"; +import { isBot } from "../addons/botDetection"; import * as ortb2 from "iab-openrtb/v26"; import * as adcom from "iab-adcom"; import { sendTargetingUpdateEvent } from "../core/events/cache-refresh"; @@ -40,6 +41,8 @@ type TargetingResponse = { split_test_assignment?: string; }; +const TARGETING_DONE_KEY = "OPTABLE_TARGETING_DONE"; + // Determine which A/B test (if any) should be used for this request function determineABTest(abTests?: ABTestConfig[]): ABTestConfig | null { if (!abTests || abTests.length === 0) { @@ -117,6 +120,23 @@ function TargetingClearCache(config: ResolvedConfig) { ls.clearTargeting(); } +/** + * Skip targeting for bots by marking targeting as already done, + * so RTD short-circuits and returns null. No-op for real users. + * Returns whether the request was identified as a bot. + */ +export function SkipTargetingForBots(): boolean { + try { + if (typeof isBot === "function" && isBot()) { + sessionStorage.setItem(TARGETING_DONE_KEY, "1"); + return true; + } + } catch { + // isBot is unavailable or threw; fall through and treat as a real user. + } + return false; +} + // Prebid.js supports setting ortb2 object for compatible bidder adapters. // // We return the contents to be merged in ortb2 and passed to