From 7f8af9322f4a7f9225c1fca983f47644708fad51 Mon Sep 17 00:00:00 2001 From: Kernalsmelly Date: Sat, 23 May 2026 16:32:55 -0700 Subject: [PATCH] Improve Chromium fingerprint injection consistency --- .../src/fingerprint-injector.ts | 136 ++++++++++++++-- packages/fingerprint-injector/src/utils.js | 22 ++- .../fingerprint-injector.test.ts | 150 +++++++++++++++++- 3 files changed, 287 insertions(+), 21 deletions(-) diff --git a/packages/fingerprint-injector/src/fingerprint-injector.ts b/packages/fingerprint-injector/src/fingerprint-injector.ts index 41a0353d..11bd623a 100644 --- a/packages/fingerprint-injector/src/fingerprint-injector.ts +++ b/packages/fingerprint-injector/src/fingerprint-injector.ts @@ -19,6 +19,64 @@ interface EnhancedFingerprint extends Fingerprint { historyLength: number; } +function getChromiumCompatibleFingerprintOptions( + options: Partial | undefined, +): Partial { + return { + httpVersion: '1', + ...options, + }; +} + +function quoteClientHintValue(value: string): string { + return `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`; +} + +function inferPlatformClientHint(platform: string): string { + if (/win/i.test(platform)) return 'Windows'; + if (/mac/i.test(platform)) return 'macOS'; + if (/iphone|ipad|ios/i.test(platform)) return 'iOS'; + if (/android/i.test(platform)) return 'Android'; + if (/linux/i.test(platform)) return 'Linux'; + + return platform; +} + +function getChromiumClientHintHeaders( + fingerprint: Fingerprint, +): Record { + const { userAgent, userAgentData, platform } = fingerprint.navigator; + const brands = + userAgentData?.brands?.length > 0 + ? userAgentData.brands + : (() => { + const majorVersion = + userAgent.match(/(?:Chrome|Chromium)\/(\d+)/)?.[1] ?? ''; + if (!majorVersion) return []; + + return [ + { brand: 'Google Chrome', version: majorVersion }, + { brand: 'Chromium', version: majorVersion }, + { brand: 'Not.A/Brand', version: '8' }, + ]; + })(); + + if (brands.length === 0) return {}; + + return { + 'sec-ch-ua': brands + .map( + ({ brand, version }) => + `${quoteClientHintValue(brand)};v=${quoteClientHintValue(version)}`, + ) + .join(', '), + 'sec-ch-ua-mobile': userAgentData?.mobile ? '?1' : '?0', + 'sec-ch-ua-platform': quoteClientHintValue( + userAgentData?.platform || inferPlatformClientHint(platform), + ), + }; +} + declare function overrideInstancePrototype( instance: T, overrideObj: Partial, @@ -61,7 +119,9 @@ export class FingerprintInjector { browserName?: string, ): Record { const requestHeaders = [ + 'connection', 'accept-encoding', + 'accept-language', 'accept', 'cache-control', 'pragma', @@ -74,14 +134,20 @@ export class FingerprintInjector { const filteredHeaders = { ...headers }; - requestHeaders.forEach((header) => { - delete filteredHeaders[header]; + Object.keys(filteredHeaders).forEach((header) => { + if (requestHeaders.includes(header.toLowerCase())) { + delete filteredHeaders[header]; + } }); // Chromium-based controlled browsers do not support `te` header. // Probably needs more investigation, but for now, we can just remove it. if (!(browserName?.toLowerCase().includes('firefox') ?? false)) { - delete filteredHeaders.te; + Object.keys(filteredHeaders).forEach((header) => { + if (header.toLowerCase() === 'te') { + delete filteredHeaders[header]; + } + }); } return filteredHeaders; @@ -107,6 +173,9 @@ export class FingerprintInjector { const browserName = browserContext.browser()?.browserType().name(); await browserContext.setExtraHTTPHeaders({ + ...(browserName === 'chromium' + ? getChromiumClientHintHeaders(fingerprint) + : {}), ...this.onlyInjectableHeaders(headers, browserName), ...Object.fromEntries( // @ts-expect-error Accessing private property @@ -162,9 +231,10 @@ export class FingerprintInjector { deviceScaleFactor: screen.devicePixelRatio, }); - await page.setExtraHTTPHeaders( - this.onlyInjectableHeaders(headers, browserVersion), - ); + await page.setExtraHTTPHeaders({ + ...getChromiumClientHintHeaders(fingerprint), + ...this.onlyInjectableHeaders(headers, browserVersion), + }); await page.emulateMediaFeatures([ { name: 'prefers-color-scheme', value: 'dark' }, @@ -293,16 +363,44 @@ export class FingerprintInjector { } private _enhanceFingerprint(fingerprint: Fingerprint): EnhancedFingerprint { - const { navigator, ...rest } = fingerprint; + const { navigator, screen, ...rest } = fingerprint; return { ...rest, + screen: this._normalizeScreenDimensions(screen), navigator, userAgent: navigator.userAgent, historyLength: this._randomInRange(2, 6), }; } + private _normalizeScreenDimensions( + screen: Fingerprint['screen'], + ): Fingerprint['screen'] { + const normalizedScreen = { ...screen }; + const topLevelDimensions = { + outerWidth: 'width', + outerHeight: 'height', + innerWidth: 'width', + innerHeight: 'height', + clientWidth: 'width', + clientHeight: 'height', + } as const; + + for (const [dimension, fallback] of Object.entries( + topLevelDimensions, + ) as [ + keyof typeof topLevelDimensions, + (typeof topLevelDimensions)[keyof typeof topLevelDimensions], + ][]) { + if (normalizedScreen[dimension] <= 1) { + normalizedScreen[dimension] = normalizedScreen[fallback]; + } + } + + return normalizedScreen; + } + /** * Loads the contents of the `utils.js` file, which contains the helper functions for the fingerprinting script. * @@ -339,12 +437,19 @@ export async function newInjectedContext( const generator = new FingerprintGenerator(); const fingerprintWithHeaders = options?.fingerprint ?? - generator.getFingerprint(options?.fingerprintOptions); + generator.getFingerprint( + browser.browserType().name() === 'chromium' + ? getChromiumCompatibleFingerprintOptions( + options?.fingerprintOptions, + ) + : options?.fingerprintOptions, + ); - const { fingerprint, headers } = fingerprintWithHeaders; + const { fingerprint } = fingerprintWithHeaders; const context = await browser.newContext({ userAgent: fingerprint.navigator.userAgent, colorScheme: 'dark', + locale: fingerprint.navigator.language, ...options?.newContextOptions, viewport: { width: fingerprint.screen.width, @@ -352,7 +457,9 @@ export async function newInjectedContext( ...options?.newContextOptions?.viewport, }, extraHTTPHeaders: { - 'accept-language': headers['accept-language'], + ...(browser.browserType().name() === 'chromium' + ? getChromiumClientHintHeaders(fingerprint) + : {}), ...options?.newContextOptions?.extraHTTPHeaders, }, }); @@ -374,9 +481,16 @@ export async function newInjectedPage( }, ): Promise { const generator = new FingerprintGenerator(); + const browserVersion = await browser.version(); const fingerprintWithHeaders = options?.fingerprint ?? - generator.getFingerprint(options?.fingerprintOptions); + generator.getFingerprint( + browserVersion.toLowerCase().includes('chrome') + ? getChromiumCompatibleFingerprintOptions( + options?.fingerprintOptions, + ) + : options?.fingerprintOptions, + ); const page = await browser.newPage(); diff --git a/packages/fingerprint-injector/src/utils.js b/packages/fingerprint-injector/src/utils.js index d86b9e98..85c80c53 100644 --- a/packages/fingerprint-injector/src/utils.js +++ b/packages/fingerprint-injector/src/utils.js @@ -1,6 +1,4 @@ /* eslint-disable no-unused-vars */ -const isHeadlessChromium = - /headless/i.test(navigator.userAgent) && navigator.plugins.length === 0; const isChrome = navigator.userAgent.includes('Chrome'); const isFirefox = navigator.userAgent.includes('Firefox'); const isSafari = @@ -779,9 +777,27 @@ function fixPluginArray() { }); } +function fixWebdriver() { + const navigatorPrototype = Object.getPrototypeOf(navigator); + const webdriverDescriptor = Object.getOwnPropertyDescriptor( + navigatorPrototype, + 'webdriver', + ); + + if (webdriverDescriptor?.configurable) { + delete navigatorPrototype.webdriver; + } +} + function runHeadlessFixes() { try { - if (isHeadlessChromium) { + const hasHeadlessChromiumSignals = + /headless/i.test(navigator.userAgent) || + !window.chrome || + navigator.plugins.length === 0; + + if (isChrome && hasHeadlessChromiumSignals) { + fixWebdriver(); fixWindowChrome(); fixPermissions(); fixIframeContentWindow(); diff --git a/test/fingerprint-injector/fingerprint-injector.test.ts b/test/fingerprint-injector/fingerprint-injector.test.ts index fc0cf036..96ed4c16 100644 --- a/test/fingerprint-injector/fingerprint-injector.test.ts +++ b/test/fingerprint-injector/fingerprint-injector.test.ts @@ -20,6 +20,7 @@ import { beforeEach, afterAll, beforeAll, + vi, } from 'vitest'; function getHTTPBinUrl() { @@ -133,6 +134,148 @@ describe('FingerprintInjector', () => { } }); + test('filters request headers case-insensitively', () => { + // eslint-disable-next-line dot-notation + const onlyInjectable = + fpInjector['onlyInjectableHeaders'].bind(fpInjector); + + expect( + onlyInjectable( + { + Accept: 'text/html', + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Language': 'en-US', + Connection: 'keep-alive', + 'Sec-Fetch-Dest': 'document', + TE: 'trailers', + 'User-Agent': 'Mozilla/5.0', + 'X-Keep': '1', + }, + 'chromium', + ), + ).toEqual({ + 'User-Agent': 'Mozilla/5.0', + 'X-Keep': '1', + }); + }); + + test('Playwright helper uses native locale for HTTP/1 fingerprints', async () => { + const fingerprintWithHttp1Headers = { + fingerprint: { + navigator: { + language: 'en-US', + userAgent: 'Mozilla/5.0', + }, + screen: { + width: 1280, + height: 720, + }, + } as Fingerprint, + headers: { + 'Accept-Language': 'en-US', + 'User-Agent': 'Mozilla/5.0', + }, + }; + + const browser = { + browserType: () => ({ name: () => 'chromium' }), + newContext: vi.fn(), + }; + const context = { + _options: { extraHTTPHeaders: [] }, + addInitScript: vi.fn().mockResolvedValue(undefined), + browser: () => browser, + on: vi.fn(), + setExtraHTTPHeaders: vi.fn().mockResolvedValue(undefined), + }; + browser.newContext.mockResolvedValue(context); + + await newInjectedContext(browser as unknown as PWBrowser, { + fingerprint: fingerprintWithHttp1Headers, + }); + + expect(browser.newContext).toHaveBeenCalledWith( + expect.objectContaining({ + locale: 'en-US', + extraHTTPHeaders: {}, + }), + ); + }); + + test('Playwright helper defaults Chromium-generated fingerprints to HTTP/1 headers', async () => { + const browser = { + browserType: () => ({ name: () => 'chromium' }), + newContext: vi.fn(), + }; + const context = { + _options: { extraHTTPHeaders: [] }, + addInitScript: vi.fn().mockResolvedValue(undefined), + browser: () => browser, + on: vi.fn(), + setExtraHTTPHeaders: vi.fn().mockResolvedValue(undefined), + }; + browser.newContext.mockResolvedValue(context); + + await newInjectedContext(browser as unknown as PWBrowser, { + fingerprintOptions: { + browsers: ['chrome'], + devices: ['desktop'], + operatingSystems: ['linux'], + }, + }); + + expect(context.setExtraHTTPHeaders).toHaveBeenCalledWith( + expect.objectContaining({ + 'sec-ch-ua': expect.stringContaining('Chromium'), + }), + ); + expect(context.setExtraHTTPHeaders.mock.calls[0][0]).not.toHaveProperty( + 'Accept-Language', + ); + expect(context.setExtraHTTPHeaders.mock.calls[0][0]).not.toHaveProperty( + 'accept-language', + ); + expect( + context.setExtraHTTPHeaders.mock.calls[0][0]['sec-ch-ua'], + ).not.toContain('Headless'); + }); + + test('Playwright helper applies headless Chromium fixes after user agent spoofing', async () => { + const browser = await chromium.launch(); + + try { + const context = await newInjectedContext(browser, { + fingerprintOptions: { + browsers: ['chrome'], + devices: ['desktop'], + operatingSystems: ['linux'], + }, + }); + const page = await context.newPage(); + await page.goto(`file://${__dirname}/test.html`, { + waitUntil: 'commit', + }); + + const result = await page.evaluate(() => ({ + chrome: Boolean(window.chrome), + outerHeight, + outerWidth, + plugins: navigator.plugins.length, + webdriverPresent: 'webdriver' in navigator, + })); + + expect(result.chrome).toBe(true); + expect(result.outerHeight).toBeGreaterThan(1); + expect(result.outerWidth).toBeGreaterThan(1); + expect(result.plugins).toBeGreaterThan(0); + expect(result.webdriverPresent).toBe(false); + + await context.close(); + } finally { + await browser.close(); + } + }); + // @ts-expect-error test only describe.each(cases)('%s', (frameworkName, testCases) => { // @ts-expect-error test only @@ -410,13 +553,6 @@ describe('FingerprintInjector', () => { }); test('should override locales', async () => { - response = await page.goto(`https://crawlee.dev`); - const requestHeaders = response.request().headers(); - - expect( - requestHeaders['accept-language']?.includes('cs'), - ).toBe(true); - const { navigator: { language }, } = fingerprint;