test: add Selenium WebDriver integration tests

- Add Selenium WebDriver for browser extension testing - Simplify test content script (only essential DOM attributes) - Rename test-content-script.js → inner-test-content-script.js for clarity - Add DEBUG_ prefix to test event name (DEBUG_rentgen_test_request) - Auto-restore manifest.json after tests (git checkout) - Include tests in run-checks.sh script - Update Dockerfile to use Selenium instead of Marionette - Ignore tests directory in web-ext lint 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-28 11:04:16 +00:00 · 2025-10-28 11:04:16 +00:00 · b1154c4d62
commit b1154c4d62
parent 4be9a43d5b
9 changed files with 324 additions and 100 deletions
--- a/20
+++ b/20
@ -63,7 +63,7 @@ FROM builder
 # Default command shows the built artifact
 CMD ["ls", "-lh", "/app/web-ext-artifacts/"]

-# Runtime stage - for running extension in Firefox
+# Runtime stage - for running extension with Selenium WebDriver
 FROM node:lts AS runtime

 WORKDIR /app
@ -71,16 +71,10 @@ WORKDIR /app
 # Copy built extension from test_builder (includes test code)
 COPY --from=test_builder /app /app

-# Install Firefox and Xvfb for headless execution (cached layer)
+# Install Firefox and Xvfb for headless execution
 RUN apt-get update && apt-get install -y \
    firefox-esr \
    xvfb \
-    procps \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install Python and pip in a separate layer for better caching
-RUN apt-get update && apt-get install -y \
-    python3-pip \
    wget \
    && rm -rf /var/lib/apt/lists/*

@ -91,8 +85,8 @@ RUN wget -q https://github.com/mozilla/geckodriver/releases/download/v0.34.0/gec
    && rm geckodriver-v0.34.0-linux64.tar.gz \
    && chmod +x /usr/local/bin/geckodriver

-# Install Python dependencies for testing
-RUN pip3 install --break-system-packages marionette_driver
+# Install Node.js test dependencies
+RUN npm install

 # Set display for Xvfb
 ENV DISPLAY=:99
@ -104,9 +98,9 @@ CMD ["echo", "Use verify stage for testing"]
 FROM runtime AS integration_test

 # Copy verification scripts
-COPY tests/test_verify.py /app/tests/test_verify.py
+COPY tests/test_verify.mjs /app/tests/test_verify.mjs
 COPY tests/test-lib.js /app/tests/test-lib.js
-RUN chmod +x /app/tests/test_verify.py
+RUN chmod +x /app/tests/test_verify.mjs

 # Run verification and exit with proper exit code
-CMD ["python3", "/app/tests/test_verify.py"]
+CMD ["node", "/app/tests/test_verify.mjs"]
--- a/esbuild.config.js
+++ b/esbuild.config.js
@ -56,7 +56,7 @@ const entryPoints = [
 ];

 if (ENABLE_TESTS) {
-    entryPoints.push('tests/test-content-script.js');
+    entryPoints.push('tests/inner-test-content-script.js');
 }

 esbuild
@ -90,13 +90,13 @@ esbuild

            // Check if test script is already added
            const hasTestScript = manifest.content_scripts.some(
-                cs => cs.js && cs.js.includes('lib/tests/test-content-script.js')
+                cs => cs.js && cs.js.includes('lib/tests/inner-test-content-script.js')
            );

            if (!hasTestScript) {
                manifest.content_scripts.push({
                    matches: ['<all_urls>'],
-                    js: ['lib/tests/test-content-script.js'],
+                    js: ['lib/tests/inner-test-content-script.js'],
                    run_at: 'document_start'
                });

--- a/package-lock.json
+++ b/package-lock.json
@ -25,9 +25,13 @@
                "addons-linter": "^4.7.0",
                "esbuild": "^0.14.14",
                "esbuild-plugin-sass": "^1.0.1",
+                "selenium-webdriver": "^4.38.0",
                "typescript": "^4.6.4",
                "web-ext": "^6.7.0",
                "web-ext-types": "^3.2.1"
+            },
+            "engines": {
+                "node": ">=25"
            }
        },
        "node_modules/@babel/code-frame": {
@ -145,6 +149,13 @@
                "regenerator-runtime": "^0.13.4"
            }
        },
+        "node_modules/@bazel/runfiles": {
+            "version": "6.5.0",
+            "resolved": "https://registry.npmjs.org/@bazel/runfiles/-/runfiles-6.5.0.tgz",
+            "integrity": "sha512-RzahvqTkfpY2jsDxo8YItPX+/iZ6hbiikw1YhE0bA9EKBR5Og8Pa6FHn9PO9M0zaXRVsr0GFQLKbB/0rzy9SzA==",
+            "dev": true,
+            "license": "Apache-2.0"
+        },
        "node_modules/@devicefarmer/adbkit": {
            "version": "2.11.3",
            "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit/-/adbkit-2.11.3.tgz",
@ -4130,9 +4141,9 @@
            }
        },
        "node_modules/jszip": {
-            "version": "3.10.0",
-            "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.0.tgz",
-            "integrity": "sha512-LDfVtOLtOxb9RXkYOwPyNBTQDL4eUbqahtoY6x07GiDJHwSYvn8sHHIw8wINImV3MqbMNve2gSuM1DDqEKk09Q==",
+            "version": "3.10.1",
+            "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
+            "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
            "dev": true,
            "dependencies": {
                "lie": "~3.3.0",
@ -5838,6 +5849,64 @@
                "seek-table": "bin/seek-bzip-table"
            }
        },
+        "node_modules/selenium-webdriver": {
+            "version": "4.38.0",
+            "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.38.0.tgz",
+            "integrity": "sha512-5/UXXFSQmn7FGQkbcpAqvfhzflUdMWtT7QqpEgkFD6Q6rDucxB5EUfzgjmr6JbUj30QodcW3mDXehzoeS/Vy5w==",
+            "dev": true,
+            "funding": [
+                {
+                    "type": "github",
+                    "url": "https://github.com/sponsors/SeleniumHQ"
+                },
+                {
+                    "type": "opencollective",
+                    "url": "https://opencollective.com/selenium"
+                }
+            ],
+            "license": "Apache-2.0",
+            "dependencies": {
+                "@bazel/runfiles": "^6.3.1",
+                "jszip": "^3.10.1",
+                "tmp": "^0.2.5",
+                "ws": "^8.18.3"
+            },
+            "engines": {
+                "node": ">= 20.0.0"
+            }
+        },
+        "node_modules/selenium-webdriver/node_modules/tmp": {
+            "version": "0.2.5",
+            "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz",
+            "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=14.14"
+            }
+        },
+        "node_modules/selenium-webdriver/node_modules/ws": {
+            "version": "8.18.3",
+            "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz",
+            "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=10.0.0"
+            },
+            "peerDependencies": {
+                "bufferutil": "^4.0.1",
+                "utf-8-validate": ">=5.0.2"
+            },
+            "peerDependenciesMeta": {
+                "bufferutil": {
+                    "optional": true
+                },
+                "utf-8-validate": {
+                    "optional": true
+                }
+            }
+        },
        "node_modules/semver": {
            "version": "7.3.5",
            "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
@ -7377,9 +7446,9 @@
            }
        },
        "node_modules/yargs-parser": {
-            "version": "21.0.1",
-            "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.0.1.tgz",
-            "integrity": "sha512-9BK1jFpLzJROCI5TzwZL/TU4gqjK5xiHV/RfWLOahrjAko/e4DJkRDZQXfvqAsiZzzYhgAzbgz6lg48jcm4GLg==",
+            "version": "21.1.1",
+            "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
+            "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
            "dev": true,
            "engines": {
                "node": ">=12"
@ -7506,6 +7575,12 @@
                "regenerator-runtime": "^0.13.4"
            }
        },
+        "@bazel/runfiles": {
+            "version": "6.5.0",
+            "resolved": "https://registry.npmjs.org/@bazel/runfiles/-/runfiles-6.5.0.tgz",
+            "integrity": "sha512-RzahvqTkfpY2jsDxo8YItPX+/iZ6hbiikw1YhE0bA9EKBR5Og8Pa6FHn9PO9M0zaXRVsr0GFQLKbB/0rzy9SzA==",
+            "dev": true
+        },
        "@devicefarmer/adbkit": {
            "version": "2.11.3",
            "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit/-/adbkit-2.11.3.tgz",
@ -10478,9 +10553,9 @@
            }
        },
        "jszip": {
-            "version": "3.10.0",
-            "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.0.tgz",
-            "integrity": "sha512-LDfVtOLtOxb9RXkYOwPyNBTQDL4eUbqahtoY6x07GiDJHwSYvn8sHHIw8wINImV3MqbMNve2gSuM1DDqEKk09Q==",
+            "version": "3.10.1",
+            "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
+            "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
            "dev": true,
            "requires": {
                "lie": "~3.3.0",
@ -11811,6 +11886,33 @@
                "commander": "^2.8.1"
            }
        },
+        "selenium-webdriver": {
+            "version": "4.38.0",
+            "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.38.0.tgz",
+            "integrity": "sha512-5/UXXFSQmn7FGQkbcpAqvfhzflUdMWtT7QqpEgkFD6Q6rDucxB5EUfzgjmr6JbUj30QodcW3mDXehzoeS/Vy5w==",
+            "dev": true,
+            "requires": {
+                "@bazel/runfiles": "^6.3.1",
+                "jszip": "^3.10.1",
+                "tmp": "^0.2.5",
+                "ws": "^8.18.3"
+            },
+            "dependencies": {
+                "tmp": {
+                    "version": "0.2.5",
+                    "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz",
+                    "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==",
+                    "dev": true
+                },
+                "ws": {
+                    "version": "8.18.3",
+                    "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz",
+                    "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",
+                    "dev": true,
+                    "requires": {}
+                }
+            }
+        },
        "semver": {
            "version": "7.3.5",
            "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
@ -13049,9 +13151,9 @@
            }
        },
        "yargs-parser": {
-            "version": "21.0.1",
-            "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.0.1.tgz",
-            "integrity": "sha512-9BK1jFpLzJROCI5TzwZL/TU4gqjK5xiHV/RfWLOahrjAko/e4DJkRDZQXfvqAsiZzzYhgAzbgz6lg48jcm4GLg==",
+            "version": "21.1.1",
+            "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
+            "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
            "dev": true
        },
        "yauzl": {
--- a/package.json
+++ b/package.json
@ -19,7 +19,8 @@
        "create-package:firefox": "web-ext build --overwrite-dest --artifacts-dir ../web-ext-artifacts",
        "create-package:chrome": "cd dist-chrome && 7z a -tzip ../web-ext-artifacts/rentgen-chrome-0.1.10.zip * && cd ..",
        "typecheck": "tsc --noEmit",
-        "lint": "web-ext lint"
+        "lint": "web-ext lint --ignore-files 'tests/**'",
+        "test": "ENABLE_TESTS=true npm run build && node tests/test_verify.mjs; git checkout manifest.json"
    },
    "repository": {
        "type": "git",
@ -60,6 +61,7 @@
        "addons-linter": "^4.7.0",
        "esbuild": "^0.14.14",
        "esbuild-plugin-sass": "^1.0.1",
+        "selenium-webdriver": "^4.38.0",
        "typescript": "^4.6.4",
        "web-ext": "^6.7.0",
        "web-ext-types": "^3.2.1"
--- a/tests/inner-test-content-script.js
+++ b/tests/inner-test-content-script.js
@ -0,0 +1,48 @@
+// Test content script - RUNS INSIDE THE BROWSER EXTENSION
+// Only loaded during automated testing (ENABLE_TESTS=true)
+// This script proves bidirectional communication between content script and background
+
+// Browser API polyfill for Chrome/Chromium compatibility
+if (typeof browser === 'undefined') {
+    var browser = chrome;
+}
+
+// Set initial DOM marker to prove content script is injected
+(function() {
+    function setMarker() {
+        if (document.body) {
+            document.body.setAttribute('data-rentgen-injected', 'true');
+        } else {
+            // Wait for DOM ready
+            if (document.readyState === 'loading') {
+                document.addEventListener('DOMContentLoaded', () => {
+                    document.body.setAttribute('data-rentgen-injected', 'true');
+                });
+            }
+        }
+    }
+    setMarker();
+})();
+
+// Listen for test request from test script (test-lib.js)
+document.addEventListener('DEBUG_rentgen_test_request', async (event) => {
+    try {
+        // Send message to background script to get badge count for this origin
+        // This tests real Rentgen functionality: counting third-party domains
+        const response = await browser.runtime.sendMessage({
+            type: 'RENTGEN_TEST_VERIFICATION',
+            origin: window.location.origin,
+            url: window.location.href,
+            title: document.title,
+            timestamp: event.detail?.timestamp || Date.now()
+        });
+
+        // Store the badge count in DOM for the test script to read
+        if (response && response.success) {
+            document.body.setAttribute('data-rentgen-badge-count', String(response.badgeCount));
+        }
+    } catch (error) {
+        // If there's an error, the test will timeout waiting for badge count
+        console.error('Test verification error:', error);
+    }
+});
--- a/tests/run-checks.sh
+++ b/tests/run-checks.sh
@ -10,4 +10,7 @@ npm run typecheck
 echo "Running linter..."
 npm run lint

+echo "Running tests..."
+npm run test
+
 echo "✓ All code quality checks passed!"
--- a/tests/test-content-script.js
+++ b/tests/test-content-script.js
@ -1,70 +0,0 @@
-// Test content script - only for automated testing
-// This script proves bidirectional communication between content script and background
-
-// Set initial DOM marker to prove content script is injected
-
-(function() {
-    function setMarker() {
-        if (document.body) {
-            document.body.setAttribute('data-rentgen-injected', 'true');
-        } else {
-            // Wait for DOM ready
-            if (document.readyState === 'loading') {
-                document.addEventListener('DOMContentLoaded', () => {
-                    document.body.setAttribute('data-rentgen-injected', 'true');
-                });
-            }
-        }
-    }
-    setMarker();
-})();
-
-// Listen for test request from Marionette test script
-document.addEventListener('rentgen_test_request', async (event) => {
-    try {
-        // Mark that we received the event
-        document.body.setAttribute('data-rentgen-event-received', 'true');
-
-        // Extract test data from event
-        const testData = event.detail || {};
-        const timestamp = testData.timestamp || Date.now();
-
-        // Send message to background script to get badge count for this origin
-        // This tests real Rentgen functionality: counting third-party domains
-        const response = await browser.runtime.sendMessage({
-            type: 'RENTGEN_TEST_VERIFICATION',
-            origin: window.location.origin,
-            url: window.location.href,
-            title: document.title,
-            timestamp: timestamp
-        });
-
-        // Store the response from background in DOM
-        // This provides the badge count (number of third-party domains)
-        if (response && response.success) {
-            document.body.setAttribute('data-rentgen-verified', 'true');
-            document.body.setAttribute('data-rentgen-badge-count', String(response.badgeCount));
-            document.body.setAttribute('data-rentgen-origin', response.origin);
-            document.body.setAttribute('data-rentgen-cluster-ids', JSON.stringify(response.clusterIds));
-            document.body.setAttribute('data-rentgen-background-timestamp', String(response.backgroundTimestamp));
-
-            // Also dispatch a custom event with the results
-            document.dispatchEvent(new CustomEvent('rentgen_test_complete', {
-                detail: {
-                    success: true,
-                    badgeCount: response.badgeCount,
-                    origin: response.origin,
-                    clusterIds: response.clusterIds,
-                    backgroundTimestamp: response.backgroundTimestamp
-                }
-            }));
-        } else {
-            document.body.setAttribute('data-rentgen-verified', 'false');
-            document.body.setAttribute('data-rentgen-error', 'No response from background');
-        }
-    } catch (error) {
-        // Store error in DOM for debugging
-        document.body.setAttribute('data-rentgen-verified', 'false');
-        document.body.setAttribute('data-rentgen-error', String(error));
-    }
-});
--- a/tests/test-lib.js
+++ b/tests/test-lib.js
@ -32,7 +32,7 @@ async function testBadgeCount() {
    }

    // Dispatch test request to content script
-    document.dispatchEvent(new CustomEvent('rentgen_test_request', {
+    document.dispatchEvent(new CustomEvent('DEBUG_rentgen_test_request', {
        detail: { timestamp: Date.now() }
    }));

--- a/tests/test_verify.mjs
+++ b/tests/test_verify.mjs
@ -0,0 +1,145 @@
+#!/usr/bin/env node
+/**
+ * test_verify.mjs - Extension badge count verification test using Selenium WebDriver
+ *
+ * Verifies Rentgen's core functionality: counting third-party domains.
+ * Tests two scenarios:
+ * 1. Site without third-party domains (news.ycombinator.com) → badge = 0
+ * 2. Site with third-party domains (pudelek.pl) → badge > 0
+ */
+
+import { Builder, By, until } from 'selenium-webdriver';
+import firefox from 'selenium-webdriver/firefox.js';
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const isTTY = process.stdout.isTTY;
+
+function red(text) {
+    return isTTY ? `\x1b[91m${text}\x1b[0m` : text;
+}
+
+async function testBadgeCount() {
+    let driver = null;
+
+    try {
+        // Load test library
+        const testLibPath = join(__dirname, 'test-lib.js');
+        const testLib = readFileSync(testLibPath, 'utf-8');
+
+        // Find extension path (project root contains manifest.json and built files in lib/)
+        const extensionPath = join(__dirname, '..');
+
+        console.log('  Launching Firefox with extension...');
+
+        // Set up Firefox options
+        const options = new firefox.Options();
+
+        // Set headless mode based on environment
+        if (process.env.HEADLESS !== 'false') {
+            options.addArguments('-headless');
+        }
+
+        // Create WebDriver instance
+        driver = await new Builder()
+            .forBrowser('firefox')
+            .setFirefoxOptions(options)
+            .build();
+
+        // Install the extension from the dist directory
+        // Firefox WebDriver allows installing unpacked extensions
+        console.log('  Installing extension...');
+        await driver.installAddon(extensionPath, true); // true = temporary install
+
+        const results = {};
+
+        // Test 1: Site without third-party domains
+        console.log('  Testing news.ycombinator.com (expected: 0 third-party domains)...');
+        await driver.get('https://news.ycombinator.com');
+
+        // Wait for page to load
+        await driver.sleep(5000);
+
+        const badgeCountHN = await driver.executeAsyncScript(`
+            const testLibCode = arguments[0];
+            const callback = arguments[1];
+
+            eval(testLibCode);
+            testBadgeCount().then(callback).catch(err => callback(null));
+        `, testLib);
+
+        results.hn = badgeCountHN;
+        console.log(`    → Badge count: ${badgeCountHN}`);
+
+        // Test 2: Site with third-party domains
+        console.log('  Testing pudelek.pl (expected: >0 third-party domains)...');
+        await driver.get('https://pudelek.pl');
+
+        // Wait longer for tracking detection
+        await driver.sleep(10000);
+
+        const badgeCountPudelek = await driver.executeAsyncScript(`
+            const testLibCode = arguments[0];
+            const callback = arguments[1];
+
+            eval(testLibCode);
+            testBadgeCount().then(callback).catch(err => callback(null));
+        `, testLib);
+
+        results.pudelek = badgeCountPudelek;
+        console.log(`    → Badge count: ${badgeCountPudelek}`);
+
+        await driver.quit();
+
+        // Verify results
+        if (badgeCountHN === null || badgeCountPudelek === null) {
+            return { success: false, error: 'Test timed out or failed to get badge count' };
+        }
+
+        if (badgeCountHN === 0 && badgeCountPudelek > 0) {
+            return { success: true, results };
+        } else {
+            return {
+                success: false,
+                error: `Unexpected results: HN=${badgeCountHN} (expected 0), Pudelek=${badgeCountPudelek} (expected >0)`
+            };
+        }
+
+    } catch (error) {
+        if (driver) {
+            await driver.quit();
+        }
+        return { success: false, error: error.message };
+    }
+}
+
+async function main() {
+    console.log('Starting Rentgen badge count verification test...');
+    console.log('');
+
+    const result = await testBadgeCount();
+
+    console.log('');
+    if (!result.success) {
+        console.log(red(`FAIL: ${result.error}`));
+        process.exit(1);
+    }
+
+    console.log('PASS: Badge count test succeeded!');
+    console.log(`  - news.ycombinator.com: ${result.results.hn} third-party domains`);
+    console.log(`  - pudelek.pl: ${result.results.pudelek} third-party domains`);
+    process.exit(0);
+}
+
+// Handle signals
+process.on('SIGINT', () => process.exit(130));
+process.on('SIGTERM', () => process.exit(143));
+
+main().catch(error => {
+    console.error(red(`ERROR: ${error.message}`));
+    process.exit(1);
+});