From 8f47b56a20493f4d39d2023501d2cf664abca325 Mon Sep 17 00:00:00 2001 From: Jacek Wielemborek Date: Sun, 26 Oct 2025 13:23:13 +0100 Subject: [PATCH] refactor: move tests to tests/ directory and simplify verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move test_verify.py and test-content-script.js to tests/ - Remove unused test_start_extension.sh - Rename Dockerfile stages: test → code_quality, verify → integration_test - Simplify test to single assertion: 17*2+3=37 - Add red TTY output for failures - Fix runtime stage to properly copy built artifacts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- Dockerfile | 32 +- background.ts | 2 +- esbuild.config.js | 2 +- scripts/test_start_extension.sh | 84 ---- scripts/test_verify.py | 453 ------------------ .../test-content-script.js | 19 +- tests/test_verify.py | 150 ++++++ 7 files changed, 188 insertions(+), 554 deletions(-) delete mode 100644 scripts/test_start_extension.sh delete mode 100755 scripts/test_verify.py rename test-content-script.js => tests/test-content-script.js (83%) create mode 100755 tests/test_verify.py diff --git a/Dockerfile b/Dockerfile index 837e56a..73fc7a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,11 @@ COPY package.json package-lock.json ./ # Install dependencies RUN npm install +# FIXME: COPY . . invalidates cache, so we need to optionally install Firefox +# and jump back to the correct stage. It might be too complex though, so we +# either need to use build args (if cache properly) or heavily document the +# stage transitions, maybe even with a graph. + # Copy source code (respecting .dockerignore) COPY . . @@ -21,8 +26,8 @@ RUN npm run build # Create the package RUN npm run create-package -# Test stage - for running quality checks -FROM builder AS test +# Code quality stage - for running quality checks +FROM builder AS code_quality RUN npm run typecheck && npm run lint # Artifacts stage - only contains the built artifacts (for --output) @@ -38,7 +43,14 @@ FROM builder CMD ["ls", "-lh", "/app/web-ext-artifacts/"] # Runtime stage - for running extension in Firefox -FROM builder AS runtime +FROM node:lts AS runtime + +WORKDIR /app + +# Copy built artifacts from builder +COPY --from=builder /app/web-ext-artifacts /app/web-ext-artifacts +COPY --from=builder /app/package.json /app/package-lock.json ./ +COPY --from=builder /app/node_modules ./node_modules # Install Firefox and Xvfb for headless execution (cached layer) RUN apt-get update && apt-get install -y \ @@ -66,18 +78,14 @@ RUN pip3 install --break-system-packages marionette_driver # Set display for Xvfb ENV DISPLAY=:99 -# Copy startup script for extension testing -COPY scripts/test_start_extension.sh /app/test_start_extension.sh -RUN chmod +x /app/test_start_extension.sh +# Start script (not used in verify stage) +CMD ["echo", "Use verify stage for testing"] -# Start script -CMD ["/app/test_start_extension.sh"] - -# Verify stage - automated testing with exit code -FROM runtime AS verify +# Integration test stage - automated testing with exit code +FROM runtime AS integration_test # Copy verification script -COPY scripts/test_verify.py /app/test_verify.py +COPY tests/test_verify.py /app/test_verify.py RUN chmod +x /app/test_verify.py # Run verification and exit with proper exit code diff --git a/background.ts b/background.ts index 397e73d..a25475c 100644 --- a/background.ts +++ b/background.ts @@ -12,7 +12,7 @@ browser.runtime.onMessage.addListener((message: any, sender: any, sendResponse: // Perform a computation to prove the background script is running // This is not just an echo - we're doing actual processing const inputValue = message.inputValue || 0; - const computed = (inputValue * 2) + 3; // Simple but verifiable computation + const computed = (inputValue * 2) + 3; // Send back a response with computed value and metadata const response = { diff --git a/esbuild.config.js b/esbuild.config.js index 2dc4a42..e9fa453 100644 --- a/esbuild.config.js +++ b/esbuild.config.js @@ -48,7 +48,7 @@ esbuild 'components/sidebar/sidebar.tsx', 'components/report-window/report-window.tsx', 'background.ts', - 'test-content-script.js', + 'tests/test-content-script.js', 'diag.tsx', 'styles/global.scss', 'styles/fonts.scss', diff --git a/scripts/test_start_extension.sh b/scripts/test_start_extension.sh deleted file mode 100644 index 6201239..0000000 --- a/scripts/test_start_extension.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# -# test_start_extension.sh - Starts Rentgen extension in headless Firefox with verification -# -# This script is used by Docker runtime stage to: -# 1. Start Xvfb (virtual X server) on display :99 -# 2. Run web-ext with verbose logging -# 3. Verify extension was installed correctly -# 4. Verify extension code executed (by checking ABSENCE of errors) -# -# IMPORTANT: console.error from background pages does NOT appear in web-ext stdout -# (Firefox limitation). Verification works by: -# - Checking that extension installed -# - Checking NO JavaScript errors in logs -# - No errors = code executed correctly -# - -set -e - -echo "Starting Xvfb on display :99..." -Xvfb :99 -screen 0 1024x768x24 & -XVFB_PID=$! -sleep 2 - -echo "Xvfb started with PID: $XVFB_PID" -echo "Starting web-ext run with verbose logging..." -echo "========================================" - -# Run web-ext with verbose logging and capture output -npx web-ext run --verbose 2>&1 | tee /tmp/web-ext.log & -WEBEXT_PID=$! - -# Wait for extension installation confirmation -echo "Waiting for extension to install..." -for i in {1..30}; do - if grep -q "Installed /app as a temporary add-on" /tmp/web-ext.log 2>/dev/null; then - echo "========================================" - echo "✓ SUCCESS: Extension installed!" - echo "✓ Firefox is running in headless mode" - echo "✓ Extension: rentgen@internet-czas-dzialac.pl" - - # Give extension time to initialize - sleep 3 - - # CRITICAL: Check for JavaScript errors - echo "" - echo "Checking for JavaScript errors in extension code..." - - # Filter out unrelated Firefox errors (BackupService, RSLoader, etc.) - if grep -i "JavaScript error.*background.js\|SyntaxError.*background\|ReferenceError.*background" /tmp/web-ext.log 2>/dev/null | grep -v "BackupService\|RSLoader"; then - echo "" - echo "========================================" - echo "✗✗✗ CRITICAL ERROR ✗✗✗" - echo "========================================" - echo "Found JavaScript errors in background.js!" - echo "Extension installed but CODE DID NOT EXECUTE!" - echo "" - echo "Errors:" - grep -i "JavaScript error.*background.js\|SyntaxError.*background\|ReferenceError.*background" /tmp/web-ext.log 2>/dev/null | head -10 - echo "========================================" - exit 1 - else - echo "✓ NO JavaScript errors in background.js" - echo "✓ Extension code executed successfully!" - echo "" - echo "NOTE: console.error from background pages does NOT" - echo " appear in web-ext logs (Firefox limitation)." - echo " Absence of errors = proof of execution." - fi - - echo "" - echo "✓ Process info:" - ps aux | grep -E "(firefox|Xvfb)" | grep -v grep | head -3 - echo "========================================" - echo "Extension is ready and VERIFIED working." - echo "Press Ctrl+C to stop." - echo "========================================" - break - fi - sleep 1 -done - -# Keep container running and show logs -wait $WEBEXT_PID diff --git a/scripts/test_verify.py b/scripts/test_verify.py deleted file mode 100755 index 6fd1f95..0000000 --- a/scripts/test_verify.py +++ /dev/null @@ -1,453 +0,0 @@ -#!/usr/bin/env python3 -""" -test_verify.py - Verifies extension and exits (doesn't wait forever) - -This script is a Python version of test_verify.sh that: -- Starts Firefox with the extension -- Verifies extension loaded without errors -- EXITS after verification (instead of waiting forever) - -Used for automated tests in CI/Docker. -Uses guard clauses pattern for cleaner flow control. -""" - -import sys -import time -import subprocess -import os -import signal -import json -from pathlib import Path - - -def print_header(text: str) -> None: - """Print formatted header.""" - print(f"\n{text}") - - -def print_success(text: str) -> None: - """Print success message.""" - print(f"✓ {text}") - - -def print_error(text: str) -> None: - """Print error message.""" - print(f"✗ {text}") - - -def start_xvfb() -> int: - """Start Xvfb virtual X server. Returns PID.""" - print_header("Starting Xvfb on display :99...") - - xvfb = subprocess.Popen( - ["Xvfb", ":99", "-screen", "0", "1024x768x24"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL - ) - - os.environ["DISPLAY"] = ":99" - time.sleep(2) - - print_success(f"Xvfb started with PID: {xvfb.pid}") - return xvfb.pid - - -def start_webext() -> tuple[int, Path]: - """Start web-ext and log output to file. Returns (PID, log_path).""" - print_header("Starting web-ext run with verbose logging...") - print("=" * 40) - - log_path = Path("/tmp/web-ext.log") - log_file = open(log_path, "w") - - # Enable Marionette on port 2828 - # Use --arg to pass arguments to Firefox - webext = subprocess.Popen( - ["npx", "web-ext", "run", "--verbose", - "--arg=-marionette", - "--arg=--marionette-port", - "--arg=2828"], - stdout=log_file, - stderr=subprocess.STDOUT - ) - - return webext.pid, log_path - - -def wait_for_extension_install(log_path: Path, timeout_seconds: int = 60) -> bool: - """Wait for extension installation confirmation in logs.""" - print_header("Waiting for extension to install...") - - for i in range(timeout_seconds): - if not log_path.exists(): - time.sleep(1) - continue - - content = log_path.read_text() - - # Check for Firefox crashes or errors - if "firefox crashed" in content.lower() or "panic" in content.lower(): - print_error("Firefox crashed or panicked!") - print("Last 20 lines of log:") - print("\n".join(content.split("\n")[-20:])) - return False - - # Check if Marionette is mentioned - if i == 10 and "marionette" in content.lower(): - print_success("Marionette mentioned in logs") - - # Look for any of these success indicators - success_indicators = [ - "Installed /app as a temporary add-on", - "as a temporary add-on", - "Extension added", - "Installed temporary add-on", - "installed from", - "Installing manifest at" - ] - - if any(indicator in content for indicator in success_indicators): - print("=" * 40) - print_success("Extension installed!") - print_success("Firefox is running in headless mode") - print_success("Extension: rentgen@internet-czas-dzialac.pl") - - # Check for Marionette port in logs - if "2828" in content: - print_success("Marionette port 2828 found in logs") - return True - - # Print progress every 10 seconds - if i > 0 and i % 10 == 0: - print(f"Still waiting... ({i}s)") - - time.sleep(1) - - # If we timeout, print the last part of the log - print_error("Timeout! Last 30 lines of log:") - if log_path.exists(): - content = log_path.read_text() - print("\n".join(content.split("\n")[-30:])) - - return False - - -def check_javascript_errors(log_path: Path) -> list[str]: - """Check for JavaScript errors in extension code. Returns list of errors.""" - print_header("Checking for JavaScript errors in extension code...") - - content = log_path.read_text() - - # Filter out unrelated Firefox errors - error_patterns = [ - "JavaScript error.*background.js", - "SyntaxError.*background", - "ReferenceError.*background" - ] - - errors = [] - for line in content.split("\n"): - # Skip BackupService and RSLoader errors (Firefox internal) - if "BackupService" in line or "RSLoader" in line: - continue - - for pattern in error_patterns: - import re - if re.search(pattern, line, re.IGNORECASE): - errors.append(line) - break - - return errors - - -def check_content_script_marker_in_logs(log_path: Path) -> tuple[bool, str]: - """Check if background received message from content script in web-ext logs. - This proves: background script → tabs.create() → content script injection → runtime.sendMessage() → background received - Returns (success, message).""" - try: - content = log_path.read_text() - - # Look for background test marker (content script sends message to background, which logs it) - import re - pattern = r'\[RENTGEN_BACKGROUND_TEST\] Received verification from content script: (.+)' - match = re.search(pattern, content) - - if match: - try: - import json - data = json.loads(match.group(1)) - title = data.get('title', '') - timestamp = data.get('timestamp', 0) - url = data.get('url', '') - - # Verify page title contains "Example" (proves page loaded correctly) - if 'Example' in title or 'example' in title.lower(): - return True, f"Content script verified! Title: '{title}', URL: {url}, timestamp: {timestamp}" - else: - return False, f"Page title unexpected: '{title}' (expected 'Example')" - except json.JSONDecodeError: - return False, f"Found marker but couldn't parse JSON: {match.group(1)}" - - return False, "No background test marker found in logs (content script may not have executed)" - - except Exception as e: - return False, f"Log check failed: {e}" - - -def extract_debugger_port(log_path: Path) -> str | None: - """Extract debugger port from web-ext logs.""" - content = log_path.read_text() - import re - match = re.search(r"start-debugger-server (\d+)", content) - if match: - return match.group(1) - return None - - -def test_with_marionette() -> tuple[bool, str]: - """Test using Firefox Marionette (WebDriver) protocol.""" - try: - from marionette_driver.marionette import Marionette - from marionette_driver import errors - import time - - print_header("Testing with Marionette WebDriver...") - - # Try to connect to Marionette on default port - client = Marionette(host='localhost', port=2828) - - try: - client.start_session() - print_success("Connected to Marionette") - - # Stage 1: Test simple JavaScript execution - print_header("Stage 1: Testing JavaScript execution (2 + 3)...") - result = client.execute_script("return 2 + 3;") - - if result == 5: - print_success(f"JavaScript execution works! 2 + 3 = {result}") - else: - return False, f"Unexpected result from 2+3: {result}" - - # Stage 2: Navigate to example.com - print_header("Stage 2: Navigating to example.com...") - client.navigate("https://example.com") - time.sleep(3) # Wait for page to load - - title = client.title - print_success(f"Page loaded, title: {title}") - - if "Example" not in title: - return False, f"Unexpected page title: {title}" - - # Stage 3: Test bidirectional communication with extension - print_header("Stage 3: Testing bidirectional extension communication...") - - # Generate a test value for verification - test_value = 17 # Arbitrary test value - expected_result = (test_value * 2) + 3 # Should be 37 - - # Dispatch event to content script and wait for background script response - verification_result = client.execute_script(""" - const testValue = arguments[0]; - const expectedResult = arguments[1]; - - // First check if content script is injected - const injected = document.body.getAttribute('data-rentgen-injected'); - if (injected !== 'true') { - return { success: false, error: 'Content script not injected' }; - } - - // Dispatch test request to content script - document.dispatchEvent(new CustomEvent('rentgen_test_request', { - detail: { - value: testValue, - timestamp: Date.now() - } - })); - - // Wait for the full round-trip communication - return new Promise((resolve) => { - let attempts = 0; - const checkInterval = setInterval(() => { - attempts++; - - // Check if content script processed the message - const verified = document.body.getAttribute('data-rentgen-verified'); - const computed = document.body.getAttribute('data-rentgen-computed'); - const formula = document.body.getAttribute('data-rentgen-formula'); - const bgTimestamp = document.body.getAttribute('data-rentgen-background-timestamp'); - const error = document.body.getAttribute('data-rentgen-error'); - - if (verified === 'true' && computed) { - clearInterval(checkInterval); - resolve({ - success: true, - verified: true, - computed: parseInt(computed), - formula: formula, - backgroundTimestamp: bgTimestamp, - expectedResult: expectedResult, - testValue: testValue - }); - } else if (verified === 'false' || attempts > 20) { - clearInterval(checkInterval); - resolve({ - success: false, - error: error || 'Timeout waiting for background response', - attempts: attempts, - verified: verified - }); - } - }, 100); // Check every 100ms, up to 2 seconds - }); - """, script_args=[test_value, expected_result], script_timeout=3000) - - # Analyze the results - if verification_result.get('success'): - computed = verification_result.get('computed') - expected = verification_result.get('expectedResult') - formula = verification_result.get('formula', '') - - print_success("Content script successfully communicated with background!") - print_success(f"Background computed: {formula}") - - if computed == expected: - print_success(f"Computation verified! {test_value} → {computed} (expected {expected})") - return True, f"Full extension stack verified! Background computed ({test_value}*2)+3={computed}" - else: - print_error(f"Computation mismatch: got {computed}, expected {expected}") - return False, f"Background returned wrong value: {computed} != {expected}" - else: - error = verification_result.get('error', 'Unknown error') - print_error(f"Extension communication failed: {error}") - return False, f"Content script/background communication failed: {error}" - - finally: - try: - client.close() - except: - pass - - except ImportError: - return False, "marionette_driver not installed" - except ConnectionRefusedError: - return False, "Cannot connect to Marionette on port 2828 (Firefox may not have Marionette enabled)" - except Exception as e: - return False, f"Marionette test failed: {e}" - - -def cleanup(xvfb_pid: int, webext_pid: int) -> None: - """Cleanup processes.""" - try: - os.kill(webext_pid, signal.SIGTERM) - except ProcessLookupError: - pass - - try: - os.kill(xvfb_pid, signal.SIGTERM) - except ProcessLookupError: - pass - - -def main() -> int: - """Main test verification logic using guard clauses.""" - - # Start Xvfb - xvfb_pid = start_xvfb() - - # Start web-ext - webext_pid, log_path = start_webext() - - # Guard: Check if extension installed - if not wait_for_extension_install(log_path): - print_error("Extension failed to install within 60s") - cleanup(xvfb_pid, webext_pid) - return 1 - - # Give extension time to initialize - time.sleep(3) - - # Guard: Check for JavaScript errors - errors = check_javascript_errors(log_path) - if errors: - print() - print("=" * 40) - print("✗✗✗ CRITICAL ERROR ✗✗✗") - print("=" * 40) - print("Found JavaScript errors in background.js!") - print("Extension installed but CODE DID NOT EXECUTE!") - print() - print("Errors:") - for error in errors[:10]: - print(f" {error}") - print("=" * 40) - - cleanup(xvfb_pid, webext_pid) - return 1 - - print_success("NO JavaScript errors in background.js") - - # Functional test: Verify extension code execution via Marionette - print_header("Functional test: Verifying extension code execution...") - - # Give extension time to initialize - time.sleep(5) - - # Test with Marionette WebDriver - marionette_success, marionette_msg = test_with_marionette() - - # Guard: Check if Marionette tests passed - if not marionette_success: - print_error("Marionette test failed") - print_error(f"Reason: {marionette_msg}") - print_error("Extension installed but could not verify execution via Marionette") - cleanup(xvfb_pid, webext_pid) - return 1 - - print_success("Extension code VERIFIED executing!") - print() - print(f"Test results: {marionette_msg}") - print() - print("This proves UNDENIABLY:") - print(" - Firefox Marionette WebDriver works") - print(" - JavaScript execution works (2+3=5)") - print(" - Page navigation works (example.com loaded)") - print(" - Content script is injected and listening for events") - print(" - Content script sends messages to background script") - print(" - Background script receives messages and computes results") - print(" - Background script sends responses back to content script") - print(" - Full bidirectional communication verified with computed proof") - print(" - The extension is not just installed but ACTIVELY EXECUTING") - - # Show process info - print() - print_success("Process info:") - subprocess.run(["ps", "aux"], stdout=subprocess.PIPE, text=True, check=False) - result = subprocess.run( - ["bash", "-c", "ps aux | grep -E '(firefox|Xvfb)' | grep -v grep | head -3"], - capture_output=True, - text=True - ) - print(result.stdout) - - print("=" * 40) - print("Extension is VERIFIED working!") - print("=" * 40) - - # Cleanup - cleanup(xvfb_pid, webext_pid) - - print("Test completed successfully.") - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except KeyboardInterrupt: - print("\nInterrupted by user") - sys.exit(130) - except Exception as e: - print(f"Unexpected error: {e}", file=sys.stderr) - sys.exit(1) diff --git a/test-content-script.js b/tests/test-content-script.js similarity index 83% rename from test-content-script.js rename to tests/test-content-script.js index ee3023b..507b67c 100644 --- a/test-content-script.js +++ b/tests/test-content-script.js @@ -2,9 +2,22 @@ // This script proves bidirectional communication between content script and background // Set initial DOM marker to prove content script is injected -if (document.body) { - document.body.setAttribute('data-rentgen-injected', 'true'); -} + +(function() { + function setMarker() { + if (document.body) { + document.body.setAttribute('data-rentgen-injected', 'true'); + } else { + // Wait for DOM ready + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', () => { + document.body.setAttribute('data-rentgen-injected', 'true'); + }); + } + } + } + setMarker(); +})(); // Listen for test request from Marionette test script document.addEventListener('rentgen_test_request', async (event) => { diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100755 index 0000000..bb1ab1a --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +""" +test_verify.py - Minimal extension verification test + +Verifies the extension background script is executing by testing +bidirectional communication with a simple addition operation. +""" + +import sys +import time +import subprocess +import os +import signal + + +def is_tty(): + """Check if stdout is a TTY.""" + return sys.stdout.isatty() + + +def red(text): + """Return red text if TTY, otherwise plain text.""" + if is_tty(): + return f"\033[91m{text}\033[0m" + return text + + +def start_xvfb(): + """Start Xvfb virtual X server. Returns PID.""" + xvfb = subprocess.Popen( + ["Xvfb", ":99", "-screen", "0", "1024x768x24"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + os.environ["DISPLAY"] = ":99" + time.sleep(2) + return xvfb.pid + + +def start_webext(): + """Start web-ext with Marionette enabled. Returns PID.""" + webext = subprocess.Popen( + ["npx", "web-ext", "run", + "--arg=-marionette", + "--arg=--marionette-port", + "--arg=2828"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + return webext.pid + + +def test_addition(): + """Test background script via Marionette. Returns (success, result).""" + try: + from marionette_driver.marionette import Marionette + + # Wait for Firefox to start + time.sleep(10) + + # Connect to Marionette + client = Marionette(host='localhost', port=2828) + client.start_session() + + # Navigate to any page (needed for content script injection) + client.navigate("https://example.com") + time.sleep(5) + + # Test: background should compute (17 * 2) + 3 = 37 + test_value = 17 + expected = 37 + + result = client.execute_script(""" + const testValue = arguments[0]; + + // Check if content script loaded + if (!document.body.getAttribute('data-rentgen-injected')) { + return -1; // Content script not loaded + } + + // Dispatch test request to content script + document.dispatchEvent(new CustomEvent('rentgen_test_request', { + detail: { value: testValue, timestamp: Date.now() } + })); + + // Wait for background response + return new Promise((resolve) => { + let attempts = 0; + const checkInterval = setInterval(() => { + attempts++; + const computed = document.body.getAttribute('data-rentgen-computed'); + + if (computed) { + clearInterval(checkInterval); + resolve(parseInt(computed)); + } else if (attempts > 50) { + clearInterval(checkInterval); + resolve(null); + } + }, 100); + }); + """, script_args=[test_value], script_timeout=10000) + + client.close() + + if result == expected: + return True, expected + else: + return False, result + + except Exception as e: + return False, str(e) + + +def cleanup(xvfb_pid, webext_pid): + """Kill processes.""" + try: + os.kill(webext_pid, signal.SIGTERM) + except: + pass + try: + os.kill(xvfb_pid, signal.SIGTERM) + except: + pass + + +def main(): + """Main test.""" + xvfb_pid = start_xvfb() + webext_pid = start_webext() + + success, result = test_addition() + + cleanup(xvfb_pid, webext_pid) + + if not success: + print(red(f"FAIL: Expected 37, got {result}")) + return 1 + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + sys.exit(130) + except Exception as e: + print(red(f"ERROR: {e}")) + sys.exit(1)