From 43e0c6c7f86deec898f3747c15bfd952693cedac Mon Sep 17 00:00:00 2001 From: Kuba Orlik Date: Mon, 4 Oct 2021 18:51:51 +0200 Subject: [PATCH] Add README and dedupe data entries --- README.md | 7 ++++ extended-request.ts | 70 ++++++++++++++++++++++++++++++----- memory.ts | 7 +--- package-lock.json | 40 +++++++++++++++++++- package.json | 4 +- problematic.js | 8 ---- request-cluster.ts | 90 ++++++++++++++++++++------------------------- sidebar.tsx | 16 ++++---- util.ts | 3 -- 9 files changed, 160 insertions(+), 85 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..8e46ffd --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# Problematic-requests, aka ICD scanner + +Wtyczka pokazująca, jakie dane zostały ~~wykradzione~~ wysłane do podmiotów trzecich przez odwiedzane strony. + +## TODO: + +- Używać https://github.com/InteractiveAdvertisingBureau/iabtcf-es/tree/master/modules/core#iabtcfcore do wizualizacji "zgód" zebranych przez CMP-y od IAB diff --git a/extended-request.ts b/extended-request.ts index 5884459..b70ff64 100644 --- a/extended-request.ts +++ b/extended-request.ts @@ -1,12 +1,19 @@ import { StolenDataEntry } from "./request-cluster"; -import { getshorthost, Request } from "./util"; +import { getshorthost, parseCookie, Request } from "./util"; export default class ExtendedRequest { public tabId: number; public url: string; public requestHeaders: Request["requestHeaders"]; + public origin: string; + public initialized = false; - async getOrigin() { + async init() { + await this.cacheOrigin(); + this.initialized = true; + } + + async cacheOrigin(): Promise { let url: string; if (this.data.tabId && this.data.tabId >= 0) { const tab = await browser.tabs.get(this.data.tabId); @@ -14,12 +21,19 @@ export default class ExtendedRequest { } else { url = (this.data as any).frameAncestors[0].url; } - return url; + this.origin = url; } - async isThirdParty() { + getOrigin(): string { + if (!this.initialized) { + throw new Error("initialize first!!"); + } + return this.origin; + } + + isThirdParty() { const request_url = new URL(this.data.url); - const origin_url = new URL(await this.getOrigin()); + const origin_url = new URL(this.getOrigin()); if (request_url.host.includes(origin_url.host)) { return false; } @@ -37,8 +51,33 @@ export default class ExtendedRequest { .value; } - async exposesOrigin() { - return this.getReferer().includes(new URL(await this.getOrigin()).host); + exposesOrigin() { + const url = new URL(this.getOrigin()); + const host = url.host; + const path = url.pathname; + return ( + this.getReferer().includes(host) || + this.getAllStolenData().filter( + (entry) => entry.value.includes(host) || entry.value.includes(path) + ).length > 0 + ); + } + + getAllStolenData(): StolenDataEntry[] { + return [ + ...this.getPathParams(), + ...this.getCookieData(), + ...this.getQueryParams(), + ]; + } + + getCookieData(): StolenDataEntry[] { + if (!this.hasCookie() || this.getCookie() === undefined) { + return []; + } + return Object.entries(parseCookie(this.getCookie())) + .map(([key, value]) => [key, value || ""]) + .map(([key, value]) => new StolenDataEntry(this, "cookie", key, value)); } hasReferer() { @@ -49,7 +88,7 @@ export default class ExtendedRequest { return this.data.requestHeaders.some((h) => h.name === "Cookie"); } - getCookie() { + getCookie(): string { return this.requestHeaders.find((h) => h.name == "Cookie")?.value; } @@ -62,12 +101,25 @@ export default class ExtendedRequest { return path .split(";") .map((e) => e.split("=")) + .map(([key, value]) => [key, value || ""]) .map( ([key, value]) => - new StolenDataEntry("pathname", key, decodeURIComponent(value)) + new StolenDataEntry(this, "pathname", key, decodeURIComponent(value)) ); } + getQueryParams(): StolenDataEntry[] { + const url = new URL(this.data.url); + return Array.from((url.searchParams as any).entries()) + .map(([key, value]) => [key, value || ""]) + .map(([key, value]) => { + try { + value = decodeURIComponent(value); + } catch (e) {} + return new StolenDataEntry(this, "queryparams", key, value); + }); + } + constructor(public data: Request) { this.tabId = data.tabId; this.url = data.url; diff --git a/memory.ts b/memory.ts index f4eec54..bc73e09 100644 --- a/memory.ts +++ b/memory.ts @@ -6,11 +6,8 @@ import { RequestCluster } from "./request-cluster"; class Memory extends EventEmitter { tab_to_history = {} as Record>; async register(request: ExtendedRequest) { - if ( - (await request.isThirdParty()) && - request.hasReferer() && - (await request.exposesOrigin()) - ) { + await request.init(); + if (request.isThirdParty() && request.exposesOrigin()) { if (!this.tab_to_history[request.tabId]) { this.tab_to_history[request.tabId] = {}; } diff --git a/package-lock.json b/package-lock.json index da5d143..aa7f130 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,10 +9,12 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "consent-string": "^1.5.2", "esbuild": "^0.13.3", "events": "^3.3.0", "react": "^17.0.2", - "react-dom": "^17.0.2" + "react-dom": "^17.0.2", + "tai-password-strength": "^1.1.3" }, "devDependencies": { "@types/events": "^3.0.0", @@ -58,6 +60,19 @@ "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==", "dev": true }, + "node_modules/base-64": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", + "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs=" + }, + "node_modules/consent-string": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz", + "integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==", + "dependencies": { + "base-64": "^0.1.0" + } + }, "node_modules/csstype": { "version": "3.0.9", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz", @@ -349,6 +364,11 @@ "object-assign": "^4.1.1" } }, + "node_modules/tai-password-strength": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz", + "integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA==" + }, "node_modules/web-ext-types": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz", @@ -395,6 +415,19 @@ "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==", "dev": true }, + "base-64": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", + "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs=" + }, + "consent-string": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz", + "integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==", + "requires": { + "base-64": "^0.1.0" + } + }, "csstype": { "version": "3.0.9", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz", @@ -571,6 +604,11 @@ "object-assign": "^4.1.1" } }, + "tai-password-strength": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz", + "integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA==" + }, "web-ext-types": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz", diff --git a/package.json b/package.json index 995ec41..87a8032 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,12 @@ "author": "", "license": "ISC", "dependencies": { + "consent-string": "^1.5.2", "esbuild": "^0.13.3", "events": "^3.3.0", "react": "^17.0.2", - "react-dom": "^17.0.2" + "react-dom": "^17.0.2", + "tai-password-strength": "^1.1.3" }, "devDependencies": { "@types/events": "^3.0.0", diff --git a/problematic.js b/problematic.js index c239b22..c0a1dd3 100644 --- a/problematic.js +++ b/problematic.js @@ -7,7 +7,6 @@ function gethost(url) { } function getshorthost(host) { - console.log("getshort", host); return host.split(".").slice(-2).join("."); } @@ -16,11 +15,6 @@ async function isThirdParty(request) { const request_url = new URL(request.url); const origin_url = new URL(await getOrigin(request)); /* console.log(request_url.ho, origin_url, request_url.includes(origin_url)); */ - console.log( - request_url.host, - origin_url.host, - request_url.host.includes(origin_url.host) - ); if (request_url.host.includes(origin_url.host)) { return false; } @@ -85,11 +79,9 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) { if (sender.tab) { return; } - console.log("got message!", request); if (request?.msg === "get_memory") { sendResponse(memory); } else if (request?.msg === "clear_memory") { - console.log("memory cleared"); memory = {}; } }); diff --git a/request-cluster.ts b/request-cluster.ts index c8c629c..63e1ca7 100644 --- a/request-cluster.ts +++ b/request-cluster.ts @@ -1,9 +1,25 @@ import { EventEmitter } from "events"; import ExtendedRequest from "./extended-request"; -import { parseCookie } from "./util"; + +export type Sources = "cookie" | "pathname" | "queryparams"; export class StolenDataEntry { - constructor(public type: string, public name: string, public value: string) {} + constructor( + public request: ExtendedRequest, + public source: Sources, + public name: string, + public value: string + ) {} + + getPriority() { + let priority = 0; + priority += this.value.length; + const url = new URL(this.request.getOrigin()); + if (this.value.includes(url.host) || this.value.includes(url.pathname)) { + priority += 100; + } + return priority; + } } export class RequestCluster extends EventEmitter { @@ -25,56 +41,28 @@ export class RequestCluster extends EventEmitter { return false; } - getCookiesContent({ - minValueLength, - }: { - minValueLength: number; - }): StolenDataEntry[] { - this.getQueryParamsContent({ minValueLength }); - const cookieValues = new Set(); - for (const request of this.requests) { - if (request.hasCookie()) { - cookieValues.add(request.getCookie()); - } - } - return Array.from(cookieValues.values()) - .map(parseCookie) - .map((o) => Object.entries(o)) + getStolenData(filter: { minValueLength: number }): StolenDataEntry[] { + return this.requests + .map((request) => request.getAllStolenData()) .reduce((a, b) => a.concat(b), []) - .map(([key, value]) => new StolenDataEntry("cookie", key, value)) - .filter((e) => e.value.length >= minValueLength); - } - - getQueryParamsContent({ - minValueLength, - }: { - minValueLength: number; - }): StolenDataEntry[] { - const result = []; - for (const request of this.requests) { - console.log(request.data.url); - } - return result; - } - - getPathnameParamsContent({ - minValueLength, - }: { - minValueLength: number; - }): StolenDataEntry[] { - let result = []; - for (const request of this.requests) { - result = [...result, ...request.getPathParams()]; - } - console.log("PATHNAME PARAMS FOR", this.id, result); - return result; - } - - getStolenData(filter: { minValueLength: number }) { - return [ - ...this.getCookiesContent(filter), - ...this.getPathnameParamsContent(filter), - ]; + .filter((entry) => { + return entry.value.length >= filter.minValueLength; + }) + .sort((entry1, entry2) => + entry1.getPriority() > entry2.getPriority() ? -1 : 1 + ) + .filter((element, index, array) => { + // remove duplicate neighbours + if (index == 0) { + return true; + } + if ( + element.name != array[index - 1].name || + element.value != array[index - 1].value + ) { + return true; + } + }); } static sortCompare(a: RequestCluster, b: RequestCluster) { diff --git a/sidebar.tsx b/sidebar.tsx index 5c37956..6b62156 100644 --- a/sidebar.tsx +++ b/sidebar.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useState } from "react"; import ReactDOM from "react-dom"; import memory from "./memory"; -import { RequestCluster } from "./request-cluster"; +import { RequestCluster, Sources } from "./request-cluster"; import { Tab, useEmitter } from "./util"; async function getTabByID(id: number) { @@ -10,7 +10,6 @@ async function getTabByID(id: number) { } async function getCurrentTab() { - console.log("getCurrentTab"); const [tab] = await browser.tabs.query({ active: true, windowId: browser.windows.WINDOW_ID_CURRENT, @@ -27,7 +26,6 @@ const TabDropdown = ({ }) => { const [tabs, setTabs] = useState([]); useEffect(() => { - console.log("useEffect!"); browser.tabs.query({ currentWindow: true }).then(setTabs); }, []); return ( @@ -35,7 +33,6 @@ const TabDropdown = ({ id="tab_dropdown" value={pickedTab} onChange={async (e) => { - console.log(e.target.value); setPickedTab(parseInt(e.target.value)); }} > @@ -60,6 +57,11 @@ const StolenDataRow = ({ minValueLength: number; }) => { const cluster = memory.getClustersForTab(tabID)[shorthost]; + const icons: Record = { + cookie: "🍪", + pathname: "🛣", + queryparams: "🅿", + }; return (

@@ -73,7 +75,8 @@ const StolenDataRow = ({ {entry.name} - {entry.value} + {icons[entry.source]} + {entry.value} ))} @@ -140,9 +143,8 @@ const Options = ({ minValueLength, setMinValueLength }) => { }; const Sidebar = () => { - console.log("rendering!"); const [pickedTab, setPickedTab] = useState(null); - const [minValueLength, setMinValueLength] = useState(5); + const [minValueLength, setMinValueLength] = useState(7); const counter = useEmitter(memory); return ( <> diff --git a/util.ts b/util.ts index 754f51d..17a9ed4 100644 --- a/util.ts +++ b/util.ts @@ -16,11 +16,8 @@ export function getshorthost(host: string) { export function useEmitter(e: EventEmitter) { const [counter, setCounter] = useState(0); useEffect(() => { - console.log("useEmitter!"); const callback = () => { - console.log("Detected memory change!"); setCounter((counter) => counter + 1); - console.log("RT:", counter + 1); }; e.on("change", callback); return () => {