Add README and dedupe data entries

README.md

@@ -0,0 +1,7 @@
+# Problematic-requests, aka ICD scanner
+
+Wtyczka pokazująca, jakie dane zostały ~~wykradzione~~ wysłane do podmiotów trzecich przez odwiedzane strony.
+
+## TODO:
+
+- Używać https://github.com/InteractiveAdvertisingBureau/iabtcf-es/tree/master/modules/core#iabtcfcore do wizualizacji "zgód" zebranych przez CMP-y od IAB

extended-request.ts

@@ -1,12 +1,19 @@
 import { StolenDataEntry } from "./request-cluster";
-import { getshorthost, Request } from "./util";
+import { getshorthost, parseCookie, Request } from "./util";
 
 export default class ExtendedRequest {
   public tabId: number;
   public url: string;
   public requestHeaders: Request["requestHeaders"];
+  public origin: string;
+  public initialized = false;
 
-  async getOrigin() {
+  async init() {
+    await this.cacheOrigin();
+    this.initialized = true;
+  }
+
+  async cacheOrigin(): Promise<void> {
     let url: string;
     if (this.data.tabId && this.data.tabId >= 0) {
       const tab = await browser.tabs.get(this.data.tabId);
@@ -14,12 +21,19 @@ export default class ExtendedRequest {
     } else {
       url = (this.data as any).frameAncestors[0].url;
     }
-    return url;
+    this.origin = url;
   }
 
-  async isThirdParty() {
+  getOrigin(): string {
+    if (!this.initialized) {
+      throw new Error("initialize first!!");
+    }
+    return this.origin;
+  }
+
+  isThirdParty() {
     const request_url = new URL(this.data.url);
-    const origin_url = new URL(await this.getOrigin());
+    const origin_url = new URL(this.getOrigin());
     if (request_url.host.includes(origin_url.host)) {
       return false;
     }
@@ -37,8 +51,33 @@ export default class ExtendedRequest {
       .value;
   }
 
-  async exposesOrigin() {
+  exposesOrigin() {
-    return this.getReferer().includes(new URL(await this.getOrigin()).host);
+    const url = new URL(this.getOrigin());
+    const host = url.host;
+    const path = url.pathname;
+    return (
+      this.getReferer().includes(host) ||
+      this.getAllStolenData().filter(
+        (entry) => entry.value.includes(host) || entry.value.includes(path)
+      ).length > 0
+    );
+  }
+
+  getAllStolenData(): StolenDataEntry[] {
+    return [
+      ...this.getPathParams(),
+      ...this.getCookieData(),
+      ...this.getQueryParams(),
+    ];
+  }
+
+  getCookieData(): StolenDataEntry[] {
+    if (!this.hasCookie() || this.getCookie() === undefined) {
+      return [];
+    }
+    return Object.entries(parseCookie(this.getCookie()))
+      .map(([key, value]) => [key, value || ""])
+      .map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
   }
 
   hasReferer() {
@@ -49,7 +88,7 @@ export default class ExtendedRequest {
     return this.data.requestHeaders.some((h) => h.name === "Cookie");
   }
 
-  getCookie() {
+  getCookie(): string {
     return this.requestHeaders.find((h) => h.name == "Cookie")?.value;
   }
 
@@ -62,12 +101,25 @@ export default class ExtendedRequest {
     return path
       .split(";")
       .map((e) => e.split("="))
+      .map(([key, value]) => [key, value || ""])
       .map(
         ([key, value]) =>
-          new StolenDataEntry("pathname", key, decodeURIComponent(value))
+          new StolenDataEntry(this, "pathname", key, decodeURIComponent(value))
       );
   }
 
+  getQueryParams(): StolenDataEntry[] {
+    const url = new URL(this.data.url);
+    return Array.from((url.searchParams as any).entries())
+      .map(([key, value]) => [key, value || ""])
+      .map(([key, value]) => {
+        try {
+          value = decodeURIComponent(value);
+        } catch (e) {}
+        return new StolenDataEntry(this, "queryparams", key, value);
+      });
+  }
+
   constructor(public data: Request) {
     this.tabId = data.tabId;
     this.url = data.url;

memory.ts

@@ -6,11 +6,8 @@ import { RequestCluster } from "./request-cluster";
 class Memory extends EventEmitter {
   tab_to_history = {} as Record<string, Record<string, RequestCluster>>;
   async register(request: ExtendedRequest) {
-    if (
+    await request.init();
-      (await request.isThirdParty()) &&
+    if (request.isThirdParty() && request.exposesOrigin()) {
-      request.hasReferer() &&
-      (await request.exposesOrigin())
-    ) {
       if (!this.tab_to_history[request.tabId]) {
         this.tab_to_history[request.tabId] = {};
       }

package-lock.json

@@ -9,10 +9,12 @@
       "version": "1.0.0",
       "license": "ISC",
       "dependencies": {
+        "consent-string": "^1.5.2",
         "esbuild": "^0.13.3",
         "events": "^3.3.0",
         "react": "^17.0.2",
-        "react-dom": "^17.0.2"
+        "react-dom": "^17.0.2",
+        "tai-password-strength": "^1.1.3"
       },
       "devDependencies": {
         "@types/events": "^3.0.0",
@@ -58,6 +60,19 @@
       "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
       "dev": true
     },
+    "node_modules/base-64": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
+      "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
+    },
+    "node_modules/consent-string": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
+      "integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
+      "dependencies": {
+        "base-64": "^0.1.0"
+      }
+    },
     "node_modules/csstype": {
       "version": "3.0.9",
       "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
@@ -349,6 +364,11 @@
         "object-assign": "^4.1.1"
       }
     },
+    "node_modules/tai-password-strength": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
+      "integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
+    },
     "node_modules/web-ext-types": {
       "version": "3.2.1",
       "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",
@@ -395,6 +415,19 @@
       "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
       "dev": true
     },
+    "base-64": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
+      "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
+    },
+    "consent-string": {
+      "version": "1.5.2",
+      "resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
+      "integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
+      "requires": {
+        "base-64": "^0.1.0"
+      }
+    },
     "csstype": {
       "version": "3.0.9",
       "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
@@ -571,6 +604,11 @@
         "object-assign": "^4.1.1"
       }
     },
+    "tai-password-strength": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
+      "integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
+    },
     "web-ext-types": {
       "version": "3.2.1",
       "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",

package.json

@@ -14,10 +14,12 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "consent-string": "^1.5.2",
     "esbuild": "^0.13.3",
     "events": "^3.3.0",
     "react": "^17.0.2",
-    "react-dom": "^17.0.2"
+    "react-dom": "^17.0.2",
+    "tai-password-strength": "^1.1.3"
   },
   "devDependencies": {
     "@types/events": "^3.0.0",

problematic.js

@@ -7,7 +7,6 @@ function gethost(url) {
 }
 
 function getshorthost(host) {
-  console.log("getshort", host);
   return host.split(".").slice(-2).join(".");
 }
 
@@ -16,11 +15,6 @@ async function isThirdParty(request) {
   const request_url = new URL(request.url);
   const origin_url = new URL(await getOrigin(request));
   /* console.log(request_url.ho, origin_url, request_url.includes(origin_url)); */
-  console.log(
-    request_url.host,
-    origin_url.host,
-    request_url.host.includes(origin_url.host)
-  );
   if (request_url.host.includes(origin_url.host)) {
     return false;
   }
@@ -85,11 +79,9 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
   if (sender.tab) {
     return;
   }
-  console.log("got message!", request);
   if (request?.msg === "get_memory") {
     sendResponse(memory);
   } else if (request?.msg === "clear_memory") {
-    console.log("memory cleared");
     memory = {};
   }
 });

request-cluster.ts

@@ -1,9 +1,25 @@
 import { EventEmitter } from "events";
 import ExtendedRequest from "./extended-request";
-import { parseCookie } from "./util";
+
+export type Sources = "cookie" | "pathname" | "queryparams";
 
 export class StolenDataEntry {
-  constructor(public type: string, public name: string, public value: string) {}
+  constructor(
+    public request: ExtendedRequest,
+    public source: Sources,
+    public name: string,
+    public value: string
+  ) {}
+
+  getPriority() {
+    let priority = 0;
+    priority += this.value.length;
+    const url = new URL(this.request.getOrigin());
+    if (this.value.includes(url.host) || this.value.includes(url.pathname)) {
+      priority += 100;
+    }
+    return priority;
+  }
 }
 
 export class RequestCluster extends EventEmitter {
@@ -25,56 +41,28 @@ export class RequestCluster extends EventEmitter {
     return false;
   }
 
-  getCookiesContent({
+  getStolenData(filter: { minValueLength: number }): StolenDataEntry[] {
-    minValueLength,
+    return this.requests
-  }: {
+      .map((request) => request.getAllStolenData())
-    minValueLength: number;
-  }): StolenDataEntry[] {
-    this.getQueryParamsContent({ minValueLength });
-    const cookieValues = new Set<string>();
-    for (const request of this.requests) {
-      if (request.hasCookie()) {
-        cookieValues.add(request.getCookie());
-      }
-    }
-    return Array.from(cookieValues.values())
-      .map(parseCookie)
-      .map((o) => Object.entries(o))
       .reduce((a, b) => a.concat(b), [])
-      .map(([key, value]) => new StolenDataEntry("cookie", key, value))
+      .filter((entry) => {
-      .filter((e) => e.value.length >= minValueLength);
+        return entry.value.length >= filter.minValueLength;
-  }
+      })
-
+      .sort((entry1, entry2) =>
-  getQueryParamsContent({
+        entry1.getPriority() > entry2.getPriority() ? -1 : 1
-    minValueLength,
+      )
-  }: {
+      .filter((element, index, array) => {
-    minValueLength: number;
+        // remove duplicate neighbours
-  }): StolenDataEntry[] {
+        if (index == 0) {
-    const result = [];
+          return true;
-    for (const request of this.requests) {
+        }
-      console.log(request.data.url);
+        if (
-    }
+          element.name != array[index - 1].name ||
-    return result;
+          element.value != array[index - 1].value
-  }
+        ) {
-
+          return true;
-  getPathnameParamsContent({
+        }
-    minValueLength,
+      });
-  }: {
-    minValueLength: number;
-  }): StolenDataEntry[] {
-    let result = [];
-    for (const request of this.requests) {
-      result = [...result, ...request.getPathParams()];
-    }
-    console.log("PATHNAME PARAMS FOR", this.id, result);
-    return result;
-  }
-
-  getStolenData(filter: { minValueLength: number }) {
-    return [
-      ...this.getCookiesContent(filter),
-      ...this.getPathnameParamsContent(filter),
-    ];
   }
 
   static sortCompare(a: RequestCluster, b: RequestCluster) {

sidebar.tsx

@@ -1,7 +1,7 @@
 import React, { useEffect, useState } from "react";
 import ReactDOM from "react-dom";
 import memory from "./memory";
-import { RequestCluster } from "./request-cluster";
+import { RequestCluster, Sources } from "./request-cluster";
 import { Tab, useEmitter } from "./util";
 
 async function getTabByID(id: number) {
@@ -10,7 +10,6 @@ async function getTabByID(id: number) {
 }
 
 async function getCurrentTab() {
-  console.log("getCurrentTab");
   const [tab] = await browser.tabs.query({
     active: true,
     windowId: browser.windows.WINDOW_ID_CURRENT,
@@ -27,7 +26,6 @@ const TabDropdown = ({
 }) => {
   const [tabs, setTabs] = useState([]);
   useEffect(() => {
-    console.log("useEffect!");
     browser.tabs.query({ currentWindow: true }).then(setTabs);
   }, []);
   return (
@@ -35,7 +33,6 @@ const TabDropdown = ({
       id="tab_dropdown"
       value={pickedTab}
       onChange={async (e) => {
-        console.log(e.target.value);
         setPickedTab(parseInt(e.target.value));
       }}
     >
@@ -60,6 +57,11 @@ const StolenDataRow = ({
   minValueLength: number;
 }) => {
   const cluster = memory.getClustersForTab(tabID)[shorthost];
+  const icons: Record<Sources, string> = {
+    cookie: "🍪",
+    pathname: "🛣",
+    queryparams: "🅿",
+  };
   return (
     <div>
       <h2>
@@ -73,7 +75,8 @@ const StolenDataRow = ({
               <th style={{ maxWidth: "200px", wordWrap: "break-word" }}>
                 {entry.name}
               </th>
-              <td>{entry.value}</td>
+              <td>{icons[entry.source]}</td>
+              <td style={{ wordWrap: "anywhere" as any }}>{entry.value}</td>
             </tr>
           ))}
         </tbody>
@@ -140,9 +143,8 @@ const Options = ({ minValueLength, setMinValueLength }) => {
 };
 
 const Sidebar = () => {
-  console.log("rendering!");
   const [pickedTab, setPickedTab] = useState<number | null>(null);
-  const [minValueLength, setMinValueLength] = useState<number | null>(5);
+  const [minValueLength, setMinValueLength] = useState<number | null>(7);
   const counter = useEmitter(memory);
   return (
     <>

util.ts

@@ -16,11 +16,8 @@ export function getshorthost(host: string) {
 export function useEmitter(e: EventEmitter) {
   const [counter, setCounter] = useState<number>(0);
   useEffect(() => {
-    console.log("useEmitter!");
     const callback = () => {
-      console.log("Detected memory change!");
       setCounter((counter) => counter + 1);
-      console.log("RT:", counter + 1);
     };
     e.on("change", callback);
     return () => {