Add README and dedupe data entries

This commit is contained in:
Kuba Orlik 2021-10-04 18:51:51 +02:00
parent 73fa9a8976
commit 43e0c6c7f8
9 changed files with 160 additions and 85 deletions

7
README.md Normal file
View File

@ -0,0 +1,7 @@
# Problematic-requests, aka ICD scanner
Wtyczka pokazująca, jakie dane zostały ~~wykradzione~~ wysłane do podmiotów trzecich przez odwiedzane strony.
## TODO:
- Używać https://github.com/InteractiveAdvertisingBureau/iabtcf-es/tree/master/modules/core#iabtcfcore do wizualizacji "zgód" zebranych przez CMP-y od IAB

View File

@ -1,12 +1,19 @@
import { StolenDataEntry } from "./request-cluster"; import { StolenDataEntry } from "./request-cluster";
import { getshorthost, Request } from "./util"; import { getshorthost, parseCookie, Request } from "./util";
export default class ExtendedRequest { export default class ExtendedRequest {
public tabId: number; public tabId: number;
public url: string; public url: string;
public requestHeaders: Request["requestHeaders"]; public requestHeaders: Request["requestHeaders"];
public origin: string;
public initialized = false;
async getOrigin() { async init() {
await this.cacheOrigin();
this.initialized = true;
}
async cacheOrigin(): Promise<void> {
let url: string; let url: string;
if (this.data.tabId && this.data.tabId >= 0) { if (this.data.tabId && this.data.tabId >= 0) {
const tab = await browser.tabs.get(this.data.tabId); const tab = await browser.tabs.get(this.data.tabId);
@ -14,12 +21,19 @@ export default class ExtendedRequest {
} else { } else {
url = (this.data as any).frameAncestors[0].url; url = (this.data as any).frameAncestors[0].url;
} }
return url; this.origin = url;
} }
async isThirdParty() { getOrigin(): string {
if (!this.initialized) {
throw new Error("initialize first!!");
}
return this.origin;
}
isThirdParty() {
const request_url = new URL(this.data.url); const request_url = new URL(this.data.url);
const origin_url = new URL(await this.getOrigin()); const origin_url = new URL(this.getOrigin());
if (request_url.host.includes(origin_url.host)) { if (request_url.host.includes(origin_url.host)) {
return false; return false;
} }
@ -37,8 +51,33 @@ export default class ExtendedRequest {
.value; .value;
} }
async exposesOrigin() { exposesOrigin() {
return this.getReferer().includes(new URL(await this.getOrigin()).host); const url = new URL(this.getOrigin());
const host = url.host;
const path = url.pathname;
return (
this.getReferer().includes(host) ||
this.getAllStolenData().filter(
(entry) => entry.value.includes(host) || entry.value.includes(path)
).length > 0
);
}
getAllStolenData(): StolenDataEntry[] {
return [
...this.getPathParams(),
...this.getCookieData(),
...this.getQueryParams(),
];
}
getCookieData(): StolenDataEntry[] {
if (!this.hasCookie() || this.getCookie() === undefined) {
return [];
}
return Object.entries(parseCookie(this.getCookie()))
.map(([key, value]) => [key, value || ""])
.map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
} }
hasReferer() { hasReferer() {
@ -49,7 +88,7 @@ export default class ExtendedRequest {
return this.data.requestHeaders.some((h) => h.name === "Cookie"); return this.data.requestHeaders.some((h) => h.name === "Cookie");
} }
getCookie() { getCookie(): string {
return this.requestHeaders.find((h) => h.name == "Cookie")?.value; return this.requestHeaders.find((h) => h.name == "Cookie")?.value;
} }
@ -62,12 +101,25 @@ export default class ExtendedRequest {
return path return path
.split(";") .split(";")
.map((e) => e.split("=")) .map((e) => e.split("="))
.map(([key, value]) => [key, value || ""])
.map( .map(
([key, value]) => ([key, value]) =>
new StolenDataEntry("pathname", key, decodeURIComponent(value)) new StolenDataEntry(this, "pathname", key, decodeURIComponent(value))
); );
} }
getQueryParams(): StolenDataEntry[] {
const url = new URL(this.data.url);
return Array.from((url.searchParams as any).entries())
.map(([key, value]) => [key, value || ""])
.map(([key, value]) => {
try {
value = decodeURIComponent(value);
} catch (e) {}
return new StolenDataEntry(this, "queryparams", key, value);
});
}
constructor(public data: Request) { constructor(public data: Request) {
this.tabId = data.tabId; this.tabId = data.tabId;
this.url = data.url; this.url = data.url;

View File

@ -6,11 +6,8 @@ import { RequestCluster } from "./request-cluster";
class Memory extends EventEmitter { class Memory extends EventEmitter {
tab_to_history = {} as Record<string, Record<string, RequestCluster>>; tab_to_history = {} as Record<string, Record<string, RequestCluster>>;
async register(request: ExtendedRequest) { async register(request: ExtendedRequest) {
if ( await request.init();
(await request.isThirdParty()) && if (request.isThirdParty() && request.exposesOrigin()) {
request.hasReferer() &&
(await request.exposesOrigin())
) {
if (!this.tab_to_history[request.tabId]) { if (!this.tab_to_history[request.tabId]) {
this.tab_to_history[request.tabId] = {}; this.tab_to_history[request.tabId] = {};
} }

40
package-lock.json generated
View File

@ -9,10 +9,12 @@
"version": "1.0.0", "version": "1.0.0",
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"consent-string": "^1.5.2",
"esbuild": "^0.13.3", "esbuild": "^0.13.3",
"events": "^3.3.0", "events": "^3.3.0",
"react": "^17.0.2", "react": "^17.0.2",
"react-dom": "^17.0.2" "react-dom": "^17.0.2",
"tai-password-strength": "^1.1.3"
}, },
"devDependencies": { "devDependencies": {
"@types/events": "^3.0.0", "@types/events": "^3.0.0",
@ -58,6 +60,19 @@
"integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==", "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
"dev": true "dev": true
}, },
"node_modules/base-64": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
"integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
},
"node_modules/consent-string": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
"integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
"dependencies": {
"base-64": "^0.1.0"
}
},
"node_modules/csstype": { "node_modules/csstype": {
"version": "3.0.9", "version": "3.0.9",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
@ -349,6 +364,11 @@
"object-assign": "^4.1.1" "object-assign": "^4.1.1"
} }
}, },
"node_modules/tai-password-strength": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
"integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
},
"node_modules/web-ext-types": { "node_modules/web-ext-types": {
"version": "3.2.1", "version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz", "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",
@ -395,6 +415,19 @@
"integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==", "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
"dev": true "dev": true
}, },
"base-64": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
"integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
},
"consent-string": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
"integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
"requires": {
"base-64": "^0.1.0"
}
},
"csstype": { "csstype": {
"version": "3.0.9", "version": "3.0.9",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
@ -571,6 +604,11 @@
"object-assign": "^4.1.1" "object-assign": "^4.1.1"
} }
}, },
"tai-password-strength": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
"integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
},
"web-ext-types": { "web-ext-types": {
"version": "3.2.1", "version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz", "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",

View File

@ -14,10 +14,12 @@
"author": "", "author": "",
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"consent-string": "^1.5.2",
"esbuild": "^0.13.3", "esbuild": "^0.13.3",
"events": "^3.3.0", "events": "^3.3.0",
"react": "^17.0.2", "react": "^17.0.2",
"react-dom": "^17.0.2" "react-dom": "^17.0.2",
"tai-password-strength": "^1.1.3"
}, },
"devDependencies": { "devDependencies": {
"@types/events": "^3.0.0", "@types/events": "^3.0.0",

View File

@ -7,7 +7,6 @@ function gethost(url) {
} }
function getshorthost(host) { function getshorthost(host) {
console.log("getshort", host);
return host.split(".").slice(-2).join("."); return host.split(".").slice(-2).join(".");
} }
@ -16,11 +15,6 @@ async function isThirdParty(request) {
const request_url = new URL(request.url); const request_url = new URL(request.url);
const origin_url = new URL(await getOrigin(request)); const origin_url = new URL(await getOrigin(request));
/* console.log(request_url.ho, origin_url, request_url.includes(origin_url)); */ /* console.log(request_url.ho, origin_url, request_url.includes(origin_url)); */
console.log(
request_url.host,
origin_url.host,
request_url.host.includes(origin_url.host)
);
if (request_url.host.includes(origin_url.host)) { if (request_url.host.includes(origin_url.host)) {
return false; return false;
} }
@ -85,11 +79,9 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
if (sender.tab) { if (sender.tab) {
return; return;
} }
console.log("got message!", request);
if (request?.msg === "get_memory") { if (request?.msg === "get_memory") {
sendResponse(memory); sendResponse(memory);
} else if (request?.msg === "clear_memory") { } else if (request?.msg === "clear_memory") {
console.log("memory cleared");
memory = {}; memory = {};
} }
}); });

View File

@ -1,9 +1,25 @@
import { EventEmitter } from "events"; import { EventEmitter } from "events";
import ExtendedRequest from "./extended-request"; import ExtendedRequest from "./extended-request";
import { parseCookie } from "./util";
export type Sources = "cookie" | "pathname" | "queryparams";
export class StolenDataEntry { export class StolenDataEntry {
constructor(public type: string, public name: string, public value: string) {} constructor(
public request: ExtendedRequest,
public source: Sources,
public name: string,
public value: string
) {}
getPriority() {
let priority = 0;
priority += this.value.length;
const url = new URL(this.request.getOrigin());
if (this.value.includes(url.host) || this.value.includes(url.pathname)) {
priority += 100;
}
return priority;
}
} }
export class RequestCluster extends EventEmitter { export class RequestCluster extends EventEmitter {
@ -25,56 +41,28 @@ export class RequestCluster extends EventEmitter {
return false; return false;
} }
getCookiesContent({ getStolenData(filter: { minValueLength: number }): StolenDataEntry[] {
minValueLength, return this.requests
}: { .map((request) => request.getAllStolenData())
minValueLength: number;
}): StolenDataEntry[] {
this.getQueryParamsContent({ minValueLength });
const cookieValues = new Set<string>();
for (const request of this.requests) {
if (request.hasCookie()) {
cookieValues.add(request.getCookie());
}
}
return Array.from(cookieValues.values())
.map(parseCookie)
.map((o) => Object.entries(o))
.reduce((a, b) => a.concat(b), []) .reduce((a, b) => a.concat(b), [])
.map(([key, value]) => new StolenDataEntry("cookie", key, value)) .filter((entry) => {
.filter((e) => e.value.length >= minValueLength); return entry.value.length >= filter.minValueLength;
})
.sort((entry1, entry2) =>
entry1.getPriority() > entry2.getPriority() ? -1 : 1
)
.filter((element, index, array) => {
// remove duplicate neighbours
if (index == 0) {
return true;
} }
if (
getQueryParamsContent({ element.name != array[index - 1].name ||
minValueLength, element.value != array[index - 1].value
}: { ) {
minValueLength: number; return true;
}): StolenDataEntry[] {
const result = [];
for (const request of this.requests) {
console.log(request.data.url);
} }
return result; });
}
getPathnameParamsContent({
minValueLength,
}: {
minValueLength: number;
}): StolenDataEntry[] {
let result = [];
for (const request of this.requests) {
result = [...result, ...request.getPathParams()];
}
console.log("PATHNAME PARAMS FOR", this.id, result);
return result;
}
getStolenData(filter: { minValueLength: number }) {
return [
...this.getCookiesContent(filter),
...this.getPathnameParamsContent(filter),
];
} }
static sortCompare(a: RequestCluster, b: RequestCluster) { static sortCompare(a: RequestCluster, b: RequestCluster) {

View File

@ -1,7 +1,7 @@
import React, { useEffect, useState } from "react"; import React, { useEffect, useState } from "react";
import ReactDOM from "react-dom"; import ReactDOM from "react-dom";
import memory from "./memory"; import memory from "./memory";
import { RequestCluster } from "./request-cluster"; import { RequestCluster, Sources } from "./request-cluster";
import { Tab, useEmitter } from "./util"; import { Tab, useEmitter } from "./util";
async function getTabByID(id: number) { async function getTabByID(id: number) {
@ -10,7 +10,6 @@ async function getTabByID(id: number) {
} }
async function getCurrentTab() { async function getCurrentTab() {
console.log("getCurrentTab");
const [tab] = await browser.tabs.query({ const [tab] = await browser.tabs.query({
active: true, active: true,
windowId: browser.windows.WINDOW_ID_CURRENT, windowId: browser.windows.WINDOW_ID_CURRENT,
@ -27,7 +26,6 @@ const TabDropdown = ({
}) => { }) => {
const [tabs, setTabs] = useState([]); const [tabs, setTabs] = useState([]);
useEffect(() => { useEffect(() => {
console.log("useEffect!");
browser.tabs.query({ currentWindow: true }).then(setTabs); browser.tabs.query({ currentWindow: true }).then(setTabs);
}, []); }, []);
return ( return (
@ -35,7 +33,6 @@ const TabDropdown = ({
id="tab_dropdown" id="tab_dropdown"
value={pickedTab} value={pickedTab}
onChange={async (e) => { onChange={async (e) => {
console.log(e.target.value);
setPickedTab(parseInt(e.target.value)); setPickedTab(parseInt(e.target.value));
}} }}
> >
@ -60,6 +57,11 @@ const StolenDataRow = ({
minValueLength: number; minValueLength: number;
}) => { }) => {
const cluster = memory.getClustersForTab(tabID)[shorthost]; const cluster = memory.getClustersForTab(tabID)[shorthost];
const icons: Record<Sources, string> = {
cookie: "🍪",
pathname: "🛣",
queryparams: "🅿",
};
return ( return (
<div> <div>
<h2> <h2>
@ -73,7 +75,8 @@ const StolenDataRow = ({
<th style={{ maxWidth: "200px", wordWrap: "break-word" }}> <th style={{ maxWidth: "200px", wordWrap: "break-word" }}>
{entry.name} {entry.name}
</th> </th>
<td>{entry.value}</td> <td>{icons[entry.source]}</td>
<td style={{ wordWrap: "anywhere" as any }}>{entry.value}</td>
</tr> </tr>
))} ))}
</tbody> </tbody>
@ -140,9 +143,8 @@ const Options = ({ minValueLength, setMinValueLength }) => {
}; };
const Sidebar = () => { const Sidebar = () => {
console.log("rendering!");
const [pickedTab, setPickedTab] = useState<number | null>(null); const [pickedTab, setPickedTab] = useState<number | null>(null);
const [minValueLength, setMinValueLength] = useState<number | null>(5); const [minValueLength, setMinValueLength] = useState<number | null>(7);
const counter = useEmitter(memory); const counter = useEmitter(memory);
return ( return (
<> <>

View File

@ -16,11 +16,8 @@ export function getshorthost(host: string) {
export function useEmitter(e: EventEmitter) { export function useEmitter(e: EventEmitter) {
const [counter, setCounter] = useState<number>(0); const [counter, setCounter] = useState<number>(0);
useEffect(() => { useEffect(() => {
console.log("useEmitter!");
const callback = () => { const callback = () => {
console.log("Detected memory change!");
setCounter((counter) => counter + 1); setCounter((counter) => counter + 1);
console.log("RT:", counter + 1);
}; };
e.on("change", callback); e.on("change", callback);
return () => { return () => {