Add README and dedupe data entries
This commit is contained in:
parent
73fa9a8976
commit
43e0c6c7f8
|
|
@ -0,0 +1,7 @@
|
|||
# Problematic-requests, aka ICD scanner
|
||||
|
||||
Wtyczka pokazująca, jakie dane zostały ~~wykradzione~~ wysłane do podmiotów trzecich przez odwiedzane strony.
|
||||
|
||||
## TODO:
|
||||
|
||||
- Używać https://github.com/InteractiveAdvertisingBureau/iabtcf-es/tree/master/modules/core#iabtcfcore do wizualizacji "zgód" zebranych przez CMP-y od IAB
|
||||
|
|
@ -1,12 +1,19 @@
|
|||
import { StolenDataEntry } from "./request-cluster";
|
||||
import { getshorthost, Request } from "./util";
|
||||
import { getshorthost, parseCookie, Request } from "./util";
|
||||
|
||||
export default class ExtendedRequest {
|
||||
public tabId: number;
|
||||
public url: string;
|
||||
public requestHeaders: Request["requestHeaders"];
|
||||
public origin: string;
|
||||
public initialized = false;
|
||||
|
||||
async getOrigin() {
|
||||
async init() {
|
||||
await this.cacheOrigin();
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
async cacheOrigin(): Promise<void> {
|
||||
let url: string;
|
||||
if (this.data.tabId && this.data.tabId >= 0) {
|
||||
const tab = await browser.tabs.get(this.data.tabId);
|
||||
|
|
@ -14,12 +21,19 @@ export default class ExtendedRequest {
|
|||
} else {
|
||||
url = (this.data as any).frameAncestors[0].url;
|
||||
}
|
||||
return url;
|
||||
this.origin = url;
|
||||
}
|
||||
|
||||
async isThirdParty() {
|
||||
getOrigin(): string {
|
||||
if (!this.initialized) {
|
||||
throw new Error("initialize first!!");
|
||||
}
|
||||
return this.origin;
|
||||
}
|
||||
|
||||
isThirdParty() {
|
||||
const request_url = new URL(this.data.url);
|
||||
const origin_url = new URL(await this.getOrigin());
|
||||
const origin_url = new URL(this.getOrigin());
|
||||
if (request_url.host.includes(origin_url.host)) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -37,8 +51,33 @@ export default class ExtendedRequest {
|
|||
.value;
|
||||
}
|
||||
|
||||
async exposesOrigin() {
|
||||
return this.getReferer().includes(new URL(await this.getOrigin()).host);
|
||||
exposesOrigin() {
|
||||
const url = new URL(this.getOrigin());
|
||||
const host = url.host;
|
||||
const path = url.pathname;
|
||||
return (
|
||||
this.getReferer().includes(host) ||
|
||||
this.getAllStolenData().filter(
|
||||
(entry) => entry.value.includes(host) || entry.value.includes(path)
|
||||
).length > 0
|
||||
);
|
||||
}
|
||||
|
||||
getAllStolenData(): StolenDataEntry[] {
|
||||
return [
|
||||
...this.getPathParams(),
|
||||
...this.getCookieData(),
|
||||
...this.getQueryParams(),
|
||||
];
|
||||
}
|
||||
|
||||
getCookieData(): StolenDataEntry[] {
|
||||
if (!this.hasCookie() || this.getCookie() === undefined) {
|
||||
return [];
|
||||
}
|
||||
return Object.entries(parseCookie(this.getCookie()))
|
||||
.map(([key, value]) => [key, value || ""])
|
||||
.map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
|
||||
}
|
||||
|
||||
hasReferer() {
|
||||
|
|
@ -49,7 +88,7 @@ export default class ExtendedRequest {
|
|||
return this.data.requestHeaders.some((h) => h.name === "Cookie");
|
||||
}
|
||||
|
||||
getCookie() {
|
||||
getCookie(): string {
|
||||
return this.requestHeaders.find((h) => h.name == "Cookie")?.value;
|
||||
}
|
||||
|
||||
|
|
@ -62,12 +101,25 @@ export default class ExtendedRequest {
|
|||
return path
|
||||
.split(";")
|
||||
.map((e) => e.split("="))
|
||||
.map(([key, value]) => [key, value || ""])
|
||||
.map(
|
||||
([key, value]) =>
|
||||
new StolenDataEntry("pathname", key, decodeURIComponent(value))
|
||||
new StolenDataEntry(this, "pathname", key, decodeURIComponent(value))
|
||||
);
|
||||
}
|
||||
|
||||
getQueryParams(): StolenDataEntry[] {
|
||||
const url = new URL(this.data.url);
|
||||
return Array.from((url.searchParams as any).entries())
|
||||
.map(([key, value]) => [key, value || ""])
|
||||
.map(([key, value]) => {
|
||||
try {
|
||||
value = decodeURIComponent(value);
|
||||
} catch (e) {}
|
||||
return new StolenDataEntry(this, "queryparams", key, value);
|
||||
});
|
||||
}
|
||||
|
||||
constructor(public data: Request) {
|
||||
this.tabId = data.tabId;
|
||||
this.url = data.url;
|
||||
|
|
|
|||
|
|
@ -6,11 +6,8 @@ import { RequestCluster } from "./request-cluster";
|
|||
class Memory extends EventEmitter {
|
||||
tab_to_history = {} as Record<string, Record<string, RequestCluster>>;
|
||||
async register(request: ExtendedRequest) {
|
||||
if (
|
||||
(await request.isThirdParty()) &&
|
||||
request.hasReferer() &&
|
||||
(await request.exposesOrigin())
|
||||
) {
|
||||
await request.init();
|
||||
if (request.isThirdParty() && request.exposesOrigin()) {
|
||||
if (!this.tab_to_history[request.tabId]) {
|
||||
this.tab_to_history[request.tabId] = {};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,10 +9,12 @@
|
|||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"consent-string": "^1.5.2",
|
||||
"esbuild": "^0.13.3",
|
||||
"events": "^3.3.0",
|
||||
"react": "^17.0.2",
|
||||
"react-dom": "^17.0.2"
|
||||
"react-dom": "^17.0.2",
|
||||
"tai-password-strength": "^1.1.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/events": "^3.0.0",
|
||||
|
|
@ -58,6 +60,19 @@
|
|||
"integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/base-64": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
|
||||
"integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
|
||||
},
|
||||
"node_modules/consent-string": {
|
||||
"version": "1.5.2",
|
||||
"resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
|
||||
"integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
|
||||
"dependencies": {
|
||||
"base-64": "^0.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/csstype": {
|
||||
"version": "3.0.9",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
|
||||
|
|
@ -349,6 +364,11 @@
|
|||
"object-assign": "^4.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/tai-password-strength": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
|
||||
"integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
|
||||
},
|
||||
"node_modules/web-ext-types": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",
|
||||
|
|
@ -395,6 +415,19 @@
|
|||
"integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
|
||||
"dev": true
|
||||
},
|
||||
"base-64": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
|
||||
"integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
|
||||
},
|
||||
"consent-string": {
|
||||
"version": "1.5.2",
|
||||
"resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
|
||||
"integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
|
||||
"requires": {
|
||||
"base-64": "^0.1.0"
|
||||
}
|
||||
},
|
||||
"csstype": {
|
||||
"version": "3.0.9",
|
||||
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
|
||||
|
|
@ -571,6 +604,11 @@
|
|||
"object-assign": "^4.1.1"
|
||||
}
|
||||
},
|
||||
"tai-password-strength": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
|
||||
"integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
|
||||
},
|
||||
"web-ext-types": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",
|
||||
|
|
|
|||
|
|
@ -14,10 +14,12 @@
|
|||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"consent-string": "^1.5.2",
|
||||
"esbuild": "^0.13.3",
|
||||
"events": "^3.3.0",
|
||||
"react": "^17.0.2",
|
||||
"react-dom": "^17.0.2"
|
||||
"react-dom": "^17.0.2",
|
||||
"tai-password-strength": "^1.1.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/events": "^3.0.0",
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ function gethost(url) {
|
|||
}
|
||||
|
||||
function getshorthost(host) {
|
||||
console.log("getshort", host);
|
||||
return host.split(".").slice(-2).join(".");
|
||||
}
|
||||
|
||||
|
|
@ -16,11 +15,6 @@ async function isThirdParty(request) {
|
|||
const request_url = new URL(request.url);
|
||||
const origin_url = new URL(await getOrigin(request));
|
||||
/* console.log(request_url.ho, origin_url, request_url.includes(origin_url)); */
|
||||
console.log(
|
||||
request_url.host,
|
||||
origin_url.host,
|
||||
request_url.host.includes(origin_url.host)
|
||||
);
|
||||
if (request_url.host.includes(origin_url.host)) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -85,11 +79,9 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
|
|||
if (sender.tab) {
|
||||
return;
|
||||
}
|
||||
console.log("got message!", request);
|
||||
if (request?.msg === "get_memory") {
|
||||
sendResponse(memory);
|
||||
} else if (request?.msg === "clear_memory") {
|
||||
console.log("memory cleared");
|
||||
memory = {};
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,9 +1,25 @@
|
|||
import { EventEmitter } from "events";
|
||||
import ExtendedRequest from "./extended-request";
|
||||
import { parseCookie } from "./util";
|
||||
|
||||
export type Sources = "cookie" | "pathname" | "queryparams";
|
||||
|
||||
export class StolenDataEntry {
|
||||
constructor(public type: string, public name: string, public value: string) {}
|
||||
constructor(
|
||||
public request: ExtendedRequest,
|
||||
public source: Sources,
|
||||
public name: string,
|
||||
public value: string
|
||||
) {}
|
||||
|
||||
getPriority() {
|
||||
let priority = 0;
|
||||
priority += this.value.length;
|
||||
const url = new URL(this.request.getOrigin());
|
||||
if (this.value.includes(url.host) || this.value.includes(url.pathname)) {
|
||||
priority += 100;
|
||||
}
|
||||
return priority;
|
||||
}
|
||||
}
|
||||
|
||||
export class RequestCluster extends EventEmitter {
|
||||
|
|
@ -25,56 +41,28 @@ export class RequestCluster extends EventEmitter {
|
|||
return false;
|
||||
}
|
||||
|
||||
getCookiesContent({
|
||||
minValueLength,
|
||||
}: {
|
||||
minValueLength: number;
|
||||
}): StolenDataEntry[] {
|
||||
this.getQueryParamsContent({ minValueLength });
|
||||
const cookieValues = new Set<string>();
|
||||
for (const request of this.requests) {
|
||||
if (request.hasCookie()) {
|
||||
cookieValues.add(request.getCookie());
|
||||
}
|
||||
}
|
||||
return Array.from(cookieValues.values())
|
||||
.map(parseCookie)
|
||||
.map((o) => Object.entries(o))
|
||||
getStolenData(filter: { minValueLength: number }): StolenDataEntry[] {
|
||||
return this.requests
|
||||
.map((request) => request.getAllStolenData())
|
||||
.reduce((a, b) => a.concat(b), [])
|
||||
.map(([key, value]) => new StolenDataEntry("cookie", key, value))
|
||||
.filter((e) => e.value.length >= minValueLength);
|
||||
}
|
||||
|
||||
getQueryParamsContent({
|
||||
minValueLength,
|
||||
}: {
|
||||
minValueLength: number;
|
||||
}): StolenDataEntry[] {
|
||||
const result = [];
|
||||
for (const request of this.requests) {
|
||||
console.log(request.data.url);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
getPathnameParamsContent({
|
||||
minValueLength,
|
||||
}: {
|
||||
minValueLength: number;
|
||||
}): StolenDataEntry[] {
|
||||
let result = [];
|
||||
for (const request of this.requests) {
|
||||
result = [...result, ...request.getPathParams()];
|
||||
}
|
||||
console.log("PATHNAME PARAMS FOR", this.id, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
getStolenData(filter: { minValueLength: number }) {
|
||||
return [
|
||||
...this.getCookiesContent(filter),
|
||||
...this.getPathnameParamsContent(filter),
|
||||
];
|
||||
.filter((entry) => {
|
||||
return entry.value.length >= filter.minValueLength;
|
||||
})
|
||||
.sort((entry1, entry2) =>
|
||||
entry1.getPriority() > entry2.getPriority() ? -1 : 1
|
||||
)
|
||||
.filter((element, index, array) => {
|
||||
// remove duplicate neighbours
|
||||
if (index == 0) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
element.name != array[index - 1].name ||
|
||||
element.value != array[index - 1].value
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
static sortCompare(a: RequestCluster, b: RequestCluster) {
|
||||
|
|
|
|||
16
sidebar.tsx
16
sidebar.tsx
|
|
@ -1,7 +1,7 @@
|
|||
import React, { useEffect, useState } from "react";
|
||||
import ReactDOM from "react-dom";
|
||||
import memory from "./memory";
|
||||
import { RequestCluster } from "./request-cluster";
|
||||
import { RequestCluster, Sources } from "./request-cluster";
|
||||
import { Tab, useEmitter } from "./util";
|
||||
|
||||
async function getTabByID(id: number) {
|
||||
|
|
@ -10,7 +10,6 @@ async function getTabByID(id: number) {
|
|||
}
|
||||
|
||||
async function getCurrentTab() {
|
||||
console.log("getCurrentTab");
|
||||
const [tab] = await browser.tabs.query({
|
||||
active: true,
|
||||
windowId: browser.windows.WINDOW_ID_CURRENT,
|
||||
|
|
@ -27,7 +26,6 @@ const TabDropdown = ({
|
|||
}) => {
|
||||
const [tabs, setTabs] = useState([]);
|
||||
useEffect(() => {
|
||||
console.log("useEffect!");
|
||||
browser.tabs.query({ currentWindow: true }).then(setTabs);
|
||||
}, []);
|
||||
return (
|
||||
|
|
@ -35,7 +33,6 @@ const TabDropdown = ({
|
|||
id="tab_dropdown"
|
||||
value={pickedTab}
|
||||
onChange={async (e) => {
|
||||
console.log(e.target.value);
|
||||
setPickedTab(parseInt(e.target.value));
|
||||
}}
|
||||
>
|
||||
|
|
@ -60,6 +57,11 @@ const StolenDataRow = ({
|
|||
minValueLength: number;
|
||||
}) => {
|
||||
const cluster = memory.getClustersForTab(tabID)[shorthost];
|
||||
const icons: Record<Sources, string> = {
|
||||
cookie: "🍪",
|
||||
pathname: "🛣",
|
||||
queryparams: "🅿",
|
||||
};
|
||||
return (
|
||||
<div>
|
||||
<h2>
|
||||
|
|
@ -73,7 +75,8 @@ const StolenDataRow = ({
|
|||
<th style={{ maxWidth: "200px", wordWrap: "break-word" }}>
|
||||
{entry.name}
|
||||
</th>
|
||||
<td>{entry.value}</td>
|
||||
<td>{icons[entry.source]}</td>
|
||||
<td style={{ wordWrap: "anywhere" as any }}>{entry.value}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
|
|
@ -140,9 +143,8 @@ const Options = ({ minValueLength, setMinValueLength }) => {
|
|||
};
|
||||
|
||||
const Sidebar = () => {
|
||||
console.log("rendering!");
|
||||
const [pickedTab, setPickedTab] = useState<number | null>(null);
|
||||
const [minValueLength, setMinValueLength] = useState<number | null>(5);
|
||||
const [minValueLength, setMinValueLength] = useState<number | null>(7);
|
||||
const counter = useEmitter(memory);
|
||||
return (
|
||||
<>
|
||||
|
|
|
|||
3
util.ts
3
util.ts
|
|
@ -16,11 +16,8 @@ export function getshorthost(host: string) {
|
|||
export function useEmitter(e: EventEmitter) {
|
||||
const [counter, setCounter] = useState<number>(0);
|
||||
useEffect(() => {
|
||||
console.log("useEmitter!");
|
||||
const callback = () => {
|
||||
console.log("Detected memory change!");
|
||||
setCounter((counter) => counter + 1);
|
||||
console.log("RT:", counter + 1);
|
||||
};
|
||||
e.on("change", callback);
|
||||
return () => {
|
||||
|
|
|
|||
Loading…
Reference in New Issue