icd/rentgen
icd/rentgen
3
2
Fork 1
Code Issues 73 Pull Requests 2 Projects Releases 6 Wiki Activity

Add README and dedupe data entries

Browse Source
This commit is contained in:
Kuba Orlik 2021-10-04 18:51:51 +02:00
parent 73fa9a8976
commit 43e0c6c7f8
9 changed files with 160 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README.md Normal file
View File

@ -0,0 +1,7 @@
# Problematic-requests, aka ICD scanner
Wtyczka pokazująca, jakie dane zostały ~~wykradzione~~ wysłane do podmiotów trzecich przez odwiedzane strony.
## TODO:
- Używać https://github.com/InteractiveAdvertisingBureau/iabtcf-es/tree/master/modules/core#iabtcfcore do wizualizacji "zgód" zebranych przez CMP-y od IAB

70
extended-request.ts
View File

@ -1,12 +1,19 @@
import { StolenDataEntry } from "./request-cluster";
import { getshorthost, Request } from "./util";
import { getshorthost, parseCookie, Request } from "./util";
export default class ExtendedRequest {
public tabId: number;
public url: string;
public requestHeaders: Request["requestHeaders"];
public origin: string;
public initialized = false;
async getOrigin() {
async init() {
await this.cacheOrigin();
this.initialized = true;
}
async cacheOrigin(): Promise<void> {
let url: string;
if (this.data.tabId && this.data.tabId >= 0) {
const tab = await browser.tabs.get(this.data.tabId);
@ -14,12 +21,19 @@ export default class ExtendedRequest {
} else {
url = (this.data as any).frameAncestors[0].url;
}
return url;
this.origin = url;
}
async isThirdParty() {
getOrigin(): string {
if (!this.initialized) {
throw new Error("initialize first!!");
}
return this.origin;
}
isThirdParty() {
const request_url = new URL(this.data.url);
const origin_url = new URL(await this.getOrigin());
const origin_url = new URL(this.getOrigin());
if (request_url.host.includes(origin_url.host)) {
return false;
}
@ -37,8 +51,33 @@ export default class ExtendedRequest {
.value;
}
async exposesOrigin() {
return this.getReferer().includes(new URL(await this.getOrigin()).host);
exposesOrigin() {
const url = new URL(this.getOrigin());
const host = url.host;
const path = url.pathname;
return (
this.getReferer().includes(host) ||
this.getAllStolenData().filter(
(entry) => entry.value.includes(host) || entry.value.includes(path)
).length > 0
);
}
getAllStolenData(): StolenDataEntry[] {
return [
...this.getPathParams(),
...this.getCookieData(),
...this.getQueryParams(),
];
}
getCookieData(): StolenDataEntry[] {
if (!this.hasCookie() || this.getCookie() === undefined) {
return [];
}
return Object.entries(parseCookie(this.getCookie()))
.map(([key, value]) => [key, value || ""])
.map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
}
hasReferer() {
@ -49,7 +88,7 @@ export default class ExtendedRequest {
return this.data.requestHeaders.some((h) => h.name === "Cookie");
}
getCookie() {
getCookie(): string {
return this.requestHeaders.find((h) => h.name == "Cookie")?.value;
}
@ -62,12 +101,25 @@ export default class ExtendedRequest {
return path
.split(";")
.map((e) => e.split("="))
.map(([key, value]) => [key, value || ""])
.map(
([key, value]) =>
new StolenDataEntry("pathname", key, decodeURIComponent(value))
new StolenDataEntry(this, "pathname", key, decodeURIComponent(value))
);
}
getQueryParams(): StolenDataEntry[] {
const url = new URL(this.data.url);
return Array.from((url.searchParams as any).entries())
.map(([key, value]) => [key, value || ""])
.map(([key, value]) => {
try {
value = decodeURIComponent(value);
} catch (e) {}
return new StolenDataEntry(this, "queryparams", key, value);
});
}
constructor(public data: Request) {
this.tabId = data.tabId;
this.url = data.url;

7
memory.ts
View File

@ -6,11 +6,8 @@ import { RequestCluster } from "./request-cluster";
class Memory extends EventEmitter {
tab_to_history = {} as Record<string, Record<string, RequestCluster>>;
async register(request: ExtendedRequest) {
if (
(await request.isThirdParty()) &&
request.hasReferer() &&
(await request.exposesOrigin())
) {
await request.init();
if (request.isThirdParty() && request.exposesOrigin()) {
if (!this.tab_to_history[request.tabId]) {
this.tab_to_history[request.tabId] = {};
}

40
package-lock.json generated
View File

@ -9,10 +9,12 @@
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"consent-string": "^1.5.2",
"esbuild": "^0.13.3",
"events": "^3.3.0",
"react": "^17.0.2",
"react-dom": "^17.0.2"
"react-dom": "^17.0.2",
"tai-password-strength": "^1.1.3"
},
"devDependencies": {
"@types/events": "^3.0.0",
@ -58,6 +60,19 @@
"integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
"dev": true
},
"node_modules/base-64": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
"integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
},
"node_modules/consent-string": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
"integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
"dependencies": {
"base-64": "^0.1.0"
}
},
"node_modules/csstype": {
"version": "3.0.9",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
@ -349,6 +364,11 @@
"object-assign": "^4.1.1"
}
},
"node_modules/tai-password-strength": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
"integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
},
"node_modules/web-ext-types": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",
@ -395,6 +415,19 @@
"integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==",
"dev": true
},
"base-64": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
"integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
},
"consent-string": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/consent-string/-/consent-string-1.5.2.tgz",
"integrity": "sha512-xzfHnFzHQSupiamNY93UGn8FggPajHYExI45pzadhVpXVaj3ztnhnA7lYjKXl09pKRQKCT4hvjytt+2eoH7Jaw==",
"requires": {
"base-64": "^0.1.0"
}
},
"csstype": {
"version": "3.0.9",
"resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.9.tgz",
@ -571,6 +604,11 @@
"object-assign": "^4.1.1"
}
},
"tai-password-strength": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/tai-password-strength/-/tai-password-strength-1.1.3.tgz",
"integrity": "sha512-GZVtM7wEbgp9IZ9CkdGbpnx0MflFDonzehQIPO0tx3KXMq1ImLiLK33N+ziC4rm8BVd7jrq93kBCOP6VJ4DdzA=="
},
"web-ext-types": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",

4
package.json
View File

@ -14,10 +14,12 @@
"author": "",
"license": "ISC",
"dependencies": {
"consent-string": "^1.5.2",
"esbuild": "^0.13.3",
"events": "^3.3.0",
"react": "^17.0.2",
"react-dom": "^17.0.2"
"react-dom": "^17.0.2",
"tai-password-strength": "^1.1.3"
},
"devDependencies": {
"@types/events": "^3.0.0",

8
problematic.js
View File

@ -7,7 +7,6 @@ function gethost(url) {
}
function getshorthost(host) {
console.log("getshort", host);
return host.split(".").slice(-2).join(".");
}
@ -16,11 +15,6 @@ async function isThirdParty(request) {
const request_url = new URL(request.url);
const origin_url = new URL(await getOrigin(request));
/* console.log(request_url.ho, origin_url, request_url.includes(origin_url)); */
console.log(
request_url.host,
origin_url.host,
request_url.host.includes(origin_url.host)
);
if (request_url.host.includes(origin_url.host)) {
return false;
}
@ -85,11 +79,9 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
if (sender.tab) {
return;
}
console.log("got message!", request);
if (request?.msg === "get_memory") {
sendResponse(memory);
} else if (request?.msg === "clear_memory") {
console.log("memory cleared");
memory = {};
}
});

90
request-cluster.ts
View File

@ -1,9 +1,25 @@
import { EventEmitter } from "events";
import ExtendedRequest from "./extended-request";
import { parseCookie } from "./util";
export type Sources = "cookie" | "pathname" | "queryparams";
export class StolenDataEntry {
constructor(public type: string, public name: string, public value: string) {}
constructor(
public request: ExtendedRequest,
public source: Sources,
public name: string,
public value: string
) {}
getPriority() {
let priority = 0;
priority += this.value.length;
const url = new URL(this.request.getOrigin());
if (this.value.includes(url.host) || this.value.includes(url.pathname)) {
priority += 100;
}
return priority;
}
}
export class RequestCluster extends EventEmitter {
@ -25,56 +41,28 @@ export class RequestCluster extends EventEmitter {
return false;
}
getCookiesContent({
minValueLength,
}: {
minValueLength: number;
}): StolenDataEntry[] {
this.getQueryParamsContent({ minValueLength });
const cookieValues = new Set<string>();
for (const request of this.requests) {
if (request.hasCookie()) {
cookieValues.add(request.getCookie());
}
}
return Array.from(cookieValues.values())
.map(parseCookie)
.map((o) => Object.entries(o))
getStolenData(filter: { minValueLength: number }): StolenDataEntry[] {
return this.requests
.map((request) => request.getAllStolenData())
.reduce((a, b) => a.concat(b), [])
.map(([key, value]) => new StolenDataEntry("cookie", key, value))
.filter((e) => e.value.length >= minValueLength);
}
getQueryParamsContent({
minValueLength,
}: {
minValueLength: number;
}): StolenDataEntry[] {
const result = [];
for (const request of this.requests) {
console.log(request.data.url);
}
return result;
}
getPathnameParamsContent({
minValueLength,
}: {
minValueLength: number;
}): StolenDataEntry[] {
let result = [];
for (const request of this.requests) {
result = [...result, ...request.getPathParams()];
}
console.log("PATHNAME PARAMS FOR", this.id, result);
return result;
}
getStolenData(filter: { minValueLength: number }) {
return [
...this.getCookiesContent(filter),
...this.getPathnameParamsContent(filter),
];
.filter((entry) => {
return entry.value.length >= filter.minValueLength;
})
.sort((entry1, entry2) =>
entry1.getPriority() > entry2.getPriority() ? -1 : 1
)
.filter((element, index, array) => {
// remove duplicate neighbours
if (index == 0) {
return true;
}
if (
element.name != array[index - 1].name ||
element.value != array[index - 1].value
) {
return true;
}
});
}
static sortCompare(a: RequestCluster, b: RequestCluster) {

16
sidebar.tsx
View File

@ -1,7 +1,7 @@
import React, { useEffect, useState } from "react";
import ReactDOM from "react-dom";
import memory from "./memory";
import { RequestCluster } from "./request-cluster";
import { RequestCluster, Sources } from "./request-cluster";
import { Tab, useEmitter } from "./util";
async function getTabByID(id: number) {
@ -10,7 +10,6 @@ async function getTabByID(id: number) {
}
async function getCurrentTab() {
console.log("getCurrentTab");
const [tab] = await browser.tabs.query({
active: true,
windowId: browser.windows.WINDOW_ID_CURRENT,
@ -27,7 +26,6 @@ const TabDropdown = ({
}) => {
const [tabs, setTabs] = useState([]);
useEffect(() => {
console.log("useEffect!");
browser.tabs.query({ currentWindow: true }).then(setTabs);
}, []);
return (
@ -35,7 +33,6 @@ const TabDropdown = ({
id="tab_dropdown"
value={pickedTab}
onChange={async (e) => {
console.log(e.target.value);
setPickedTab(parseInt(e.target.value));
}}
>
@ -60,6 +57,11 @@ const StolenDataRow = ({
minValueLength: number;
}) => {
const cluster = memory.getClustersForTab(tabID)[shorthost];
const icons: Record<Sources, string> = {
cookie: "🍪",
pathname: "🛣",
queryparams: "🅿",
};
return (
<div>
<h2>
@ -73,7 +75,8 @@ const StolenDataRow = ({
<th style={{ maxWidth: "200px", wordWrap: "break-word" }}>
{entry.name}
</th>
<td>{entry.value}</td>
<td>{icons[entry.source]}</td>
<td style={{ wordWrap: "anywhere" as any }}>{entry.value}</td>
</tr>
))}
</tbody>
@ -140,9 +143,8 @@ const Options = ({ minValueLength, setMinValueLength }) => {
};
const Sidebar = () => {
console.log("rendering!");
const [pickedTab, setPickedTab] = useState<number | null>(null);
const [minValueLength, setMinValueLength] = useState<number | null>(5);
const [minValueLength, setMinValueLength] = useState<number | null>(7);
const counter = useEmitter(memory);
return (
<>

3
util.ts
View File

@ -16,11 +16,8 @@ export function getshorthost(host: string) {
export function useEmitter(e: EventEmitter) {
const [counter, setCounter] = useState<number>(0);
useEffect(() => {
console.log("useEmitter!");
const callback = () => {
console.log("Detected memory change!");
setCounter((counter) => counter + 1);
console.log("RT:", counter + 1);
};
e.on("change", callback);
return () => {