Handle dead objects

This commit is contained in:
Arkadiusz Wieczorek 2022-04-22 13:00:02 +02:00
parent d759727208
commit 1ba2bea2fb
4 changed files with 635 additions and 597 deletions

View File

@ -26,11 +26,16 @@ const Toolbar = () => {
React.useEffect(() => { React.useEffect(() => {
const listener = async () => { const listener = async () => {
const tab = await getCurrentTab(); const tab = await getCurrentTab();
if (tab !== undefined) {
const url = new URL(tab.url); const url = new URL(tab.url);
if (url.origin.startsWith('moz-extension')) { if (url.origin.startsWith('moz-extension')) {
return; return;
} }
setOrigin(url.origin); setOrigin(url.origin);
} else {
console.warn('Out of the tab scope');
}
}; };
browser.tabs.onUpdated.addListener(listener); browser.tabs.onUpdated.addListener(listener);
@ -217,7 +222,7 @@ const Toolbar = () => {
'fullscreen=yes', 'fullscreen=yes',
].join(','); ].join(',');
window.open( window.open(
`/report-window/report-window.html?origin=${origin}`, `/components/report-window/report-window.html?origin=${origin}`,
'new_window', 'new_window',
params params
); );

View File

@ -1,11 +1,11 @@
import { StolenDataEntry } from "./stolen-data-entry"; import { StolenDataEntry } from './stolen-data-entry';
import { import {
flattenObjectEntries, flattenObjectEntries,
getshorthost, getshorthost,
parseCookie, parseCookie,
Request, Request,
safeDecodeURIComponent, safeDecodeURIComponent,
} from "./util"; } from './util';
type NameValue = { name: string; value: string }; type NameValue = { name: string; value: string };
@ -24,7 +24,7 @@ export type HAREntry = {
params: (NameValue & { params: (NameValue & {
fileName: string; fileName: string;
contentType: string; contentType: string;
comment: ""; comment: '';
})[]; })[];
text: string; text: string;
}; };
@ -40,10 +40,10 @@ export type HAREntry = {
content: { content: {
mimeType: string; mimeType: string;
size: number; size: number;
encoding: "base64"; encoding: 'base64';
text: string; text: string;
}; };
redirectURL: ""; redirectURL: '';
headersSize: number; headersSize: number;
bodySize: number; bodySize: number;
}; // not relevant }; // not relevant
@ -75,7 +75,7 @@ export default class ExtendedRequest {
public tabId: number; public tabId: number;
public url: string; public url: string;
public shorthost: string; public shorthost: string;
public requestHeaders: Request["requestHeaders"] = []; public requestHeaders: Request['requestHeaders'] = [];
public originalURL: string; public originalURL: string;
public origin: string; public origin: string;
public initialized = false; public initialized = false;
@ -84,20 +84,27 @@ export default class ExtendedRequest {
public requestBody: RequestBody; public requestBody: RequestBody;
static by_id = {} as Record<string, ExtendedRequest>; static by_id = {} as Record<string, ExtendedRequest>;
public data: Request;
constructor(public data: Request) { constructor(data: Request) {
this.tabId = data.tabId; this.tabId = data.tabId;
this.url = data.url; this.url = data.url;
this.shorthost = getshorthost(data.url); this.shorthost = getshorthost(data.url);
this.requestBody = this.requestBody = ((data as any).requestBody as undefined | RequestBody) || {};
((data as any).requestBody as undefined | RequestBody) || {}; if (this.url.includes('criteo')) {
if (this.url.includes("criteo")) {
console.log(this); console.log(this);
} }
ExtendedRequest.by_id[data.requestId] = this; ExtendedRequest.by_id[data.requestId] = this;
this.data = Object.assign({}, data);
(this.data as any).frameAncestors = [
...(data as any).frameAncestors.map((e: any) => ({ url: e.url })),
];
// console.log('→→→',(this.data as any).frameAncestors, (data as any).frameAncestors);
} }
addHeaders(headers: Request["requestHeaders"]) { addHeaders(headers: Request['requestHeaders']) {
this.requestHeaders = headers; this.requestHeaders = headers;
return this; return this;
} }
@ -113,8 +120,11 @@ export default class ExtendedRequest {
if (this.data.tabId && this.data.tabId >= 0) { if (this.data.tabId && this.data.tabId >= 0) {
const tab = await browser.tabs.get(this.data.tabId); const tab = await browser.tabs.get(this.data.tabId);
url = tab.url; url = tab.url;
} else if ((this.data as any)?.frameAncestors) { } else if (
url = (this.data as any).frameAncestors[0].url || ""; (this.data as any)?.frameAncestors &&
(this.data as any).frameAncestors[0] !== undefined
) {
url = (this.data as any).frameAncestors[0].url || '';
} else { } else {
const headers = Object.fromEntries( const headers = Object.fromEntries(
this.requestHeaders.map(({ name, value }) => [name, value]) this.requestHeaders.map(({ name, value }) => [name, value])
@ -148,8 +158,7 @@ export default class ExtendedRequest {
getReferer() { getReferer() {
return ( return (
this.requestHeaders.filter((h) => h.name === "Referer")[0]?.value || this.requestHeaders.filter((h) => h.name === 'Referer')[0]?.value || 'missing-referrer'
"missing-referrer"
); );
} }
@ -188,12 +197,9 @@ export default class ExtendedRequest {
return []; return [];
} }
return flattenObjectEntries( return flattenObjectEntries(
Object.entries(parseCookie(this.getCookie())).map(([key, value]) => [ Object.entries(parseCookie(this.getCookie())).map(([key, value]) => [key, value || '']),
key,
value || "",
]),
StolenDataEntry.parseValue StolenDataEntry.parseValue
).map(([key, value]) => new StolenDataEntry(this, "cookie", key, value)); ).map(([key, value]) => new StolenDataEntry(this, 'cookie', key, value));
} }
getRequestBodyData(): StolenDataEntry[] { getRequestBodyData(): StolenDataEntry[] {
@ -201,78 +207,65 @@ export default class ExtendedRequest {
Object.entries({ Object.entries({
...this.requestBody.formData, ...this.requestBody.formData,
...Object.fromEntries( ...Object.fromEntries(
Object.entries( Object.entries(this.requestBody.raw || {}).map(([key, value], index) => [
this.requestBody.raw || {} `${key}.${index}`,
).map(([key, value], index) => [`${key}.${index}`, value]) value,
])
), ),
}).map(([key, value]) => { }).map(([key, value]) => {
// to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/ // to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/
if ( if ((Array.isArray(value) && value.length === 1 && !value[0]) || !value) {
(Array.isArray(value) && value.length === 1 && !value[0]) || return ['requestBody', key];
!value
) {
return ["requestBody", key];
} else if (!Array.isArray(value)) { } else if (!Array.isArray(value)) {
return [ return ['raw', String.fromCharCode.apply(null, new Uint8Array(value.bytes))];
"raw",
String.fromCharCode.apply(null, new Uint8Array(value.bytes)),
];
} else { } else {
return [key, value || ""]; return [key, value || ''];
} }
}), }),
StolenDataEntry.parseValue StolenDataEntry.parseValue
).map( ).map(([key, value]) => new StolenDataEntry(this, 'request_body', key, value));
([key, value]) => new StolenDataEntry(this, "request_body", key, value)
);
return ret; return ret;
} }
hasReferer() { hasReferer() {
return this.requestHeaders.some((h) => h.name === "Referer"); return this.requestHeaders.some((h) => h.name === 'Referer');
} }
hasCookie() { hasCookie() {
return this.requestHeaders.some((h) => h.name === "Cookie"); return this.requestHeaders.some((h) => h.name === 'Cookie');
} }
getCookie(): string { getCookie(): string {
return this.requestHeaders.find((h) => h.name == "Cookie")?.value; return this.requestHeaders.find((h) => h.name == 'Cookie')?.value;
} }
getPathParams(): StolenDataEntry[] { getPathParams(): StolenDataEntry[] {
const url = new URL(this.data.url); const url = new URL(this.data.url);
const path = url.pathname; const path = url.pathname;
if (!path.includes(";")) { if (!path.includes(';')) {
return []; return [];
} }
return flattenObjectEntries( return flattenObjectEntries(
path path
.split(";") .split(';')
.map((e) => e.split("=")) .map((e) => e.split('='))
.map(([key, value]) => [key, value || ""]) .map(([key, value]) => [key, value || ''])
.map(([key, value]) => { .map(([key, value]) => {
return [ return [key, StolenDataEntry.parseValue(safeDecodeURIComponent(value))];
key,
StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
];
}) })
).map(([key, value]) => new StolenDataEntry(this, "pathname", key, value)); ).map(([key, value]) => new StolenDataEntry(this, 'pathname', key, value));
} }
getQueryParams(): StolenDataEntry[] { getQueryParams(): StolenDataEntry[] {
const url = new URL(this.data.url); const url = new URL(this.data.url);
return flattenObjectEntries( return flattenObjectEntries(
Array.from((url.searchParams as any).entries()) Array.from((url.searchParams as any).entries())
.map(([key, value]) => [key, value || ""]) .map(([key, value]) => [key, value || ''])
.map(([key, value]) => { .map(([key, value]) => {
return [ return [key, StolenDataEntry.parseValue(safeDecodeURIComponent(value))];
key,
StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
];
}) })
).map(([key, value]) => { ).map(([key, value]) => {
return new StolenDataEntry(this, "queryparams", key, value); return new StolenDataEntry(this, 'queryparams', key, value);
}); });
} }
@ -293,7 +286,7 @@ export default class ExtendedRequest {
StolenDataEntry.parseValue(safeDecodeURIComponent(header.value)), StolenDataEntry.parseValue(safeDecodeURIComponent(header.value)),
]; ];
}) })
).map(([key, value]) => new StolenDataEntry(this, "header", key, value)); ).map(([key, value]) => new StolenDataEntry(this, 'header', key, value));
} }
hasMark() { hasMark() {
@ -316,8 +309,8 @@ export default class ExtendedRequest {
toHAR(): HAREntry { toHAR(): HAREntry {
return { return {
pageref: "page_1", pageref: 'page_1',
startedDateTime: `${new Date().toJSON().replace("Z", "+01:00")}`, startedDateTime: `${new Date().toJSON().replace('Z', '+01:00')}`,
request: { request: {
bodySize: bodySize:
JSON.stringify(this.requestBody.formData || {}).length + JSON.stringify(this.requestBody.formData || {}).length +
@ -327,7 +320,7 @@ export default class ExtendedRequest {
method: this.data.method, method: this.data.method,
url: this.data.url, url: this.data.url,
headersSize: JSON.stringify(this.requestHeaders).length, headersSize: JSON.stringify(this.requestHeaders).length,
httpVersion: "HTTP/2", httpVersion: 'HTTP/2',
headers: this.requestHeaders as NameValue[], headers: this.requestHeaders as NameValue[],
cookies: this.getCookieData().map((cookie) => ({ cookies: this.getCookieData().map((cookie) => ({
name: cookie.name, name: cookie.name,
@ -338,35 +331,35 @@ export default class ExtendedRequest {
value: param.value, value: param.value,
})), })),
postData: { postData: {
mimeType: "application/x-www-form-urlencoded", mimeType: 'application/x-www-form-urlencoded',
params: this.stolenData params: this.stolenData
.filter((e) => e.source == "request_body") .filter((e) => e.source == 'request_body')
.map((e) => ({ .map((e) => ({
name: e.name, name: e.name,
value: e.value, value: e.value,
fileName: "--" + Math.ceil(Math.random() * 1000000000), fileName: '--' + Math.ceil(Math.random() * 1000000000),
contentType: "text/plain", contentType: 'text/plain',
comment: "", comment: '',
})), })),
text: this.stolenData text: this.stolenData
.filter((e) => e.source == "request_body") .filter((e) => e.source == 'request_body')
.map((e) => `${e.name}:\t${StolenDataEntry.parseValue(e.value)}`) .map((e) => `${e.name}:\t${StolenDataEntry.parseValue(e.value)}`)
.join("\n\n"), .join('\n\n'),
}, },
}, },
response: { response: {
status: 200, status: 200,
statusText: "OK", statusText: 'OK',
httpVersion: "HTTP/2", httpVersion: 'HTTP/2',
headers: [], headers: [],
cookies: [], cookies: [],
content: { content: {
mimeType: "text/plain", mimeType: 'text/plain',
size: this.getBalancedPriority(), size: this.getBalancedPriority(),
encoding: "base64", encoding: 'base64',
text: "ZG9lc24ndCBtYXR0ZXIK", text: 'ZG9lc24ndCBtYXR0ZXIK',
}, },
redirectURL: "", redirectURL: '',
headersSize: 15, headersSize: 15,
bodySize: 15, bodySize: 15,
}, },
@ -381,9 +374,9 @@ export default class ExtendedRequest {
receive: 0, receive: 0,
}, },
time: 79, time: 79,
_securityState: "secure", _securityState: 'secure',
serverIPAddress: "31.13.92.36", serverIPAddress: '31.13.92.36',
connection: "443", connection: '443',
}; };
} }
@ -402,10 +395,10 @@ export default class ExtendedRequest {
if (this.hasCookie()) { if (this.hasCookie()) {
result += 50; result += 50;
} }
if (this.stolenData.some((e) => e.classification === "location")) { if (this.stolenData.some((e) => e.classification === 'location')) {
result += 300; result += 300;
} }
if (this.url.includes("facebook")) { if (this.url.includes('facebook')) {
result += 50; result += 50;
} }
return result; return result;

View File

@ -4,7 +4,7 @@ import { EventEmitter } from 'events';
import { RequestCluster } from './request-cluster'; import { RequestCluster } from './request-cluster';
function setDomainsNumber(counter: number, tabId: number) { function setDomainsNumber(counter: number, tabId: number) {
browser.browserAction.setBadgeText({ text: counter.toString(), tabId }); browser.browserAction.setBadgeText({ text: counter < 0 ? '0' : counter.toString(), tabId });
browser.browserAction.setTitle({ browser.browserAction.setTitle({
title: 'Rentgen', title: 'Rentgen',
tabId, tabId,
@ -64,22 +64,67 @@ export default class Memory extends EventEmitter {
); );
} }
emit(eventName: string, immediate = false, data = 'any', reason: string) { private originalEmit(type: string, ...args: unknown[]) {
console.log('emitting!', eventName, data, reason); var doError = type === 'error';
setTimeout(() => super.emit(eventName, data), 0);
return; var events = (this as any)._events;
if (events !== undefined) doError = doError && events.error === undefined;
else if (!doError) return false;
// If there is no 'error' event listener then throw.
if (doError) {
var er;
if (args.length > 0) er = args[0];
if (er instanceof Error) {
// Note: The comments on the `throw` lines are intentional, they show
// up in Node's output if this results in an unhandled exception.
throw er; // Unhandled 'error' event
}
// At least give some kind of context to the user
var err = new Error('Unhandled error.' + (er ? ' (' + (er as any).message + ')' : ''));
(err as any).context = er;
throw err; // Unhandled 'error' event
}
var handler = events[type];
if (handler === undefined) return false;
if (typeof handler === 'function') {
try { try {
if (immediate) { Reflect.apply(handler, this, args);
super.emit(eventName, data); } catch (error) {
return; events[type] = undefined;
}
} else { } else {
this.throttle(() => super.emit(eventName, data)); // var len = handler.length;
var listeners = [...handler];
listeners
.filter((e) => {
try {
e.call;
} catch (error) {
return false;
} }
return true; return true;
} catch (e) { })
// debugger; .forEach((listener) => {
console.error(e); try {
Reflect.apply(listener, this, args);
} catch (error) {
console.error(error);
debugger;
} }
});
}
return true;
}
emit(eventName: string, immediate = false, data = 'any', reason: string): boolean {
setTimeout(() => this.originalEmit(eventName, data), 0);
return;
} }
getClustersForOrigin(origin: string): Record<string, RequestCluster> { getClustersForOrigin(origin: string): Record<string, RequestCluster> {

13
util.ts
View File

@ -56,18 +56,11 @@ export function useEmitter(
}); });
React.useEffect(() => { React.useEffect(() => {
const callback = (eventSubtype: string) => { const callback = (eventSubtype: string) => {
setEventCounts((eventCounts) => { setEventCounts((eventCounts) => ({
console.log({
...eventCounts, ...eventCounts,
...{ [eventSubtype]: (eventCounts[eventSubtype] || 0) + 1 }, ...{ [eventSubtype]: (eventCounts[eventSubtype] || 0) + 1 },
...{ '*': (eventCounts['*'] === undefined ? 0 : eventCounts['*']) + 1 }, ...{ '*': (eventCounts['*'] === undefined ? 0 : eventCounts['*']) + 1 },
}); }));
return {
...eventCounts,
...{ [eventSubtype]: (eventCounts[eventSubtype] || 0) + 1 },
...{ '*': (eventCounts['*'] === undefined ? 0 : eventCounts['*']) + 1 },
};
});
}; };
e.on('change', callback); e.on('change', callback);
return () => { return () => {
@ -220,6 +213,8 @@ export function flattenObject(
flattenObject(obj[i], parser, prefix + i, ret); flattenObject(obj[i], parser, prefix + i, ret);
} }
} }
} else if (obj === null) {
ret.push([key, '']);
} else if (typeof obj === 'object') { } else if (typeof obj === 'object') {
for (const [subkey, value] of Object.entries(obj)) { for (const [subkey, value] of Object.entries(obj)) {
flattenObject(value, parser, prefix + subkey, ret); flattenObject(value, parser, prefix + subkey, ret);