icd/rentgen
icd/rentgen
2
1
Fork 0
Code Issues 72 Pull Requests Projects Releases 6 Wiki Activity

Handle dead objects

Browse Source
This commit is contained in:
Arkadiusz Wieczorek 2022-04-22 13:00:02 +02:00
parent d759727208
commit 1ba2bea2fb
4 changed files with 635 additions and 597 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
components/toolbar/toolbar.tsx
View File

@ -26,11 +26,16 @@ const Toolbar = () => {
React.useEffect(() => { React.useEffect(() => {
const listener = async () => { const listener = async () => {
const tab = await getCurrentTab(); const tab = await getCurrentTab();
const url = new URL(tab.url);
if (url.origin.startsWith('moz-extension')) { if (tab !== undefined) {
return; const url = new URL(tab.url);
if (url.origin.startsWith('moz-extension')) {
return;
}
setOrigin(url.origin);
} else {
console.warn('Out of the tab scope');
} }
setOrigin(url.origin);
}; };
browser.tabs.onUpdated.addListener(listener); browser.tabs.onUpdated.addListener(listener);
@ -217,7 +222,7 @@ const Toolbar = () => {
'fullscreen=yes', 'fullscreen=yes',
].join(','); ].join(',');
window.open( window.open(
`/report-window/report-window.html?origin=${origin}`, `/components/report-window/report-window.html?origin=${origin}`,
'new_window', 'new_window',
params params
); );

757
extended-request.ts
View File

@ -1,413 +1,406 @@
import { StolenDataEntry } from "./stolen-data-entry"; import { StolenDataEntry } from './stolen-data-entry';
import { import {
flattenObjectEntries, flattenObjectEntries,
getshorthost, getshorthost,
parseCookie, parseCookie,
Request, Request,
safeDecodeURIComponent, safeDecodeURIComponent,
} from "./util"; } from './util';
type NameValue = { name: string; value: string }; type NameValue = { name: string; value: string };
export type HAREntry = { export type HAREntry = {
pageref: string; pageref: string;
startedDateTime: string; startedDateTime: string;
request: { request: {
bodySize: number; bodySize: number;
cookies: NameValue[]; cookies: NameValue[];
headers: NameValue[]; headers: NameValue[];
headersSize: number; headersSize: number;
httpVersion: string; httpVersion: string;
method: string; method: string;
postData?: { postData?: {
mimeType: string; mimeType: string;
params: (NameValue & { params: (NameValue & {
fileName: string; fileName: string;
contentType: string; contentType: string;
comment: ""; comment: '';
})[]; })[];
text: string; text: string;
};
queryString: NameValue[];
url: string;
}; };
queryString: NameValue[]; response: {
url: string; status: number;
}; statusText: string;
response: { httpVersion: string;
status: number; headers: NameValue[];
statusText: string; cookies: NameValue[];
httpVersion: string; content: {
headers: NameValue[]; mimeType: string;
cookies: NameValue[]; size: number;
content: { encoding: 'base64';
mimeType: string; text: string;
size: number; };
encoding: "base64"; redirectURL: '';
text: string; headersSize: number;
}; bodySize: number;
redirectURL: ""; }; // not relevant
headersSize: number; cache: {};
bodySize: number; timings: {};
}; // not relevant time: number;
cache: {}; _securityState: string;
timings: {}; serverIPAddress: string;
time: number; connection: string;
_securityState: string;
serverIPAddress: string;
connection: string;
}; };
const whitelisted_cookies = [ const whitelisted_cookies = [
/^Accept.*$/, /^Accept.*$/,
/^Host$/, /^Host$/,
/^Connection$/, /^Connection$/,
/^Sec-Fetch-.*$/, /^Sec-Fetch-.*$/,
/^Content-Type$/, /^Content-Type$/,
/^Cookie$/, // we're extracting it in getCookie separately anyway /^Cookie$/, // we're extracting it in getCookie separately anyway
/^User-Agent$/, /^User-Agent$/,
]; ];
type RequestBody = { type RequestBody = {
error?: string; error?: string;
formData?: Record<string, string[]>; formData?: Record<string, string[]>;
raw?: { bytes: ArrayBuffer; file?: string }[]; raw?: { bytes: ArrayBuffer; file?: string }[];
}; };
export default class ExtendedRequest { export default class ExtendedRequest {
public tabId: number; public tabId: number;
public url: string; public url: string;
public shorthost: string; public shorthost: string;
public requestHeaders: Request["requestHeaders"] = []; public requestHeaders: Request['requestHeaders'] = [];
public originalURL: string; public originalURL: string;
public origin: string; public origin: string;
public initialized = false; public initialized = false;
public stolenData: StolenDataEntry[]; public stolenData: StolenDataEntry[];
public originalPathname: string; public originalPathname: string;
public requestBody: RequestBody; public requestBody: RequestBody;
static by_id = {} as Record<string, ExtendedRequest>; static by_id = {} as Record<string, ExtendedRequest>;
public data: Request;
constructor(public data: Request) { constructor(data: Request) {
this.tabId = data.tabId; this.tabId = data.tabId;
this.url = data.url; this.url = data.url;
this.shorthost = getshorthost(data.url); this.shorthost = getshorthost(data.url);
this.requestBody = this.requestBody = ((data as any).requestBody as undefined | RequestBody) || {};
((data as any).requestBody as undefined | RequestBody) || {}; if (this.url.includes('criteo')) {
if (this.url.includes("criteo")) { console.log(this);
console.log(this);
}
ExtendedRequest.by_id[data.requestId] = this;
}
addHeaders(headers: Request["requestHeaders"]) {
this.requestHeaders = headers;
return this;
}
async init() {
await this.cacheOrigin();
this.initialized = true;
this.stolenData = this.getAllStolenData();
}
async cacheOrigin(): Promise<void> {
let url: string;
if (this.data.tabId && this.data.tabId >= 0) {
const tab = await browser.tabs.get(this.data.tabId);
url = tab.url;
} else if ((this.data as any)?.frameAncestors) {
url = (this.data as any).frameAncestors[0].url || "";
} else {
const headers = Object.fromEntries(
this.requestHeaders.map(({ name, value }) => [name, value])
);
if (headers.Referer) {
url = headers.Referer;
} else {
url = this.data.url;
}
}
this.originalURL = url;
this.origin = new URL(url).origin;
this.originalPathname = new URL(url).pathname;
}
isThirdParty() {
const request_url = new URL(this.data.url);
const origin_url = new URL(this.originalURL);
if (request_url.host.includes(origin_url.host)) {
return false;
}
if (getshorthost(request_url.host) == getshorthost(origin_url.host)) {
return false;
}
return (
request_url.origin != origin_url.origin ||
(this.data as any).urlClassification.thirdParty.length > 0
);
}
getReferer() {
return (
this.requestHeaders.filter((h) => h.name === "Referer")[0]?.value ||
"missing-referrer"
);
}
exposesOrigin() {
const url = new URL(this.originalURL);
const host = url.host;
const path = url.pathname;
const shorthost = getshorthost(host);
if (this.getReferer().includes(shorthost)) {
return true;
}
for (const entry of this.stolenData) {
if (
entry.value.includes(host) ||
entry.value.includes(path) ||
entry.value.includes(shorthost)
) {
return true;
}
}
return false;
}
private getAllStolenData(): StolenDataEntry[] {
return [
...this.getPathParams(),
...this.getCookieData(),
...this.getQueryParams(),
...this.getHeadersData(),
...this.getRequestBodyData(),
];
}
getCookieData(): StolenDataEntry[] {
if (!this.hasCookie() || this.getCookie() === undefined) {
return [];
}
return flattenObjectEntries(
Object.entries(parseCookie(this.getCookie())).map(([key, value]) => [
key,
value || "",
]),
StolenDataEntry.parseValue
).map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
}
getRequestBodyData(): StolenDataEntry[] {
const ret = flattenObjectEntries(
Object.entries({
...this.requestBody.formData,
...Object.fromEntries(
Object.entries(
this.requestBody.raw || {}
).map(([key, value], index) => [`${key}.${index}`, value])
),
}).map(([key, value]) => {
// to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/
if (
(Array.isArray(value) && value.length === 1 && !value[0]) ||
!value
) {
return ["requestBody", key];
} else if (!Array.isArray(value)) {
return [
"raw",
String.fromCharCode.apply(null, new Uint8Array(value.bytes)),
];
} else {
return [key, value || ""];
} }
}), ExtendedRequest.by_id[data.requestId] = this;
StolenDataEntry.parseValue
).map(
([key, value]) => new StolenDataEntry(this, "request_body", key, value)
);
return ret;
}
hasReferer() { this.data = Object.assign({}, data);
return this.requestHeaders.some((h) => h.name === "Referer"); (this.data as any).frameAncestors = [
} ...(data as any).frameAncestors.map((e: any) => ({ url: e.url })),
];
hasCookie() { // console.log('→→→',(this.data as any).frameAncestors, (data as any).frameAncestors);
return this.requestHeaders.some((h) => h.name === "Cookie");
}
getCookie(): string {
return this.requestHeaders.find((h) => h.name == "Cookie")?.value;
}
getPathParams(): StolenDataEntry[] {
const url = new URL(this.data.url);
const path = url.pathname;
if (!path.includes(";")) {
return [];
} }
return flattenObjectEntries(
path
.split(";")
.map((e) => e.split("="))
.map(([key, value]) => [key, value || ""])
.map(([key, value]) => {
return [
key,
StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
];
})
).map(([key, value]) => new StolenDataEntry(this, "pathname", key, value));
}
getQueryParams(): StolenDataEntry[] { addHeaders(headers: Request['requestHeaders']) {
const url = new URL(this.data.url); this.requestHeaders = headers;
return flattenObjectEntries( return this;
Array.from((url.searchParams as any).entries()) }
.map(([key, value]) => [key, value || ""])
.map(([key, value]) => {
return [
key,
StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
];
})
).map(([key, value]) => {
return new StolenDataEntry(this, "queryparams", key, value);
});
}
getHeadersData(): StolenDataEntry[] { async init() {
return flattenObjectEntries( await this.cacheOrigin();
this.requestHeaders this.initialized = true;
.filter((header) => { this.stolenData = this.getAllStolenData();
for (const regex of whitelisted_cookies) { }
if (regex.test(header.name)) {
return false; async cacheOrigin(): Promise<void> {
let url: string;
if (this.data.tabId && this.data.tabId >= 0) {
const tab = await browser.tabs.get(this.data.tabId);
url = tab.url;
} else if (
(this.data as any)?.frameAncestors &&
(this.data as any).frameAncestors[0] !== undefined
) {
url = (this.data as any).frameAncestors[0].url || '';
} else {
const headers = Object.fromEntries(
this.requestHeaders.map(({ name, value }) => [name, value])
);
if (headers.Referer) {
url = headers.Referer;
} else {
url = this.data.url;
} }
} }
return true;
})
.map((header) => {
return [
header.name,
StolenDataEntry.parseValue(safeDecodeURIComponent(header.value)),
];
})
).map(([key, value]) => new StolenDataEntry(this, "header", key, value));
}
hasMark() { this.originalURL = url;
return this.stolenData.some((data) => data.isMarked); this.origin = new URL(url).origin;
} this.originalPathname = new URL(url).pathname;
getMarkedEntries() {
return this.stolenData.filter((data) => data.isMarked);
}
getHost() {
return new URL(this.url).host;
}
matchesHAREntry(har: HAREntry): boolean {
const rq = this.data;
const hrq = har.request;
return rq.url == hrq.url;
}
toHAR(): HAREntry {
return {
pageref: "page_1",
startedDateTime: `${new Date().toJSON().replace("Z", "+01:00")}`,
request: {
bodySize:
JSON.stringify(this.requestBody.formData || {}).length +
(this.requestBody.raw || [])
.map((e) => e.bytes.byteLength)
.reduce((a, b) => a + b, 0),
method: this.data.method,
url: this.data.url,
headersSize: JSON.stringify(this.requestHeaders).length,
httpVersion: "HTTP/2",
headers: this.requestHeaders as NameValue[],
cookies: this.getCookieData().map((cookie) => ({
name: cookie.name,
value: cookie.value,
})),
queryString: this.getQueryParams().map((param) => ({
name: param.name,
value: param.value,
})),
postData: {
mimeType: "application/x-www-form-urlencoded",
params: this.stolenData
.filter((e) => e.source == "request_body")
.map((e) => ({
name: e.name,
value: e.value,
fileName: "--" + Math.ceil(Math.random() * 1000000000),
contentType: "text/plain",
comment: "",
})),
text: this.stolenData
.filter((e) => e.source == "request_body")
.map((e) => `${e.name}:\t${StolenDataEntry.parseValue(e.value)}`)
.join("\n\n"),
},
},
response: {
status: 200,
statusText: "OK",
httpVersion: "HTTP/2",
headers: [],
cookies: [],
content: {
mimeType: "text/plain",
size: this.getBalancedPriority(),
encoding: "base64",
text: "ZG9lc24ndCBtYXR0ZXIK",
},
redirectURL: "",
headersSize: 15,
bodySize: 15,
},
cache: {},
timings: {
blocked: -1,
dns: 0,
connect: 0,
ssl: 0,
send: 0,
wait: 79,
receive: 0,
},
time: 79,
_securityState: "secure",
serverIPAddress: "31.13.92.36",
connection: "443",
};
}
getMaxPriority(): number {
return Math.max(...this.stolenData.map((entry) => entry.getPriority()));
}
getBalancedPriority(): number {
let result = 0;
if (this.stolenData.some((e) => e.exposesPath())) {
result += 50;
} }
if (this.stolenData.some((e) => e.exposesHost())) {
result += 50; isThirdParty() {
const request_url = new URL(this.data.url);
const origin_url = new URL(this.originalURL);
if (request_url.host.includes(origin_url.host)) {
return false;
}
if (getshorthost(request_url.host) == getshorthost(origin_url.host)) {
return false;
}
return (
request_url.origin != origin_url.origin ||
(this.data as any).urlClassification.thirdParty.length > 0
);
} }
if (this.hasCookie()) {
result += 50; getReferer() {
return (
this.requestHeaders.filter((h) => h.name === 'Referer')[0]?.value || 'missing-referrer'
);
} }
if (this.stolenData.some((e) => e.classification === "location")) {
result += 300; exposesOrigin() {
const url = new URL(this.originalURL);
const host = url.host;
const path = url.pathname;
const shorthost = getshorthost(host);
if (this.getReferer().includes(shorthost)) {
return true;
}
for (const entry of this.stolenData) {
if (
entry.value.includes(host) ||
entry.value.includes(path) ||
entry.value.includes(shorthost)
) {
return true;
}
}
return false;
} }
if (this.url.includes("facebook")) {
result += 50; private getAllStolenData(): StolenDataEntry[] {
return [
...this.getPathParams(),
...this.getCookieData(),
...this.getQueryParams(),
...this.getHeadersData(),
...this.getRequestBodyData(),
];
}
getCookieData(): StolenDataEntry[] {
if (!this.hasCookie() || this.getCookie() === undefined) {
return [];
}
return flattenObjectEntries(
Object.entries(parseCookie(this.getCookie())).map(([key, value]) => [key, value || '']),
StolenDataEntry.parseValue
).map(([key, value]) => new StolenDataEntry(this, 'cookie', key, value));
}
getRequestBodyData(): StolenDataEntry[] {
const ret = flattenObjectEntries(
Object.entries({
...this.requestBody.formData,
...Object.fromEntries(
Object.entries(this.requestBody.raw || {}).map(([key, value], index) => [
`${key}.${index}`,
value,
])
),
}).map(([key, value]) => {
// to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/
if ((Array.isArray(value) && value.length === 1 && !value[0]) || !value) {
return ['requestBody', key];
} else if (!Array.isArray(value)) {
return ['raw', String.fromCharCode.apply(null, new Uint8Array(value.bytes))];
} else {
return [key, value || ''];
}
}),
StolenDataEntry.parseValue
).map(([key, value]) => new StolenDataEntry(this, 'request_body', key, value));
return ret;
}
hasReferer() {
return this.requestHeaders.some((h) => h.name === 'Referer');
}
hasCookie() {
return this.requestHeaders.some((h) => h.name === 'Cookie');
}
getCookie(): string {
return this.requestHeaders.find((h) => h.name == 'Cookie')?.value;
}
getPathParams(): StolenDataEntry[] {
const url = new URL(this.data.url);
const path = url.pathname;
if (!path.includes(';')) {
return [];
}
return flattenObjectEntries(
path
.split(';')
.map((e) => e.split('='))
.map(([key, value]) => [key, value || ''])
.map(([key, value]) => {
return [key, StolenDataEntry.parseValue(safeDecodeURIComponent(value))];
})
).map(([key, value]) => new StolenDataEntry(this, 'pathname', key, value));
}
getQueryParams(): StolenDataEntry[] {
const url = new URL(this.data.url);
return flattenObjectEntries(
Array.from((url.searchParams as any).entries())
.map(([key, value]) => [key, value || ''])
.map(([key, value]) => {
return [key, StolenDataEntry.parseValue(safeDecodeURIComponent(value))];
})
).map(([key, value]) => {
return new StolenDataEntry(this, 'queryparams', key, value);
});
}
getHeadersData(): StolenDataEntry[] {
return flattenObjectEntries(
this.requestHeaders
.filter((header) => {
for (const regex of whitelisted_cookies) {
if (regex.test(header.name)) {
return false;
}
}
return true;
})
.map((header) => {
return [
header.name,
StolenDataEntry.parseValue(safeDecodeURIComponent(header.value)),
];
})
).map(([key, value]) => new StolenDataEntry(this, 'header', key, value));
}
hasMark() {
return this.stolenData.some((data) => data.isMarked);
}
getMarkedEntries() {
return this.stolenData.filter((data) => data.isMarked);
}
getHost() {
return new URL(this.url).host;
}
matchesHAREntry(har: HAREntry): boolean {
const rq = this.data;
const hrq = har.request;
return rq.url == hrq.url;
}
toHAR(): HAREntry {
return {
pageref: 'page_1',
startedDateTime: `${new Date().toJSON().replace('Z', '+01:00')}`,
request: {
bodySize:
JSON.stringify(this.requestBody.formData || {}).length +
(this.requestBody.raw || [])
.map((e) => e.bytes.byteLength)
.reduce((a, b) => a + b, 0),
method: this.data.method,
url: this.data.url,
headersSize: JSON.stringify(this.requestHeaders).length,
httpVersion: 'HTTP/2',
headers: this.requestHeaders as NameValue[],
cookies: this.getCookieData().map((cookie) => ({
name: cookie.name,
value: cookie.value,
})),
queryString: this.getQueryParams().map((param) => ({
name: param.name,
value: param.value,
})),
postData: {
mimeType: 'application/x-www-form-urlencoded',
params: this.stolenData
.filter((e) => e.source == 'request_body')
.map((e) => ({
name: e.name,
value: e.value,
fileName: '--' + Math.ceil(Math.random() * 1000000000),
contentType: 'text/plain',
comment: '',
})),
text: this.stolenData
.filter((e) => e.source == 'request_body')
.map((e) => `${e.name}:\t${StolenDataEntry.parseValue(e.value)}`)
.join('\n\n'),
},
},
response: {
status: 200,
statusText: 'OK',
httpVersion: 'HTTP/2',
headers: [],
cookies: [],
content: {
mimeType: 'text/plain',
size: this.getBalancedPriority(),
encoding: 'base64',
text: 'ZG9lc24ndCBtYXR0ZXIK',
},
redirectURL: '',
headersSize: 15,
bodySize: 15,
},
cache: {},
timings: {
blocked: -1,
dns: 0,
connect: 0,
ssl: 0,
send: 0,
wait: 79,
receive: 0,
},
time: 79,
_securityState: 'secure',
serverIPAddress: '31.13.92.36',
connection: '443',
};
}
getMaxPriority(): number {
return Math.max(...this.stolenData.map((entry) => entry.getPriority()));
}
getBalancedPriority(): number {
let result = 0;
if (this.stolenData.some((e) => e.exposesPath())) {
result += 50;
}
if (this.stolenData.some((e) => e.exposesHost())) {
result += 50;
}
if (this.hasCookie()) {
result += 50;
}
if (this.stolenData.some((e) => e.classification === 'location')) {
result += 300;
}
if (this.url.includes('facebook')) {
result += 50;
}
return result;
} }
return result;
}
} }

75
memory.ts
View File

@ -4,7 +4,7 @@ import { EventEmitter } from 'events';
import { RequestCluster } from './request-cluster'; import { RequestCluster } from './request-cluster';
function setDomainsNumber(counter: number, tabId: number) { function setDomainsNumber(counter: number, tabId: number) {
browser.browserAction.setBadgeText({ text: counter.toString(), tabId }); browser.browserAction.setBadgeText({ text: counter < 0 ? '0' : counter.toString(), tabId });
browser.browserAction.setTitle({ browser.browserAction.setTitle({
title: 'Rentgen', title: 'Rentgen',
tabId, tabId,
@ -64,22 +64,67 @@ export default class Memory extends EventEmitter {
); );
} }
emit(eventName: string, immediate = false, data = 'any', reason: string) { private originalEmit(type: string, ...args: unknown[]) {
console.log('emitting!', eventName, data, reason); var doError = type === 'error';
setTimeout(() => super.emit(eventName, data), 0);
return; var events = (this as any)._events;
try { if (events !== undefined) doError = doError && events.error === undefined;
if (immediate) { else if (!doError) return false;
super.emit(eventName, data);
return; // If there is no 'error' event listener then throw.
} else { if (doError) {
this.throttle(() => super.emit(eventName, data)); var er;
if (args.length > 0) er = args[0];
if (er instanceof Error) {
// Note: The comments on the `throw` lines are intentional, they show
// up in Node's output if this results in an unhandled exception.
throw er; // Unhandled 'error' event
} }
return true; // At least give some kind of context to the user
} catch (e) { var err = new Error('Unhandled error.' + (er ? ' (' + (er as any).message + ')' : ''));
// debugger; (err as any).context = er;
console.error(e); throw err; // Unhandled 'error' event
} }
var handler = events[type];
if (handler === undefined) return false;
if (typeof handler === 'function') {
try {
Reflect.apply(handler, this, args);
} catch (error) {
events[type] = undefined;
}
} else {
// var len = handler.length;
var listeners = [...handler];
listeners
.filter((e) => {
try {
e.call;
} catch (error) {
return false;
}
return true;
})
.forEach((listener) => {
try {
Reflect.apply(listener, this, args);
} catch (error) {
console.error(error);
debugger;
}
});
}
return true;
}
emit(eventName: string, immediate = false, data = 'any', reason: string): boolean {
setTimeout(() => this.originalEmit(eventName, data), 0);
return;
} }
getClustersForOrigin(origin: string): Record<string, RequestCluster> { getClustersForOrigin(origin: string): Record<string, RequestCluster> {

385
util.ts
View File

@ -6,273 +6,268 @@ export type Unarray<T> = T extends Array<infer R> ? R : T;
export type Tab = Unarray<Unpromisify<ReturnType<typeof browser.tabs.query>>>; export type Tab = Unarray<Unpromisify<ReturnType<typeof browser.tabs.query>>>;
export type Request = { export type Request = {
cookieStoreId?: string; cookieStoreId?: string;
documentUrl?: string; // RL of the document in which the resource will be loaded. For example, if the web page at "https://example.com" contains an image or an iframe, then the documentUrl for the image or iframe will be "https://example.com". For a top-level document, documentUrl is undefined. documentUrl?: string; // RL of the document in which the resource will be loaded. For example, if the web page at "https://example.com" contains an image or an iframe, then the documentUrl for the image or iframe will be "https://example.com". For a top-level document, documentUrl is undefined.
frameId: number; frameId: number;
incognito?: boolean; incognito?: boolean;
method: string; method: string;
originUrl: string; originUrl: string;
parentFrameId: number; parentFrameId: number;
proxyInfo?: { proxyInfo?: {
host: string; host: string;
port: number; port: number;
type: string; type: string;
username: string; username: string;
proxyDNS: boolean; proxyDNS: boolean;
failoverTimeout: number; failoverTimeout: number;
}; };
requestHeaders?: { name: string; value?: string; binaryValue?: number[] }[]; requestHeaders?: { name: string; value?: string; binaryValue?: number[] }[];
requestId: string; requestId: string;
tabId: number; tabId: number;
thirdParty?: boolean; thirdParty?: boolean;
timeStamp: number; timeStamp: number;
type: string; type: string;
url: string; // the target of the request; url: string; // the target of the request;
urlClassification?: { firstParty: string[]; thirdParty: string[] }; urlClassification?: { firstParty: string[]; thirdParty: string[] };
}; };
export function getshorthost(host: string) { export function getshorthost(host: string) {
const parts = host const parts = host
.replace(/^.*:\/\//, '') .replace(/^.*:\/\//, '')
.replace(/\/.*$/, '') .replace(/\/.*$/, '')
.split('.'); .split('.');
let lookback = !['co', 'com'].includes(parts.at(-2)) ? -2 : -3; let lookback = !['co', 'com'].includes(parts.at(-2)) ? -2 : -3;
if (parts.at(-2) == 'doubleclick' || parts.at(-2) == 'google') { if (parts.at(-2) == 'doubleclick' || parts.at(-2) == 'google') {
lookback = -4; // to distinguish between google ads and stats lookback = -4; // to distinguish between google ads and stats
} else if (parts.at(-2) == 'google') { } else if (parts.at(-2) == 'google') {
lookback = -3; // to distinguish various google services lookback = -3; // to distinguish various google services
} }
return parts.slice(lookback).join('.'); return parts.slice(lookback).join('.');
} }
export function useEmitter( export function useEmitter(
e: EventEmitter e: EventEmitter
): [ ): [
Record<string, number | undefined>, Record<string, number | undefined>,
React.Dispatch<React.SetStateAction<Record<string, number | undefined>>> React.Dispatch<React.SetStateAction<Record<string, number | undefined>>>
] { ] {
const [eventCounts, setEventCounts] = React.useState<Record<string, number | undefined>>({ const [eventCounts, setEventCounts] = React.useState<Record<string, number | undefined>>({
'*': 0, '*': 0,
}); });
React.useEffect(() => { React.useEffect(() => {
const callback = (eventSubtype: string) => { const callback = (eventSubtype: string) => {
setEventCounts((eventCounts) => { setEventCounts((eventCounts) => ({
console.log({ ...eventCounts,
...eventCounts, ...{ [eventSubtype]: (eventCounts[eventSubtype] || 0) + 1 },
...{ [eventSubtype]: (eventCounts[eventSubtype] || 0) + 1 }, ...{ '*': (eventCounts['*'] === undefined ? 0 : eventCounts['*']) + 1 },
...{ '*': (eventCounts['*'] === undefined ? 0 : eventCounts['*']) + 1 }, }));
}); };
return { e.on('change', callback);
...eventCounts, return () => {
...{ [eventSubtype]: (eventCounts[eventSubtype] || 0) + 1 }, e.removeListener('change', callback);
...{ '*': (eventCounts['*'] === undefined ? 0 : eventCounts['*']) + 1 }, };
}; }, []);
}); return [eventCounts, setEventCounts];
};
e.on('change', callback);
return () => {
e.removeListener('change', callback);
};
}, []);
return [eventCounts, setEventCounts];
} }
export function parseCookie(cookie: string): Record<string, string> { export function parseCookie(cookie: string): Record<string, string> {
return cookie return cookie
.split(';') .split(';')
.map((l) => l.split('=')) .map((l) => l.split('='))
.reduce( .reduce(
(acc, [key, value]) => ({ (acc, [key, value]) => ({
...acc, ...acc,
[key]: value, [key]: value,
}), }),
{} {}
); );
} }
export async function getTabByID(id: number) { export async function getTabByID(id: number) {
const tabs = await browser.tabs.query({ currentWindow: true }); const tabs = await browser.tabs.query({ currentWindow: true });
return tabs.find((tab) => tab.id == id); return tabs.find((tab) => tab.id == id);
} }
export function parseToObject(str: unknown): Record<string | symbol, unknown> { export function parseToObject(str: unknown): Record<string | symbol, unknown> {
let result: Record<string | symbol, unknown>; let result: Record<string | symbol, unknown>;
let original_string: string; let original_string: string;
if (typeof str === 'string') { if (typeof str === 'string') {
original_string = str; original_string = str;
result = JSON.parse(str); result = JSON.parse(str);
} else if (typeof str == 'object') { } else if (typeof str == 'object') {
result = str as Record<string | symbol, unknown>; result = str as Record<string | symbol, unknown>;
original_string = (result[Symbol.for('originalString')] as string) || JSON.stringify(str); original_string = (result[Symbol.for('originalString')] as string) || JSON.stringify(str);
} }
result[Symbol.for('originalString')] = original_string; result[Symbol.for('originalString')] = original_string;
return result; return result;
} }
export function isJSONObject(str: unknown): str is Record<string, unknown> | string | number { export function isJSONObject(str: unknown): str is Record<string, unknown> | string | number {
try { try {
const firstChar = JSON.stringify(parseToObject(str))[0]; const firstChar = JSON.stringify(parseToObject(str))[0];
return ['{', '['].includes(firstChar); return ['{', '['].includes(firstChar);
} catch (e) { } catch (e) {
return false; return false;
} }
} }
export function isURL(str: unknown): str is string { export function isURL(str: unknown): str is string {
try { try {
return !!(typeof str === 'string' && new URL(str)); return !!(typeof str === 'string' && new URL(str));
} catch (e) { } catch (e) {
return false; return false;
} }
} }
export function hyphenate(str: string): string { export function hyphenate(str: string): string {
return str.replace(/[_\[A-Z]/g, `${String.fromCharCode(173)}$&`); return str.replace(/[_\[A-Z]/g, `${String.fromCharCode(173)}$&`);
} }
export function unique<T>(array: T[]): Array<T> { export function unique<T>(array: T[]): Array<T> {
return Array.from(new Set<T>(array)); return Array.from(new Set<T>(array));
} }
export function allSubhosts(host: string) { export function allSubhosts(host: string) {
const parts = host.split('.'); const parts = host.split('.');
const result = []; const result = [];
for (let i = 0; i < parts.length - 2; i++) { for (let i = 0; i < parts.length - 2; i++) {
result.push(parts.slice(i).join('.')); result.push(parts.slice(i).join('.'));
} }
return result; return result;
} }
export function reduceConcat<T>(a: T[], b: T[]): T[] { export function reduceConcat<T>(a: T[], b: T[]): T[] {
return a.concat(b); return a.concat(b);
} }
export function getDate() { export function getDate() {
const d = new Date(); const d = new Date();
return `${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, '0')}-${d return `${d.getFullYear()}-${(d.getMonth() + 1).toString().padStart(2, '0')}-${d
.getDate() .getDate()
.toString() .toString()
.padStart(2, '0')}`; .padStart(2, '0')}`;
} }
export function toBase64(file: File): Promise<string> { export function toBase64(file: File): Promise<string> {
return new Promise((resolve) => { return new Promise((resolve) => {
const FR = new FileReader(); const FR = new FileReader();
FR.addEventListener('load', (e) => { FR.addEventListener('load', (e) => {
resolve(e.target.result as string); resolve(e.target.result as string);
}); });
FR.readAsDataURL(file); FR.readAsDataURL(file);
}); });
} }
export function makeThrottle(interval: number) { export function makeThrottle(interval: number) {
let last_emit = 0; let last_emit = 0;
function emit(callback: () => void) { function emit(callback: () => void) {
if (Date.now() - last_emit > interval) { if (Date.now() - last_emit > interval) {
callback(); callback();
last_emit = Date.now(); last_emit = Date.now();
return true; return true;
} else { } else {
return false; return false;
} }
} }
return function (callback: () => void) { return function (callback: () => void) {
if (!emit(callback)) { if (!emit(callback)) {
setTimeout(() => emit(callback), interval); setTimeout(() => emit(callback), interval);
} }
}; };
} }
export function isSameURL(url1: string, url2: string): boolean { export function isSameURL(url1: string, url2: string): boolean {
if (url1 === url2) { if (url1 === url2) {
return true; return true;
} }
url1 = url1.replace(/^https?:\/\//, '').replace(/\/$/, ''); url1 = url1.replace(/^https?:\/\//, '').replace(/\/$/, '');
url2 = url2.replace(/^https?:\/\//, '').replace(/\/$/, ''); url2 = url2.replace(/^https?:\/\//, '').replace(/\/$/, '');
return url1 === url2; return url1 === url2;
} }
export function isBase64(s: string): boolean { export function isBase64(s: string): boolean {
try { try {
atob(s); atob(s);
return true; return true;
} catch (e) {} } catch (e) {}
return false; return false;
} }
export function isBase64JSON(s: unknown): s is string { export function isBase64JSON(s: unknown): s is string {
return typeof s === 'string' && isBase64(s) && isJSONObject(atob(s)); return typeof s === 'string' && isBase64(s) && isJSONObject(atob(s));
} }
export function flattenObject( export function flattenObject(
obj: unknown, obj: unknown,
parser: (to_parse: unknown) => string | Record<string, unknown> = (id) => id.toString(), parser: (to_parse: unknown) => string | Record<string, unknown> = (id) => id.toString(),
key = '', key = '',
ret = [] as [string, string][], ret = [] as [string, string][],
parsed = false parsed = false
): [string, string][] { ): [string, string][] {
const prefix = key === '' ? '' : `${key}.`; const prefix = key === '' ? '' : `${key}.`;
if (Array.isArray(obj)) { if (Array.isArray(obj)) {
if (obj.length == 1) { if (obj.length == 1) {
flattenObject(obj[0], parser, key, ret); flattenObject(obj[0], parser, key, ret);
} else { } else {
for (let i in obj) { for (let i in obj) {
flattenObject(obj[i], parser, prefix + i, ret); flattenObject(obj[i], parser, prefix + i, ret);
} }
} }
} else if (typeof obj === 'object') { } else if (obj === null) {
for (const [subkey, value] of Object.entries(obj)) { ret.push([key, '']);
flattenObject(value, parser, prefix + subkey, ret); } else if (typeof obj === 'object') {
} for (const [subkey, value] of Object.entries(obj)) {
} else if (!parsed) { flattenObject(value, parser, prefix + subkey, ret);
flattenObject(parser(obj), parser, key, ret, true); }
} else if (typeof obj === 'string') { } else if (!parsed) {
ret.push([key, obj]); flattenObject(parser(obj), parser, key, ret, true);
} else { } else if (typeof obj === 'string') {
throw new Error('Something went wrong when parsing ' + obj); ret.push([key, obj]);
} } else {
return ret; throw new Error('Something went wrong when parsing ' + obj);
}
return ret;
} }
export function flattenObjectEntries( export function flattenObjectEntries(
entries: [string, unknown][], entries: [string, unknown][],
parser: (to_parse: unknown) => string | Record<string, unknown> = (id) => id.toString() parser: (to_parse: unknown) => string | Record<string, unknown> = (id) => id.toString()
): [string, string][] { ): [string, string][] {
return flattenObject(Object.fromEntries(entries), parser); return flattenObject(Object.fromEntries(entries), parser);
} }
export function maskString( export function maskString(
str: string, str: string,
max_fraction_remaining: number, max_fraction_remaining: number,
max_chars_total: number max_chars_total: number
): string { ): string {
const amount_of_chars_to_cut = const amount_of_chars_to_cut =
str.length - Math.min(str.length * max_fraction_remaining, max_chars_total); str.length - Math.min(str.length * max_fraction_remaining, max_chars_total);
if (amount_of_chars_to_cut == 0) { if (amount_of_chars_to_cut == 0) {
return str; return str;
} }
return ( return (
str.slice(0, str.length / 2 - amount_of_chars_to_cut / 2) + str.slice(0, str.length / 2 - amount_of_chars_to_cut / 2) +
'(...)' + '(...)' +
str.slice(str.length / 2 + amount_of_chars_to_cut / 2) str.slice(str.length / 2 + amount_of_chars_to_cut / 2)
); );
} }
export function safeDecodeURIComponent(s: string) { export function safeDecodeURIComponent(s: string) {
try { try {
return decodeURIComponent(s); return decodeURIComponent(s);
} catch (e) { } catch (e) {
return s; return s;
} }
} }
export function normalizeForClassname(string: string) { export function normalizeForClassname(string: string) {
return string.replace(/[^a-z0-9]/gi, '-'); return string.replace(/[^a-z0-9]/gi, '-');
} }
export function wordlist(words: string[]) { export function wordlist(words: string[]) {
return words.reduce( return words.reduce(
(acc, word, i) => `${acc}${i > 0 ? (i < words.length - 1 ? ',' : ' i') : ''} ${word}`, (acc, word, i) => `${acc}${i > 0 ? (i < words.length - 1 ? ',' : ' i') : ''} ${word}`,
'' ''
); );
} }