import { StolenDataEntry } from "./stolen-data-entry"; import { flattenObjectEntries, getshorthost, parseCookie, Request, safeDecodeURIComponent, } from "./util"; type NameValue = { name: string; value: string }; export type HAREntry = { pageref: string; startedDateTime: string; request: { bodySize: number; cookies: NameValue[]; headers: NameValue[]; headersSize: number; httpVersion: string; method: string; postData?: { mimeType: string; params: (NameValue & { fileName: string; contentType: string; comment: ""; })[]; text: string; }; queryString: NameValue[]; url: string; }; response: { status: number; statusText: string; httpVersion: string; headers: NameValue[]; cookies: NameValue[]; content: { mimeType: string; size: number; encoding: "base64"; text: string; }; redirectURL: ""; headersSize: number; bodySize: number; }; // not relevant cache: {}; timings: {}; time: number; _securityState: string; serverIPAddress: string; connection: string; }; const whitelisted_cookies = [ /^Accept.*$/, /^Host$/, /^Connection$/, /^Sec-Fetch-.*$/, /^Content-Type$/, /^Cookie$/, // we're extracting it in getCookie separately anyway /^User-Agent$/, ]; type RequestBody = { error?: string; formData?: Record; raw?: { bytes: ArrayBuffer; file?: string }[]; }; export default class ExtendedRequest { public tabId: number; public url: string; public shorthost: string; public requestHeaders: Request["requestHeaders"] = []; public originalURL: string; public origin: string; public initialized = false; public stolenData: StolenDataEntry[]; public originalPathname: string; public requestBody: RequestBody; static by_id = {} as Record; constructor(public data: Request) { this.tabId = data.tabId; this.url = data.url; this.shorthost = getshorthost(data.url); this.requestBody = ((data as any).requestBody as undefined | RequestBody) || {}; if (this.url.includes("criteo")) { console.log(this); } ExtendedRequest.by_id[data.requestId] = this; } addHeaders(headers: Request["requestHeaders"]) { this.requestHeaders = headers; return this; } async init() { await this.cacheOrigin(); this.initialized = true; this.stolenData = this.getAllStolenData(); } async cacheOrigin(): Promise { let url: string; if (this.data.tabId && this.data.tabId >= 0) { const tab = await browser.tabs.get(this.data.tabId); url = tab.url; } else if ((this.data as any)?.frameAncestors) { url = (this.data as any).frameAncestors[0].url || ""; } else { const headers = Object.fromEntries( this.requestHeaders.map(({ name, value }) => [name, value]) ); if (headers.Referer) { url = headers.Referer; } else { url = this.data.url; } } this.originalURL = url; this.origin = new URL(url).origin; this.originalPathname = new URL(url).pathname; } isThirdParty() { const request_url = new URL(this.data.url); const origin_url = new URL(this.originalURL); if (request_url.host.includes(origin_url.host)) { return false; } if (getshorthost(request_url.host) == getshorthost(origin_url.host)) { return false; } return ( request_url.origin != origin_url.origin || (this.data as any).urlClassification.thirdParty.length > 0 ); } getReferer() { return ( this.requestHeaders.filter((h) => h.name === "Referer")[0]?.value || "missing-referrer" ); } exposesOrigin() { const url = new URL(this.originalURL); const host = url.host; const path = url.pathname; const shorthost = getshorthost(host); if (this.getReferer().includes(shorthost)) { return true; } for (const entry of this.stolenData) { if ( entry.value.includes(host) || entry.value.includes(path) || entry.value.includes(shorthost) ) { return true; } } return false; } private getAllStolenData(): StolenDataEntry[] { return [ ...this.getPathParams(), ...this.getCookieData(), ...this.getQueryParams(), ...this.getHeadersData(), ...this.getRequestBodyData(), ]; } getCookieData(): StolenDataEntry[] { if (!this.hasCookie() || this.getCookie() === undefined) { return []; } return flattenObjectEntries( Object.entries(parseCookie(this.getCookie())).map(([key, value]) => [ key, value || "", ]), StolenDataEntry.parseValue ).map(([key, value]) => new StolenDataEntry(this, "cookie", key, value)); } getRequestBodyData(): StolenDataEntry[] { const ret = flattenObjectEntries( Object.entries({ ...this.requestBody.formData, ...Object.fromEntries( Object.entries( this.requestBody.raw || {} ).map(([key, value], index) => [`${key}.${index}`, value]) ), }).map(([key, value]) => { // to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/ if ( (Array.isArray(value) && value.length === 1 && !value[0]) || !value ) { return ["requestBody", key]; } else if (!Array.isArray(value)) { return [ "raw", String.fromCharCode.apply(null, new Uint8Array(value.bytes)), ]; } else { return [key, value || ""]; } }), StolenDataEntry.parseValue ).map( ([key, value]) => new StolenDataEntry(this, "request_body", key, value) ); return ret; } hasReferer() { return this.requestHeaders.some((h) => h.name === "Referer"); } hasCookie() { return this.requestHeaders.some((h) => h.name === "Cookie"); } getCookie(): string { return this.requestHeaders.find((h) => h.name == "Cookie")?.value; } getPathParams(): StolenDataEntry[] { const url = new URL(this.data.url); const path = url.pathname; if (!path.includes(";")) { return []; } return flattenObjectEntries( path .split(";") .map((e) => e.split("=")) .map(([key, value]) => [key, value || ""]) .map(([key, value]) => { return [ key, StolenDataEntry.parseValue(safeDecodeURIComponent(value)), ]; }) ).map(([key, value]) => new StolenDataEntry(this, "pathname", key, value)); } getQueryParams(): StolenDataEntry[] { const url = new URL(this.data.url); return flattenObjectEntries( Array.from((url.searchParams as any).entries()) .map(([key, value]) => [key, value || ""]) .map(([key, value]) => { return [ key, StolenDataEntry.parseValue(safeDecodeURIComponent(value)), ]; }) ).map(([key, value]) => { return new StolenDataEntry(this, "queryparams", key, value); }); } getHeadersData(): StolenDataEntry[] { return flattenObjectEntries( this.requestHeaders .filter((header) => { for (const regex of whitelisted_cookies) { if (regex.test(header.name)) { return false; } } return true; }) .map((header) => { return [ header.name, StolenDataEntry.parseValue(safeDecodeURIComponent(header.value)), ]; }) ).map(([key, value]) => new StolenDataEntry(this, "header", key, value)); } hasMark() { return this.stolenData.some((data) => data.isMarked); } getMarkedEntries() { return this.stolenData.filter((data) => data.isMarked); } getHost() { return new URL(this.url).host; } matchesHAREntry(har: HAREntry): boolean { const rq = this.data; const hrq = har.request; return rq.url == hrq.url; } toHAR(): HAREntry { return { pageref: "page_1", startedDateTime: `${new Date().toJSON().replace("Z", "+01:00")}`, request: { bodySize: JSON.stringify(this.requestBody.formData || {}).length + (this.requestBody.raw || []) .map((e) => e.bytes.byteLength) .reduce((a, b) => a + b, 0), method: this.data.method, url: this.data.url, headersSize: JSON.stringify(this.requestHeaders).length, httpVersion: "HTTP/2", headers: this.requestHeaders as NameValue[], cookies: this.getCookieData().map((cookie) => ({ name: cookie.name, value: cookie.value, })), queryString: this.getQueryParams().map((param) => ({ name: param.name, value: param.value, })), postData: { mimeType: "application/x-www-form-urlencoded", params: this.stolenData .filter((e) => e.source == "request_body") .map((e) => ({ name: e.name, value: e.value, fileName: "--" + Math.ceil(Math.random() * 1000000000), contentType: "text/plain", comment: "", })), text: this.stolenData .filter((e) => e.source == "request_body") .map((e) => `${e.name}:\t${StolenDataEntry.parseValue(e.value)}`) .join("\n\n"), }, }, response: { status: 200, statusText: "OK", httpVersion: "HTTP/2", headers: [], cookies: [], content: { mimeType: "text/plain", size: this.getBalancedPriority(), encoding: "base64", text: "ZG9lc24ndCBtYXR0ZXIK", }, redirectURL: "", headersSize: 15, bodySize: 15, }, cache: {}, timings: { blocked: -1, dns: 0, connect: 0, ssl: 0, send: 0, wait: 79, receive: 0, }, time: 79, _securityState: "secure", serverIPAddress: "31.13.92.36", connection: "443", }; } getMaxPriority(): number { return Math.max(...this.stolenData.map((entry) => entry.getPriority())); } getBalancedPriority(): number { let result = 0; if (this.stolenData.some((e) => e.exposesPath())) { result += 50; } if (this.stolenData.some((e) => e.exposesHost())) { result += 50; } if (this.hasCookie()) { result += 50; } if (this.stolenData.some((e) => e.classification === "location")) { result += 300; } if (this.url.includes("facebook")) { result += 50; } return result; } }