From 928effa1ad46d3c61c9965d4bedf630b745cd1ae Mon Sep 17 00:00:00 2001 From: Kuba Orlik Date: Fri, 26 Nov 2021 20:58:31 +0100 Subject: [PATCH] Add support for POST body inspection --- extended-request.ts | 83 ++++++++++++++++++++++++++++---- memory.ts | 13 ++++- report-window/domain-summary.tsx | 7 +-- stolen-data-entry.ts | 12 +++-- util.ts | 13 ++++- 5 files changed, 107 insertions(+), 21 deletions(-) diff --git a/extended-request.ts b/extended-request.ts index 1202e2d..f923fb3 100644 --- a/extended-request.ts +++ b/extended-request.ts @@ -4,6 +4,7 @@ import { getshorthost, parseCookie, Request, + safeDecodeURIComponent, } from "./util"; type NameValue = { name: string; value: string }; @@ -60,22 +61,41 @@ const whitelisted_cookies = [ /^User-Agent$/, ]; +type RequestBody = { + error?: string; + formData?: Record; + raw?: { bytes: ArrayBuffer; file?: string }[]; +}; + export default class ExtendedRequest { public tabId: number; public url: string; public shorthost: string; - public requestHeaders: Request["requestHeaders"]; + public requestHeaders: Request["requestHeaders"] = []; public originalURL: string; public origin: string; public initialized = false; public stolenData: StolenDataEntry[]; public originalPathname: string; + public requestBody: RequestBody; + + static by_id = {} as Record; constructor(public data: Request) { this.tabId = data.tabId; this.url = data.url; - this.requestHeaders = data.requestHeaders; this.shorthost = getshorthost(data.url); + this.requestBody = + ((data as any).requestBody as undefined | RequestBody) || {}; + if (this.url.includes("criteo")) { + console.log(this); + } + ExtendedRequest.by_id[data.requestId] = this; + } + + addHeaders(headers: Request["requestHeaders"]) { + this.requestHeaders = headers; + return this; } async init() { @@ -93,7 +113,7 @@ export default class ExtendedRequest { url = (this.data as any).frameAncestors[0].url || ""; } else { const headers = Object.fromEntries( - this.data.requestHeaders.map(({ name, value }) => [name, value]) + this.requestHeaders.map(({ name, value }) => [name, value]) ); if (headers.Referer) { url = headers.Referer; @@ -124,7 +144,7 @@ export default class ExtendedRequest { getReferer() { return ( - this.data.requestHeaders.filter((h) => h.name === "Referer")[0]?.value || + this.requestHeaders.filter((h) => h.name === "Referer")[0]?.value || "missing-referrer" ); } @@ -155,6 +175,7 @@ export default class ExtendedRequest { ...this.getCookieData(), ...this.getQueryParams(), ...this.getHeadersData(), + ...this.getRequestBodyData(), ]; } @@ -171,12 +192,48 @@ export default class ExtendedRequest { ).map(([key, value]) => new StolenDataEntry(this, "cookie", key, value)); } + getRequestBodyData(): StolenDataEntry[] { + const ret = flattenObjectEntries( + Object.entries({ + ...this.requestBody.formData, + ...Object.fromEntries( + Object.entries( + this.requestBody.raw || {} + ).map(([key, value], index) => [`${key}.${index}`, value]) + ), + }) + .map(([key, value]) => { + // to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/ + if ( + (Array.isArray(value) && value.length === 1 && !value[0]) || + !value + ) { + return ["requestBody", key]; + } else if (!Array.isArray(value)) { + return [ + "raw", + String.fromCharCode.apply(null, new Uint8Array(value.bytes)), + ]; + } else { + return [key, value || ""]; + } + }) + .map(([key, value]) => { + const parsed = StolenDataEntry.parseValue(value); + return [key, parsed]; + }) as [string, unknown][] + ).map( + ([key, value]) => new StolenDataEntry(this, "request_body", key, value) + ); + return ret; + } + hasReferer() { - return this.data.requestHeaders.some((h) => h.name === "Referer"); + return this.requestHeaders.some((h) => h.name === "Referer"); } hasCookie() { - return this.data.requestHeaders.some((h) => h.name === "Cookie"); + return this.requestHeaders.some((h) => h.name === "Cookie"); } getCookie(): string { @@ -195,7 +252,10 @@ export default class ExtendedRequest { .map((e) => e.split("=")) .map(([key, value]) => [key, value || ""]) .map(([key, value]) => { - return [key, StolenDataEntry.parseValue(decodeURIComponent(value))]; + return [ + key, + StolenDataEntry.parseValue(safeDecodeURIComponent(value)), + ]; }) ).map(([key, value]) => new StolenDataEntry(this, "pathname", key, value)); } @@ -206,7 +266,10 @@ export default class ExtendedRequest { Array.from((url.searchParams as any).entries()) .map(([key, value]) => [key, value || ""]) .map(([key, value]) => { - return [key, StolenDataEntry.parseValue(decodeURIComponent(value))]; + return [ + key, + StolenDataEntry.parseValue(safeDecodeURIComponent(value)), + ]; }) ).map(([key, value]) => { return new StolenDataEntry(this, "queryparams", key, value); @@ -215,7 +278,7 @@ export default class ExtendedRequest { getHeadersData(): StolenDataEntry[] { return flattenObjectEntries( - this.data.requestHeaders + this.requestHeaders .filter((header) => { for (const regex of whitelisted_cookies) { if (regex.test(header.name)) { @@ -261,7 +324,7 @@ export default class ExtendedRequest { url: this.data.url, headersSize: 100, httpVersion: "HTTP/2", - headers: this.data.requestHeaders as NameValue[], + headers: this.requestHeaders as NameValue[], cookies: this.getCookieData().map((cookie) => ({ name: cookie.name, value: cookie.value, diff --git a/memory.ts b/memory.ts index f6da3e4..6db0ad0 100644 --- a/memory.ts +++ b/memory.ts @@ -26,9 +26,20 @@ export default class Memory extends EventEmitter { constructor() { super(); + + browser.webRequest.onBeforeRequest.addListener( + async (request) => { + new ExtendedRequest(request); + }, + { urls: [""] }, + ["requestBody"] + ); browser.webRequest.onBeforeSendHeaders.addListener( async (request) => { - this.register(new ExtendedRequest(request)); + const extendedRequest = ExtendedRequest.by_id[ + request.requestId + ].addHeaders(request.requestHeaders || []); + this.register(extendedRequest); }, { urls: [""] }, ["requestHeaders"] diff --git a/report-window/domain-summary.tsx b/report-window/domain-summary.tsx index fc44d58..030c8ae 100644 --- a/report-window/domain-summary.tsx +++ b/report-window/domain-summary.tsx @@ -1,10 +1,6 @@ import React from "react"; import { RequestCluster } from "../request-cluster"; -import { - Classifications, - Sources, - StolenDataEntry, -} from "../stolen-data-entry"; +import { Classifications, Sources } from "../stolen-data-entry"; const emailClassifications: Record = { id: "sztucznie nadane mi ID", @@ -17,6 +13,7 @@ const emailSources: Record = { cookie: "z pliku Cookie", pathname: "jako części adresu URL", queryparams: "jako część adresu URL (query-params)", + request_body: "w body zapytania POST", }; export default function DomainSummary({ diff --git a/stolen-data-entry.ts b/stolen-data-entry.ts index b405285..f3deb06 100644 --- a/stolen-data-entry.ts +++ b/stolen-data-entry.ts @@ -10,9 +10,15 @@ import { isURL, maskString, parseToObject, + safeDecodeURIComponent, } from "./util"; -export type Sources = "cookie" | "pathname" | "queryparams" | "header"; +export type Sources = + | "cookie" + | "pathname" + | "queryparams" + | "header" + | "request_body"; export const Classifications = { id: "Sztucznie nadane ID", @@ -243,14 +249,14 @@ export class StolenDataEntry extends EventEmitter { exposesPath() { return ( this.request.originalPathname !== "/" && - [this.value, decodeURIComponent(this.value)].some((haystack) => + [this.value, safeDecodeURIComponent(this.value)].some((haystack) => haystack.includes(this.request.originalPathname) ) ); } exposesHost() { - return [this.value, decodeURIComponent(this.value)].some((haystack) => + return [this.value, safeDecodeURIComponent(this.value)].some((haystack) => haystack.includes(getshorthost(this.request.origin)) ); } diff --git a/util.ts b/util.ts index fd8c2f4..982b368 100644 --- a/util.ts +++ b/util.ts @@ -104,7 +104,8 @@ export function isJSONObject( str: unknown ): str is Record | string | number { try { - return JSON.stringify(parseToObject(str))[0] == "{"; + const firstChar = JSON.stringify(parseToObject(str))[0]; + return ["{", "["].includes(firstChar); } catch (e) { return false; } @@ -211,7 +212,7 @@ export function flattenObject( ret.push([`${key}.${subkey}`, subvalue]); } } else { - ret.push([key, value.toString()]); + ret.push([key, value ? value.toString() : ""]); } } return ret; @@ -239,3 +240,11 @@ export function maskString( str.slice(str.length / 2 + amount_of_chars_to_cut / 2) ); } + +export function safeDecodeURIComponent(s: string) { + try { + return decodeURIComponent(s); + } catch (e) { + return e; + } +}