Add support for POST body inspection

This commit is contained in:
Kuba Orlik 2021-11-26 20:58:31 +01:00
parent da1789503b
commit 928effa1ad
5 changed files with 107 additions and 21 deletions

View File

@ -4,6 +4,7 @@ import {
getshorthost, getshorthost,
parseCookie, parseCookie,
Request, Request,
safeDecodeURIComponent,
} from "./util"; } from "./util";
type NameValue = { name: string; value: string }; type NameValue = { name: string; value: string };
@ -60,22 +61,41 @@ const whitelisted_cookies = [
/^User-Agent$/, /^User-Agent$/,
]; ];
type RequestBody = {
error?: string;
formData?: Record<string, string[]>;
raw?: { bytes: ArrayBuffer; file?: string }[];
};
export default class ExtendedRequest { export default class ExtendedRequest {
public tabId: number; public tabId: number;
public url: string; public url: string;
public shorthost: string; public shorthost: string;
public requestHeaders: Request["requestHeaders"]; public requestHeaders: Request["requestHeaders"] = [];
public originalURL: string; public originalURL: string;
public origin: string; public origin: string;
public initialized = false; public initialized = false;
public stolenData: StolenDataEntry[]; public stolenData: StolenDataEntry[];
public originalPathname: string; public originalPathname: string;
public requestBody: RequestBody;
static by_id = {} as Record<string, ExtendedRequest>;
constructor(public data: Request) { constructor(public data: Request) {
this.tabId = data.tabId; this.tabId = data.tabId;
this.url = data.url; this.url = data.url;
this.requestHeaders = data.requestHeaders;
this.shorthost = getshorthost(data.url); this.shorthost = getshorthost(data.url);
this.requestBody =
((data as any).requestBody as undefined | RequestBody) || {};
if (this.url.includes("criteo")) {
console.log(this);
}
ExtendedRequest.by_id[data.requestId] = this;
}
addHeaders(headers: Request["requestHeaders"]) {
this.requestHeaders = headers;
return this;
} }
async init() { async init() {
@ -93,7 +113,7 @@ export default class ExtendedRequest {
url = (this.data as any).frameAncestors[0].url || ""; url = (this.data as any).frameAncestors[0].url || "";
} else { } else {
const headers = Object.fromEntries( const headers = Object.fromEntries(
this.data.requestHeaders.map(({ name, value }) => [name, value]) this.requestHeaders.map(({ name, value }) => [name, value])
); );
if (headers.Referer) { if (headers.Referer) {
url = headers.Referer; url = headers.Referer;
@ -124,7 +144,7 @@ export default class ExtendedRequest {
getReferer() { getReferer() {
return ( return (
this.data.requestHeaders.filter((h) => h.name === "Referer")[0]?.value || this.requestHeaders.filter((h) => h.name === "Referer")[0]?.value ||
"missing-referrer" "missing-referrer"
); );
} }
@ -155,6 +175,7 @@ export default class ExtendedRequest {
...this.getCookieData(), ...this.getCookieData(),
...this.getQueryParams(), ...this.getQueryParams(),
...this.getHeadersData(), ...this.getHeadersData(),
...this.getRequestBodyData(),
]; ];
} }
@ -171,12 +192,48 @@ export default class ExtendedRequest {
).map(([key, value]) => new StolenDataEntry(this, "cookie", key, value)); ).map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
} }
getRequestBodyData(): StolenDataEntry[] {
const ret = flattenObjectEntries(
Object.entries({
...this.requestBody.formData,
...Object.fromEntries(
Object.entries(
this.requestBody.raw || {}
).map(([key, value], index) => [`${key}.${index}`, value])
),
})
.map(([key, value]) => {
// to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/
if (
(Array.isArray(value) && value.length === 1 && !value[0]) ||
!value
) {
return ["requestBody", key];
} else if (!Array.isArray(value)) {
return [
"raw",
String.fromCharCode.apply(null, new Uint8Array(value.bytes)),
];
} else {
return [key, value || ""];
}
})
.map(([key, value]) => {
const parsed = StolenDataEntry.parseValue(value);
return [key, parsed];
}) as [string, unknown][]
).map(
([key, value]) => new StolenDataEntry(this, "request_body", key, value)
);
return ret;
}
hasReferer() { hasReferer() {
return this.data.requestHeaders.some((h) => h.name === "Referer"); return this.requestHeaders.some((h) => h.name === "Referer");
} }
hasCookie() { hasCookie() {
return this.data.requestHeaders.some((h) => h.name === "Cookie"); return this.requestHeaders.some((h) => h.name === "Cookie");
} }
getCookie(): string { getCookie(): string {
@ -195,7 +252,10 @@ export default class ExtendedRequest {
.map((e) => e.split("=")) .map((e) => e.split("="))
.map(([key, value]) => [key, value || ""]) .map(([key, value]) => [key, value || ""])
.map(([key, value]) => { .map(([key, value]) => {
return [key, StolenDataEntry.parseValue(decodeURIComponent(value))]; return [
key,
StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
];
}) })
).map(([key, value]) => new StolenDataEntry(this, "pathname", key, value)); ).map(([key, value]) => new StolenDataEntry(this, "pathname", key, value));
} }
@ -206,7 +266,10 @@ export default class ExtendedRequest {
Array.from((url.searchParams as any).entries()) Array.from((url.searchParams as any).entries())
.map(([key, value]) => [key, value || ""]) .map(([key, value]) => [key, value || ""])
.map(([key, value]) => { .map(([key, value]) => {
return [key, StolenDataEntry.parseValue(decodeURIComponent(value))]; return [
key,
StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
];
}) })
).map(([key, value]) => { ).map(([key, value]) => {
return new StolenDataEntry(this, "queryparams", key, value); return new StolenDataEntry(this, "queryparams", key, value);
@ -215,7 +278,7 @@ export default class ExtendedRequest {
getHeadersData(): StolenDataEntry[] { getHeadersData(): StolenDataEntry[] {
return flattenObjectEntries( return flattenObjectEntries(
this.data.requestHeaders this.requestHeaders
.filter((header) => { .filter((header) => {
for (const regex of whitelisted_cookies) { for (const regex of whitelisted_cookies) {
if (regex.test(header.name)) { if (regex.test(header.name)) {
@ -261,7 +324,7 @@ export default class ExtendedRequest {
url: this.data.url, url: this.data.url,
headersSize: 100, headersSize: 100,
httpVersion: "HTTP/2", httpVersion: "HTTP/2",
headers: this.data.requestHeaders as NameValue[], headers: this.requestHeaders as NameValue[],
cookies: this.getCookieData().map((cookie) => ({ cookies: this.getCookieData().map((cookie) => ({
name: cookie.name, name: cookie.name,
value: cookie.value, value: cookie.value,

View File

@ -26,9 +26,20 @@ export default class Memory extends EventEmitter {
constructor() { constructor() {
super(); super();
browser.webRequest.onBeforeRequest.addListener(
async (request) => {
new ExtendedRequest(request);
},
{ urls: ["<all_urls>"] },
["requestBody"]
);
browser.webRequest.onBeforeSendHeaders.addListener( browser.webRequest.onBeforeSendHeaders.addListener(
async (request) => { async (request) => {
this.register(new ExtendedRequest(request)); const extendedRequest = ExtendedRequest.by_id[
request.requestId
].addHeaders(request.requestHeaders || []);
this.register(extendedRequest);
}, },
{ urls: ["<all_urls>"] }, { urls: ["<all_urls>"] },
["requestHeaders"] ["requestHeaders"]

View File

@ -1,10 +1,6 @@
import React from "react"; import React from "react";
import { RequestCluster } from "../request-cluster"; import { RequestCluster } from "../request-cluster";
import { import { Classifications, Sources } from "../stolen-data-entry";
Classifications,
Sources,
StolenDataEntry,
} from "../stolen-data-entry";
const emailClassifications: Record<keyof typeof Classifications, string> = { const emailClassifications: Record<keyof typeof Classifications, string> = {
id: "sztucznie nadane mi ID", id: "sztucznie nadane mi ID",
@ -17,6 +13,7 @@ const emailSources: Record<Sources, string> = {
cookie: "z pliku Cookie", cookie: "z pliku Cookie",
pathname: "jako części adresu URL", pathname: "jako części adresu URL",
queryparams: "jako część adresu URL (query-params)", queryparams: "jako część adresu URL (query-params)",
request_body: "w body zapytania POST",
}; };
export default function DomainSummary({ export default function DomainSummary({

View File

@ -10,9 +10,15 @@ import {
isURL, isURL,
maskString, maskString,
parseToObject, parseToObject,
safeDecodeURIComponent,
} from "./util"; } from "./util";
export type Sources = "cookie" | "pathname" | "queryparams" | "header"; export type Sources =
| "cookie"
| "pathname"
| "queryparams"
| "header"
| "request_body";
export const Classifications = <const>{ export const Classifications = <const>{
id: "Sztucznie nadane ID", id: "Sztucznie nadane ID",
@ -243,14 +249,14 @@ export class StolenDataEntry extends EventEmitter {
exposesPath() { exposesPath() {
return ( return (
this.request.originalPathname !== "/" && this.request.originalPathname !== "/" &&
[this.value, decodeURIComponent(this.value)].some((haystack) => [this.value, safeDecodeURIComponent(this.value)].some((haystack) =>
haystack.includes(this.request.originalPathname) haystack.includes(this.request.originalPathname)
) )
); );
} }
exposesHost() { exposesHost() {
return [this.value, decodeURIComponent(this.value)].some((haystack) => return [this.value, safeDecodeURIComponent(this.value)].some((haystack) =>
haystack.includes(getshorthost(this.request.origin)) haystack.includes(getshorthost(this.request.origin))
); );
} }

13
util.ts
View File

@ -104,7 +104,8 @@ export function isJSONObject(
str: unknown str: unknown
): str is Record<string, unknown> | string | number { ): str is Record<string, unknown> | string | number {
try { try {
return JSON.stringify(parseToObject(str))[0] == "{"; const firstChar = JSON.stringify(parseToObject(str))[0];
return ["{", "["].includes(firstChar);
} catch (e) { } catch (e) {
return false; return false;
} }
@ -211,7 +212,7 @@ export function flattenObject(
ret.push([`${key}.${subkey}`, subvalue]); ret.push([`${key}.${subkey}`, subvalue]);
} }
} else { } else {
ret.push([key, value.toString()]); ret.push([key, value ? value.toString() : "<empty>"]);
} }
} }
return ret; return ret;
@ -239,3 +240,11 @@ export function maskString(
str.slice(str.length / 2 + amount_of_chars_to_cut / 2) str.slice(str.length / 2 + amount_of_chars_to_cut / 2)
); );
} }
export function safeDecodeURIComponent(s: string) {
try {
return decodeURIComponent(s);
} catch (e) {
return e;
}
}