Add support for POST body inspection
This commit is contained in:
parent
da1789503b
commit
928effa1ad
|
@ -4,6 +4,7 @@ import {
|
||||||
getshorthost,
|
getshorthost,
|
||||||
parseCookie,
|
parseCookie,
|
||||||
Request,
|
Request,
|
||||||
|
safeDecodeURIComponent,
|
||||||
} from "./util";
|
} from "./util";
|
||||||
|
|
||||||
type NameValue = { name: string; value: string };
|
type NameValue = { name: string; value: string };
|
||||||
|
@ -60,22 +61,41 @@ const whitelisted_cookies = [
|
||||||
/^User-Agent$/,
|
/^User-Agent$/,
|
||||||
];
|
];
|
||||||
|
|
||||||
|
type RequestBody = {
|
||||||
|
error?: string;
|
||||||
|
formData?: Record<string, string[]>;
|
||||||
|
raw?: { bytes: ArrayBuffer; file?: string }[];
|
||||||
|
};
|
||||||
|
|
||||||
export default class ExtendedRequest {
|
export default class ExtendedRequest {
|
||||||
public tabId: number;
|
public tabId: number;
|
||||||
public url: string;
|
public url: string;
|
||||||
public shorthost: string;
|
public shorthost: string;
|
||||||
public requestHeaders: Request["requestHeaders"];
|
public requestHeaders: Request["requestHeaders"] = [];
|
||||||
public originalURL: string;
|
public originalURL: string;
|
||||||
public origin: string;
|
public origin: string;
|
||||||
public initialized = false;
|
public initialized = false;
|
||||||
public stolenData: StolenDataEntry[];
|
public stolenData: StolenDataEntry[];
|
||||||
public originalPathname: string;
|
public originalPathname: string;
|
||||||
|
public requestBody: RequestBody;
|
||||||
|
|
||||||
|
static by_id = {} as Record<string, ExtendedRequest>;
|
||||||
|
|
||||||
constructor(public data: Request) {
|
constructor(public data: Request) {
|
||||||
this.tabId = data.tabId;
|
this.tabId = data.tabId;
|
||||||
this.url = data.url;
|
this.url = data.url;
|
||||||
this.requestHeaders = data.requestHeaders;
|
|
||||||
this.shorthost = getshorthost(data.url);
|
this.shorthost = getshorthost(data.url);
|
||||||
|
this.requestBody =
|
||||||
|
((data as any).requestBody as undefined | RequestBody) || {};
|
||||||
|
if (this.url.includes("criteo")) {
|
||||||
|
console.log(this);
|
||||||
|
}
|
||||||
|
ExtendedRequest.by_id[data.requestId] = this;
|
||||||
|
}
|
||||||
|
|
||||||
|
addHeaders(headers: Request["requestHeaders"]) {
|
||||||
|
this.requestHeaders = headers;
|
||||||
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
async init() {
|
async init() {
|
||||||
|
@ -93,7 +113,7 @@ export default class ExtendedRequest {
|
||||||
url = (this.data as any).frameAncestors[0].url || "";
|
url = (this.data as any).frameAncestors[0].url || "";
|
||||||
} else {
|
} else {
|
||||||
const headers = Object.fromEntries(
|
const headers = Object.fromEntries(
|
||||||
this.data.requestHeaders.map(({ name, value }) => [name, value])
|
this.requestHeaders.map(({ name, value }) => [name, value])
|
||||||
);
|
);
|
||||||
if (headers.Referer) {
|
if (headers.Referer) {
|
||||||
url = headers.Referer;
|
url = headers.Referer;
|
||||||
|
@ -124,7 +144,7 @@ export default class ExtendedRequest {
|
||||||
|
|
||||||
getReferer() {
|
getReferer() {
|
||||||
return (
|
return (
|
||||||
this.data.requestHeaders.filter((h) => h.name === "Referer")[0]?.value ||
|
this.requestHeaders.filter((h) => h.name === "Referer")[0]?.value ||
|
||||||
"missing-referrer"
|
"missing-referrer"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
@ -155,6 +175,7 @@ export default class ExtendedRequest {
|
||||||
...this.getCookieData(),
|
...this.getCookieData(),
|
||||||
...this.getQueryParams(),
|
...this.getQueryParams(),
|
||||||
...this.getHeadersData(),
|
...this.getHeadersData(),
|
||||||
|
...this.getRequestBodyData(),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -171,12 +192,48 @@ export default class ExtendedRequest {
|
||||||
).map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
|
).map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
getRequestBodyData(): StolenDataEntry[] {
|
||||||
|
const ret = flattenObjectEntries(
|
||||||
|
Object.entries({
|
||||||
|
...this.requestBody.formData,
|
||||||
|
...Object.fromEntries(
|
||||||
|
Object.entries(
|
||||||
|
this.requestBody.raw || {}
|
||||||
|
).map(([key, value], index) => [`${key}.${index}`, value])
|
||||||
|
),
|
||||||
|
})
|
||||||
|
.map(([key, value]) => {
|
||||||
|
// to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/
|
||||||
|
if (
|
||||||
|
(Array.isArray(value) && value.length === 1 && !value[0]) ||
|
||||||
|
!value
|
||||||
|
) {
|
||||||
|
return ["requestBody", key];
|
||||||
|
} else if (!Array.isArray(value)) {
|
||||||
|
return [
|
||||||
|
"raw",
|
||||||
|
String.fromCharCode.apply(null, new Uint8Array(value.bytes)),
|
||||||
|
];
|
||||||
|
} else {
|
||||||
|
return [key, value || ""];
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.map(([key, value]) => {
|
||||||
|
const parsed = StolenDataEntry.parseValue(value);
|
||||||
|
return [key, parsed];
|
||||||
|
}) as [string, unknown][]
|
||||||
|
).map(
|
||||||
|
([key, value]) => new StolenDataEntry(this, "request_body", key, value)
|
||||||
|
);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
hasReferer() {
|
hasReferer() {
|
||||||
return this.data.requestHeaders.some((h) => h.name === "Referer");
|
return this.requestHeaders.some((h) => h.name === "Referer");
|
||||||
}
|
}
|
||||||
|
|
||||||
hasCookie() {
|
hasCookie() {
|
||||||
return this.data.requestHeaders.some((h) => h.name === "Cookie");
|
return this.requestHeaders.some((h) => h.name === "Cookie");
|
||||||
}
|
}
|
||||||
|
|
||||||
getCookie(): string {
|
getCookie(): string {
|
||||||
|
@ -195,7 +252,10 @@ export default class ExtendedRequest {
|
||||||
.map((e) => e.split("="))
|
.map((e) => e.split("="))
|
||||||
.map(([key, value]) => [key, value || ""])
|
.map(([key, value]) => [key, value || ""])
|
||||||
.map(([key, value]) => {
|
.map(([key, value]) => {
|
||||||
return [key, StolenDataEntry.parseValue(decodeURIComponent(value))];
|
return [
|
||||||
|
key,
|
||||||
|
StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
|
||||||
|
];
|
||||||
})
|
})
|
||||||
).map(([key, value]) => new StolenDataEntry(this, "pathname", key, value));
|
).map(([key, value]) => new StolenDataEntry(this, "pathname", key, value));
|
||||||
}
|
}
|
||||||
|
@ -206,7 +266,10 @@ export default class ExtendedRequest {
|
||||||
Array.from((url.searchParams as any).entries())
|
Array.from((url.searchParams as any).entries())
|
||||||
.map(([key, value]) => [key, value || ""])
|
.map(([key, value]) => [key, value || ""])
|
||||||
.map(([key, value]) => {
|
.map(([key, value]) => {
|
||||||
return [key, StolenDataEntry.parseValue(decodeURIComponent(value))];
|
return [
|
||||||
|
key,
|
||||||
|
StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
|
||||||
|
];
|
||||||
})
|
})
|
||||||
).map(([key, value]) => {
|
).map(([key, value]) => {
|
||||||
return new StolenDataEntry(this, "queryparams", key, value);
|
return new StolenDataEntry(this, "queryparams", key, value);
|
||||||
|
@ -215,7 +278,7 @@ export default class ExtendedRequest {
|
||||||
|
|
||||||
getHeadersData(): StolenDataEntry[] {
|
getHeadersData(): StolenDataEntry[] {
|
||||||
return flattenObjectEntries(
|
return flattenObjectEntries(
|
||||||
this.data.requestHeaders
|
this.requestHeaders
|
||||||
.filter((header) => {
|
.filter((header) => {
|
||||||
for (const regex of whitelisted_cookies) {
|
for (const regex of whitelisted_cookies) {
|
||||||
if (regex.test(header.name)) {
|
if (regex.test(header.name)) {
|
||||||
|
@ -261,7 +324,7 @@ export default class ExtendedRequest {
|
||||||
url: this.data.url,
|
url: this.data.url,
|
||||||
headersSize: 100,
|
headersSize: 100,
|
||||||
httpVersion: "HTTP/2",
|
httpVersion: "HTTP/2",
|
||||||
headers: this.data.requestHeaders as NameValue[],
|
headers: this.requestHeaders as NameValue[],
|
||||||
cookies: this.getCookieData().map((cookie) => ({
|
cookies: this.getCookieData().map((cookie) => ({
|
||||||
name: cookie.name,
|
name: cookie.name,
|
||||||
value: cookie.value,
|
value: cookie.value,
|
||||||
|
|
13
memory.ts
13
memory.ts
|
@ -26,9 +26,20 @@ export default class Memory extends EventEmitter {
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
|
|
||||||
|
browser.webRequest.onBeforeRequest.addListener(
|
||||||
|
async (request) => {
|
||||||
|
new ExtendedRequest(request);
|
||||||
|
},
|
||||||
|
{ urls: ["<all_urls>"] },
|
||||||
|
["requestBody"]
|
||||||
|
);
|
||||||
browser.webRequest.onBeforeSendHeaders.addListener(
|
browser.webRequest.onBeforeSendHeaders.addListener(
|
||||||
async (request) => {
|
async (request) => {
|
||||||
this.register(new ExtendedRequest(request));
|
const extendedRequest = ExtendedRequest.by_id[
|
||||||
|
request.requestId
|
||||||
|
].addHeaders(request.requestHeaders || []);
|
||||||
|
this.register(extendedRequest);
|
||||||
},
|
},
|
||||||
{ urls: ["<all_urls>"] },
|
{ urls: ["<all_urls>"] },
|
||||||
["requestHeaders"]
|
["requestHeaders"]
|
||||||
|
|
|
@ -1,10 +1,6 @@
|
||||||
import React from "react";
|
import React from "react";
|
||||||
import { RequestCluster } from "../request-cluster";
|
import { RequestCluster } from "../request-cluster";
|
||||||
import {
|
import { Classifications, Sources } from "../stolen-data-entry";
|
||||||
Classifications,
|
|
||||||
Sources,
|
|
||||||
StolenDataEntry,
|
|
||||||
} from "../stolen-data-entry";
|
|
||||||
|
|
||||||
const emailClassifications: Record<keyof typeof Classifications, string> = {
|
const emailClassifications: Record<keyof typeof Classifications, string> = {
|
||||||
id: "sztucznie nadane mi ID",
|
id: "sztucznie nadane mi ID",
|
||||||
|
@ -17,6 +13,7 @@ const emailSources: Record<Sources, string> = {
|
||||||
cookie: "z pliku Cookie",
|
cookie: "z pliku Cookie",
|
||||||
pathname: "jako części adresu URL",
|
pathname: "jako części adresu URL",
|
||||||
queryparams: "jako część adresu URL (query-params)",
|
queryparams: "jako część adresu URL (query-params)",
|
||||||
|
request_body: "w body zapytania POST",
|
||||||
};
|
};
|
||||||
|
|
||||||
export default function DomainSummary({
|
export default function DomainSummary({
|
||||||
|
|
|
@ -10,9 +10,15 @@ import {
|
||||||
isURL,
|
isURL,
|
||||||
maskString,
|
maskString,
|
||||||
parseToObject,
|
parseToObject,
|
||||||
|
safeDecodeURIComponent,
|
||||||
} from "./util";
|
} from "./util";
|
||||||
|
|
||||||
export type Sources = "cookie" | "pathname" | "queryparams" | "header";
|
export type Sources =
|
||||||
|
| "cookie"
|
||||||
|
| "pathname"
|
||||||
|
| "queryparams"
|
||||||
|
| "header"
|
||||||
|
| "request_body";
|
||||||
|
|
||||||
export const Classifications = <const>{
|
export const Classifications = <const>{
|
||||||
id: "Sztucznie nadane ID",
|
id: "Sztucznie nadane ID",
|
||||||
|
@ -243,14 +249,14 @@ export class StolenDataEntry extends EventEmitter {
|
||||||
exposesPath() {
|
exposesPath() {
|
||||||
return (
|
return (
|
||||||
this.request.originalPathname !== "/" &&
|
this.request.originalPathname !== "/" &&
|
||||||
[this.value, decodeURIComponent(this.value)].some((haystack) =>
|
[this.value, safeDecodeURIComponent(this.value)].some((haystack) =>
|
||||||
haystack.includes(this.request.originalPathname)
|
haystack.includes(this.request.originalPathname)
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
exposesHost() {
|
exposesHost() {
|
||||||
return [this.value, decodeURIComponent(this.value)].some((haystack) =>
|
return [this.value, safeDecodeURIComponent(this.value)].some((haystack) =>
|
||||||
haystack.includes(getshorthost(this.request.origin))
|
haystack.includes(getshorthost(this.request.origin))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
13
util.ts
13
util.ts
|
@ -104,7 +104,8 @@ export function isJSONObject(
|
||||||
str: unknown
|
str: unknown
|
||||||
): str is Record<string, unknown> | string | number {
|
): str is Record<string, unknown> | string | number {
|
||||||
try {
|
try {
|
||||||
return JSON.stringify(parseToObject(str))[0] == "{";
|
const firstChar = JSON.stringify(parseToObject(str))[0];
|
||||||
|
return ["{", "["].includes(firstChar);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -211,7 +212,7 @@ export function flattenObject(
|
||||||
ret.push([`${key}.${subkey}`, subvalue]);
|
ret.push([`${key}.${subkey}`, subvalue]);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ret.push([key, value.toString()]);
|
ret.push([key, value ? value.toString() : "<empty>"]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return ret;
|
return ret;
|
||||||
|
@ -239,3 +240,11 @@ export function maskString(
|
||||||
str.slice(str.length / 2 + amount_of_chars_to_cut / 2)
|
str.slice(str.length / 2 + amount_of_chars_to_cut / 2)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function safeDecodeURIComponent(s: string) {
|
||||||
|
try {
|
||||||
|
return decodeURIComponent(s);
|
||||||
|
} catch (e) {
|
||||||
|
return e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user