Add support for POST body inspection

extended-request.ts

@@ -4,6 +4,7 @@ import {
   getshorthost,
   parseCookie,
   Request,
+  safeDecodeURIComponent,
 } from "./util";
 
 type NameValue = { name: string; value: string };
@@ -60,22 +61,41 @@ const whitelisted_cookies = [
   /^User-Agent$/,
 ];
 
+type RequestBody = {
+  error?: string;
+  formData?: Record<string, string[]>;
+  raw?: { bytes: ArrayBuffer; file?: string }[];
+};
+
 export default class ExtendedRequest {
   public tabId: number;
   public url: string;
   public shorthost: string;
-  public requestHeaders: Request["requestHeaders"];
+  public requestHeaders: Request["requestHeaders"] = [];
   public originalURL: string;
   public origin: string;
   public initialized = false;
   public stolenData: StolenDataEntry[];
   public originalPathname: string;
+  public requestBody: RequestBody;
+
+  static by_id = {} as Record<string, ExtendedRequest>;
 
   constructor(public data: Request) {
     this.tabId = data.tabId;
     this.url = data.url;
-    this.requestHeaders = data.requestHeaders;
     this.shorthost = getshorthost(data.url);
+    this.requestBody =
+      ((data as any).requestBody as undefined | RequestBody) || {};
+    if (this.url.includes("criteo")) {
+      console.log(this);
+    }
+    ExtendedRequest.by_id[data.requestId] = this;
+  }
+
+  addHeaders(headers: Request["requestHeaders"]) {
+    this.requestHeaders = headers;
+    return this;
   }
 
   async init() {
@@ -93,7 +113,7 @@ export default class ExtendedRequest {
       url = (this.data as any).frameAncestors[0].url || "";
     } else {
       const headers = Object.fromEntries(
-        this.data.requestHeaders.map(({ name, value }) => [name, value])
+        this.requestHeaders.map(({ name, value }) => [name, value])
       );
       if (headers.Referer) {
         url = headers.Referer;
@@ -124,7 +144,7 @@ export default class ExtendedRequest {
 
   getReferer() {
     return (
-      this.data.requestHeaders.filter((h) => h.name === "Referer")[0]?.value ||
+      this.requestHeaders.filter((h) => h.name === "Referer")[0]?.value ||
       "missing-referrer"
     );
   }
@@ -155,6 +175,7 @@ export default class ExtendedRequest {
       ...this.getCookieData(),
       ...this.getQueryParams(),
       ...this.getHeadersData(),
+      ...this.getRequestBodyData(),
     ];
   }
 
@@ -171,12 +192,48 @@ export default class ExtendedRequest {
     ).map(([key, value]) => new StolenDataEntry(this, "cookie", key, value));
   }
 
+  getRequestBodyData(): StolenDataEntry[] {
+    const ret = flattenObjectEntries(
+      Object.entries({
+        ...this.requestBody.formData,
+        ...Object.fromEntries(
+          Object.entries(
+            this.requestBody.raw || {}
+          ).map(([key, value], index) => [`${key}.${index}`, value])
+        ),
+      })
+        .map(([key, value]) => {
+          // to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/
+          if (
+            (Array.isArray(value) && value.length === 1 && !value[0]) ||
+            !value
+          ) {
+            return ["requestBody", key];
+          } else if (!Array.isArray(value)) {
+            return [
+              "raw",
+              String.fromCharCode.apply(null, new Uint8Array(value.bytes)),
+            ];
+          } else {
+            return [key, value || ""];
+          }
+        })
+        .map(([key, value]) => {
+          const parsed = StolenDataEntry.parseValue(value);
+          return [key, parsed];
+        }) as [string, unknown][]
+    ).map(
+      ([key, value]) => new StolenDataEntry(this, "request_body", key, value)
+    );
+    return ret;
+  }
+
   hasReferer() {
-    return this.data.requestHeaders.some((h) => h.name === "Referer");
+    return this.requestHeaders.some((h) => h.name === "Referer");
   }
 
   hasCookie() {
-    return this.data.requestHeaders.some((h) => h.name === "Cookie");
+    return this.requestHeaders.some((h) => h.name === "Cookie");
   }
 
   getCookie(): string {
@@ -195,7 +252,10 @@ export default class ExtendedRequest {
         .map((e) => e.split("="))
         .map(([key, value]) => [key, value || ""])
         .map(([key, value]) => {
-          return [key, StolenDataEntry.parseValue(decodeURIComponent(value))];
+          return [
+            key,
+            StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
+          ];
         })
     ).map(([key, value]) => new StolenDataEntry(this, "pathname", key, value));
   }
@@ -206,7 +266,10 @@ export default class ExtendedRequest {
       Array.from((url.searchParams as any).entries())
         .map(([key, value]) => [key, value || ""])
         .map(([key, value]) => {
-          return [key, StolenDataEntry.parseValue(decodeURIComponent(value))];
+          return [
+            key,
+            StolenDataEntry.parseValue(safeDecodeURIComponent(value)),
+          ];
         })
     ).map(([key, value]) => {
       return new StolenDataEntry(this, "queryparams", key, value);
@@ -215,7 +278,7 @@ export default class ExtendedRequest {
 
   getHeadersData(): StolenDataEntry[] {
     return flattenObjectEntries(
-      this.data.requestHeaders
+      this.requestHeaders
         .filter((header) => {
           for (const regex of whitelisted_cookies) {
             if (regex.test(header.name)) {
@@ -261,7 +324,7 @@ export default class ExtendedRequest {
         url: this.data.url,
         headersSize: 100,
         httpVersion: "HTTP/2",
-        headers: this.data.requestHeaders as NameValue[],
+        headers: this.requestHeaders as NameValue[],
         cookies: this.getCookieData().map((cookie) => ({
           name: cookie.name,
           value: cookie.value,

memory.ts

@@ -26,9 +26,20 @@ export default class Memory extends EventEmitter {
 
   constructor() {
     super();
+
+    browser.webRequest.onBeforeRequest.addListener(
+      async (request) => {
+        new ExtendedRequest(request);
+      },
+      { urls: ["<all_urls>"] },
+      ["requestBody"]
+    );
     browser.webRequest.onBeforeSendHeaders.addListener(
       async (request) => {
-        this.register(new ExtendedRequest(request));
+        const extendedRequest = ExtendedRequest.by_id[
+          request.requestId
+        ].addHeaders(request.requestHeaders || []);
+        this.register(extendedRequest);
       },
       { urls: ["<all_urls>"] },
       ["requestHeaders"]

report-window/domain-summary.tsx

@@ -1,10 +1,6 @@
 import React from "react";
 import { RequestCluster } from "../request-cluster";
-import {
-  Classifications,
-  Sources,
-  StolenDataEntry,
-} from "../stolen-data-entry";
+import { Classifications, Sources } from "../stolen-data-entry";
 
 const emailClassifications: Record<keyof typeof Classifications, string> = {
   id: "sztucznie nadane mi ID",
@@ -17,6 +13,7 @@ const emailSources: Record<Sources, string> = {
   cookie: "z pliku Cookie",
   pathname: "jako części adresu URL",
   queryparams: "jako część adresu URL (query-params)",
+  request_body: "w body zapytania POST",
 };
 
 export default function DomainSummary({

stolen-data-entry.ts

@@ -10,9 +10,15 @@ import {
   isURL,
   maskString,
   parseToObject,
+  safeDecodeURIComponent,
 } from "./util";
 
-export type Sources = "cookie" | "pathname" | "queryparams" | "header";
+export type Sources =
+  | "cookie"
+  | "pathname"
+  | "queryparams"
+  | "header"
+  | "request_body";
 
 export const Classifications = <const>{
   id: "Sztucznie nadane ID",
@@ -243,14 +249,14 @@ export class StolenDataEntry extends EventEmitter {
   exposesPath() {
     return (
       this.request.originalPathname !== "/" &&
-      [this.value, decodeURIComponent(this.value)].some((haystack) =>
+      [this.value, safeDecodeURIComponent(this.value)].some((haystack) =>
         haystack.includes(this.request.originalPathname)
       )
     );
   }
 
   exposesHost() {
-    return [this.value, decodeURIComponent(this.value)].some((haystack) =>
+    return [this.value, safeDecodeURIComponent(this.value)].some((haystack) =>
       haystack.includes(getshorthost(this.request.origin))
     );
   }

util.ts

@@ -104,7 +104,8 @@ export function isJSONObject(
   str: unknown
 ): str is Record<string, unknown> | string | number {
   try {
-    return JSON.stringify(parseToObject(str))[0] == "{";
+    const firstChar = JSON.stringify(parseToObject(str))[0];
+    return ["{", "["].includes(firstChar);
   } catch (e) {
     return false;
   }
@@ -211,7 +212,7 @@ export function flattenObject(
         ret.push([`${key}.${subkey}`, subvalue]);
       }
     } else {
-      ret.push([key, value.toString()]);
+      ret.push([key, value ? value.toString() : "<empty>"]);
     }
   }
   return ret;
@@ -239,3 +240,11 @@ export function maskString(
     str.slice(str.length / 2 + amount_of_chars_to_cut / 2)
   );
 }
+
+export function safeDecodeURIComponent(s: string) {
+  try {
+    return decodeURIComponent(s);
+  } catch (e) {
+    return e;
+  }
+}