419 lines
14 KiB
TypeScript
419 lines
14 KiB
TypeScript
'use strict';
|
|
import { DataLocation, StolenDataEntry } from './stolen-data-entry';
|
|
import {
|
|
flattenObjectEntries,
|
|
getshorthost,
|
|
parseCookie,
|
|
Request,
|
|
safeDecodeURIComponent,
|
|
} from './util';
|
|
|
|
type NameValue = { name: string; value: string };
|
|
|
|
export type HAREntry = {
|
|
pageref: string;
|
|
startedDateTime: string;
|
|
request: {
|
|
bodySize: number;
|
|
cookies: NameValue[];
|
|
headers: NameValue[];
|
|
headersSize: number;
|
|
httpVersion: string;
|
|
method: string;
|
|
postData?: {
|
|
mimeType: string;
|
|
params: (NameValue & {
|
|
fileName: string;
|
|
contentType: string;
|
|
comment: '';
|
|
})[];
|
|
text: string;
|
|
};
|
|
queryString: NameValue[];
|
|
url: string;
|
|
};
|
|
response: {
|
|
status: number;
|
|
statusText: string;
|
|
httpVersion: string;
|
|
headers: NameValue[];
|
|
cookies: NameValue[];
|
|
content: {
|
|
mimeType: string;
|
|
size: number;
|
|
encoding: 'base64';
|
|
text: string;
|
|
};
|
|
redirectURL: '';
|
|
headersSize: number;
|
|
bodySize: number;
|
|
}; // not relevant
|
|
cache: {};
|
|
timings: {};
|
|
time: number;
|
|
_securityState: string;
|
|
serverIPAddress: string;
|
|
connection: string;
|
|
};
|
|
|
|
const whitelisted_cookies = [
|
|
/^Accept.*$/,
|
|
/^Host$/,
|
|
/^Connection$/,
|
|
/^Sec-Fetch-.*$/,
|
|
/^Content-Type$/,
|
|
/^Cookie$/, // we're extracting it in getCookie separately anyway
|
|
/^User-Agent$/,
|
|
];
|
|
|
|
type RequestBody = {
|
|
error?: string;
|
|
formData?: Record<string, string[]>;
|
|
raw?: { bytes: ArrayBuffer; file?: string }[];
|
|
};
|
|
|
|
export default class ExtendedRequest {
|
|
public tabId: number;
|
|
public url: string;
|
|
public shorthost: string;
|
|
public requestHeaders: { name: string; value?: string; binaryValue?: number[] }[] = [];
|
|
public origin: string;
|
|
public initialized = false;
|
|
public stolenData: StolenDataEntry[] = [];
|
|
public originalURL: string | null = null; // sometimes we can only establish that the given request applied to a certain origin, not a full URL from the address bar - in case of service workers, for example. Hence the null
|
|
public originalPathname: string | null = null; // same as above
|
|
public originalHost: string;
|
|
public requestBody: RequestBody;
|
|
|
|
static by_id = {} as Record<string, ExtendedRequest>;
|
|
public data: Request;
|
|
|
|
constructor(data: Request) {
|
|
this.tabId = data.tabId;
|
|
this.url = data.url;
|
|
this.shorthost = getshorthost(data.url);
|
|
this.requestBody = ((data as any).requestBody as undefined | RequestBody) || {};
|
|
ExtendedRequest.by_id[data.requestId] = this;
|
|
|
|
this.data = Object.assign({}, data);
|
|
(this.data as any).frameAncestors = [
|
|
...((data as any)?.frameAncestors?.map((e: any) => ({ url: e.url })) || []),
|
|
]; // making a copy?
|
|
|
|
// console.log('→→→',(this.data as any).frameAncestors, (data as any).frameAncestors);
|
|
|
|
let url: string;
|
|
let is_full_url = true;
|
|
let url_comes_from: string;
|
|
if (this.data.type === 'main_frame') {
|
|
url = this.data.url;
|
|
url_comes_from = 'main_frame';
|
|
} else if (this.data.frameId === 0 && this.data.documentUrl) {
|
|
url = this.data.documentUrl;
|
|
url_comes_from = 'documentUrl';
|
|
if (this.data.tabId == -1) {
|
|
//a service worker?
|
|
url_comes_from = 'documentUrl (webworker)';
|
|
is_full_url = false;
|
|
}
|
|
} else if (
|
|
(this.data as any)?.frameAncestors &&
|
|
(this.data as any).frameAncestors[0] !== undefined
|
|
) {
|
|
url = (this.data as any).frameAncestors.at(-1).url || '';
|
|
url_comes_from = 'frameAncestors';
|
|
} else {
|
|
url = this.data.documentUrl || this.data.originUrl;
|
|
url_comes_from = 'last resort';
|
|
}
|
|
|
|
this.originalURL = is_full_url ? url : null;
|
|
this.origin = new URL(url).origin;
|
|
|
|
this.originalHost = new URL(url).host;
|
|
this.originalPathname = is_full_url ? new URL(url).pathname : null;
|
|
}
|
|
|
|
addHeaders(headers: Request['requestHeaders']) {
|
|
this.requestHeaders = headers || [];
|
|
return this;
|
|
}
|
|
|
|
init() {
|
|
this.initialized = true;
|
|
this.stolenData = this.getAllStolenData();
|
|
}
|
|
|
|
isThirdParty() {
|
|
const request_url = new URL(this.data.url);
|
|
if (request_url.host.includes(this.originalHost)) {
|
|
return false;
|
|
}
|
|
if (getshorthost(request_url.host) == getshorthost(this.originalHost)) {
|
|
return false;
|
|
}
|
|
return (
|
|
request_url.origin != this.origin ||
|
|
(this.data as any).urlClassification.thirdParty.length > 0
|
|
);
|
|
}
|
|
|
|
getReferer() {
|
|
return (
|
|
this.requestHeaders.filter((h) => h.name === 'Referer')[0]?.value || 'missing-referrer'
|
|
);
|
|
}
|
|
|
|
exposesOriginWhere(): null | DataLocation {
|
|
const host = this.originalHost;
|
|
const path = this.originalPathname || '/';
|
|
const shorthost = getshorthost(host);
|
|
if (this.getReferer().includes(shorthost)) {
|
|
return { path: this.url, source: 'header', key: 'Referer' };
|
|
}
|
|
for (const entry of this.stolenData) {
|
|
if (
|
|
entry.value.includes(host) ||
|
|
entry.value.includes(path) ||
|
|
entry.value.includes(shorthost)
|
|
) {
|
|
return entry.toDataLocation();
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
exposesOrigin() {
|
|
return this.exposesOriginWhere() !== null;
|
|
}
|
|
|
|
private getAllStolenData(): StolenDataEntry[] {
|
|
return [
|
|
...this.getPathParams(),
|
|
...this.getCookieData(),
|
|
...this.getQueryParams(),
|
|
...this.getHeadersData(),
|
|
...this.getRequestBodyData(),
|
|
];
|
|
}
|
|
|
|
getCookieData(): StolenDataEntry[] {
|
|
if (!this.hasCookie() || this.getCookie() === undefined) {
|
|
return [];
|
|
}
|
|
return flattenObjectEntries(
|
|
Object.entries(parseCookie(this.getCookie())).map(([key, value]) => [key, value || '']),
|
|
StolenDataEntry.parseValue
|
|
).map(([key, value]) => new StolenDataEntry(this, 'cookie', key, value));
|
|
}
|
|
|
|
getRequestBodyData(): StolenDataEntry[] {
|
|
const ret = flattenObjectEntries(
|
|
Object.entries({
|
|
...this.requestBody.formData,
|
|
...Object.fromEntries(
|
|
Object.entries(this.requestBody.raw || {}).map(([key, value], index) => [
|
|
`${key}.${index}`,
|
|
value,
|
|
])
|
|
),
|
|
}).map(([key, value]) => {
|
|
// to handle how ocdn.eu encrypts POST body on https://businessinsider.com.pl/
|
|
if ((Array.isArray(value) && value.length === 1 && !value[0]) || !value) {
|
|
return ['requestBody', key];
|
|
} else if (!Array.isArray(value)) {
|
|
return [
|
|
'raw',
|
|
String.fromCharCode.apply(null, Array.from(new Uint8Array(value.bytes))),
|
|
];
|
|
} else {
|
|
return [key, value || ''];
|
|
}
|
|
}),
|
|
StolenDataEntry.parseValue
|
|
).map(([key, value]) => new StolenDataEntry(this, 'request_body', key, value));
|
|
return ret;
|
|
}
|
|
|
|
hasReferer() {
|
|
return this.requestHeaders.some((h) => h.name === 'Referer');
|
|
}
|
|
|
|
hasCookie() {
|
|
return this.requestHeaders.some((h) => h.name === 'Cookie');
|
|
}
|
|
|
|
getCookie(): string {
|
|
return this.requestHeaders.find((h) => h.name == 'Cookie')?.value || '';
|
|
}
|
|
|
|
getPathParams(): StolenDataEntry[] {
|
|
const url = new URL(this.data.url);
|
|
const path = url.pathname;
|
|
if (!path.includes(';')) {
|
|
return [];
|
|
}
|
|
return flattenObjectEntries(
|
|
path
|
|
.split(';')
|
|
.map((e) => e.split('='))
|
|
.map(([key, value]) => [key, value || ''])
|
|
.map(([key, value]) => {
|
|
return [key, StolenDataEntry.parseValue(safeDecodeURIComponent(value))];
|
|
})
|
|
).map(([key, value]) => new StolenDataEntry(this, 'pathname', key, value));
|
|
}
|
|
|
|
getQueryParams(): StolenDataEntry[] {
|
|
const url = new URL(this.data.url);
|
|
return flattenObjectEntries(
|
|
(Array.from((url.searchParams as any).entries()) as [string, string][])
|
|
.map(([key, value]: [string, string]) => [key, value || ''])
|
|
.map(([key, value]) => {
|
|
return [key, StolenDataEntry.parseValue(safeDecodeURIComponent(value))];
|
|
})
|
|
).map(([key, value]) => {
|
|
return new StolenDataEntry(this, 'queryparams', key, value);
|
|
});
|
|
}
|
|
|
|
getHeadersData(): StolenDataEntry[] {
|
|
return flattenObjectEntries(
|
|
this.requestHeaders
|
|
.filter((header) => {
|
|
for (const regex of whitelisted_cookies) {
|
|
if (regex.test(header.name)) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
})
|
|
.map((header) => {
|
|
return [
|
|
header.name,
|
|
StolenDataEntry.parseValue(safeDecodeURIComponent(header.value || '')),
|
|
];
|
|
})
|
|
).map(([key, value]) => new StolenDataEntry(this, 'header', key, value));
|
|
}
|
|
|
|
hasMark() {
|
|
return this.stolenData.some((data) => data.isMarked);
|
|
}
|
|
|
|
getMarkedEntries() {
|
|
return this.stolenData.filter((data) => data.isMarked);
|
|
}
|
|
|
|
unmarkAllEntries() {
|
|
this.stolenData.forEach((entry) => entry.unmark());
|
|
}
|
|
|
|
getHost() {
|
|
return new URL(this.url).host;
|
|
}
|
|
|
|
matchesHAREntry(har: HAREntry): boolean {
|
|
const rq = this.data;
|
|
const hrq = har.request;
|
|
return rq.url == hrq.url;
|
|
}
|
|
|
|
toHAR(): HAREntry {
|
|
return {
|
|
pageref: 'page_1',
|
|
startedDateTime: `${new Date().toJSON().replace('Z', '+01:00')}`,
|
|
request: {
|
|
bodySize:
|
|
JSON.stringify(this.requestBody.formData || {}).length +
|
|
(this.requestBody.raw || [])
|
|
.map((e) => e.bytes.byteLength)
|
|
.reduce((a, b) => a + b, 0),
|
|
method: this.data.method,
|
|
url: this.data.url,
|
|
headersSize: JSON.stringify(this.requestHeaders).length,
|
|
httpVersion: 'HTTP/2',
|
|
headers: this.requestHeaders as NameValue[],
|
|
cookies: this.getCookieData().map((cookie) => ({
|
|
name: cookie.name,
|
|
value: cookie.value,
|
|
})),
|
|
queryString: this.getQueryParams().map((param) => ({
|
|
name: param.name,
|
|
value: param.value,
|
|
})),
|
|
postData: {
|
|
mimeType: 'application/x-www-form-urlencoded',
|
|
params: this.stolenData
|
|
.filter((e) => e.source == 'request_body')
|
|
.map((e) => ({
|
|
name: e.name,
|
|
value: e.value,
|
|
fileName: '--' + Math.ceil(Math.random() * 1000000000),
|
|
contentType: 'text/plain',
|
|
comment: '',
|
|
})),
|
|
text: this.stolenData
|
|
.filter((e) => e.source == 'request_body')
|
|
.map((e) => `${e.name}:\t${StolenDataEntry.parseValue(e.value)}`)
|
|
.join('\n\n'),
|
|
},
|
|
},
|
|
response: {
|
|
status: 200,
|
|
statusText: 'OK',
|
|
httpVersion: 'HTTP/2',
|
|
headers: [],
|
|
cookies: [],
|
|
content: {
|
|
mimeType: 'text/plain',
|
|
size: this.getBalancedPriority(),
|
|
encoding: 'base64',
|
|
text: 'ZG9lc24ndCBtYXR0ZXIK',
|
|
},
|
|
redirectURL: '',
|
|
headersSize: 15,
|
|
bodySize: 15,
|
|
},
|
|
cache: {},
|
|
timings: {
|
|
blocked: -1,
|
|
dns: 0,
|
|
connect: 0,
|
|
ssl: 0,
|
|
send: 0,
|
|
wait: 79,
|
|
receive: 0,
|
|
},
|
|
time: 79,
|
|
_securityState: 'secure',
|
|
serverIPAddress: '31.13.92.36',
|
|
connection: '443',
|
|
};
|
|
}
|
|
|
|
getMaxPriority(): number {
|
|
return Math.max(...this.stolenData.map((entry) => entry.getPriority()));
|
|
}
|
|
|
|
getBalancedPriority(): number {
|
|
let result = 0;
|
|
if (this.stolenData.some((e) => e.exposesPath())) {
|
|
result += 50;
|
|
}
|
|
if (this.stolenData.some((e) => e.exposesHost())) {
|
|
result += 50;
|
|
}
|
|
if (this.hasCookie()) {
|
|
result += 50;
|
|
}
|
|
if (this.stolenData.some((e) => e.classification === 'location')) {
|
|
result += 300;
|
|
}
|
|
if (this.url.includes('facebook')) {
|
|
result += 50;
|
|
}
|
|
return result;
|
|
}
|
|
}
|