Remove duplicates and group rows by same value

This commit is contained in:
Kuba Orlik 2021-11-06 22:32:40 +01:00
parent 8fc1b33977
commit b53f0b3bb0
2 changed files with 69 additions and 15 deletions

View File

@ -41,6 +41,49 @@ export class StolenDataEntry {
} }
return priority; return priority;
} }
mergeWith(entry: StolenDataEntry): MergedStolenDataEntry {
return new MergedStolenDataEntry([this, entry]);
}
hasValue(value: string) {
return this.value === value;
}
}
export class MergedStolenDataEntry {
constructor(public entries: StolenDataEntry[]) {}
hasValue(value: string) {
return this.entries.some((entry) => entry.value === value);
}
mergeWith(entry: StolenDataEntry) {
this.entries.push(entry);
return this;
}
getPriority() {
return Math.max(...this.entries.map((entry) => entry.getPriority()));
}
getUniqueKey() {
return `${this.getNames().join(":")};${this.entries
.map((e) => e.id)
.join(":")};`;
}
getNames(): string[] {
return Array.from(new Set(this.entries.map((e) => e.name)));
}
getSources(): string[] {
return Array.from(new Set(this.entries.map((e) => e.source)));
}
getValues() {
return Array.from(new Set(this.entries.map((e) => e.value)));
}
} }
export class RequestCluster extends EventEmitter { export class RequestCluster extends EventEmitter {
@ -65,7 +108,7 @@ export class RequestCluster extends EventEmitter {
getStolenData(filter: { getStolenData(filter: {
minValueLength: number; minValueLength: number;
cookiesOnly: boolean; cookiesOnly: boolean;
}): StolenDataEntry[] { }): MergedStolenDataEntry[] {
return this.requests return this.requests
.map((request) => request.getAllStolenData()) .map((request) => request.getAllStolenData())
.reduce((a, b) => a.concat(b), []) .reduce((a, b) => a.concat(b), [])
@ -73,11 +116,9 @@ export class RequestCluster extends EventEmitter {
return entry.value.length >= filter.minValueLength; return entry.value.length >= filter.minValueLength;
}) })
.filter((entry) => !filter.cookiesOnly || entry.source === "cookie") .filter((entry) => !filter.cookiesOnly || entry.source === "cookie")
.sort((entry1, entry2) => .sort((entryA, entryB) => (entryA.name > entryB.name ? -1 : 1))
entry1.getPriority() > entry2.getPriority() ? -1 : 1
)
.filter((element, index, array) => { .filter((element, index, array) => {
// remove duplicate neighbours // remove duplicates by name/value
if (index == 0) { if (index == 0) {
return true; return true;
} }
@ -87,7 +128,24 @@ export class RequestCluster extends EventEmitter {
) { ) {
return true; return true;
} }
}); })
.sort((entryA, entryB) => (entryA.value > entryB.value ? -1 : 1))
.reduce(
(acc: MergedStolenDataEntry[], entry: StolenDataEntry) => {
// group by value
const last_entry = acc.slice(-1)[0];
if (last_entry.hasValue(entry.value)) {
last_entry.mergeWith(entry);
} else {
acc.push(new MergedStolenDataEntry([entry]));
}
return acc;
},
[new MergedStolenDataEntry([])] as MergedStolenDataEntry[]
)
.sort((entry1, entry2) =>
entry1.getPriority() > entry2.getPriority() ? -1 : 1
);
} }
static sortCompare(a: RequestCluster, b: RequestCluster) { static sortCompare(a: RequestCluster, b: RequestCluster) {

View File

@ -78,19 +78,15 @@ const StolenDataRow = ({
.getStolenData({ minValueLength, cookiesOnly }) .getStolenData({ minValueLength, cookiesOnly })
.map((entry) => ( .map((entry) => (
<tr <tr
key={ key={origin + cluster.id + entry.getUniqueKey()}
origin + ";" + cluster.id + ";" + entry.id + ";" + entry.name data-key={origin + cluster.id + entry.getUniqueKey()}
}
data-key={
origin + ";" + cluster.id + ";" + entry.id + ";" + entry.name
}
> >
<th style={{ maxWidth: "200px", wordWrap: "break-word" }}> <th style={{ maxWidth: "200px", wordWrap: "break-word" }}>
{entry.name} {entry.getNames().join(",")}
</th> </th>
<td>{icons[entry.source]}</td> <td>{entry.getSources().map((source) => icons[source])}</td>
<td style={{ wordWrap: "anywhere" as any }}> <td style={{ wordWrap: "anywhere" as any }}>
{entry.value} {entry.isIAB ? "!!!!! IAB" : ""} {entry.getValues()[0]}
</td> </td>
</tr> </tr>
))} ))}