From 09c0e4df0be112c1bad90c6b83fc313f6578dbcb Mon Sep 17 00:00:00 2001 From: Kuba Orlik Date: Sat, 9 Jul 2022 21:14:03 +0200 Subject: [PATCH] Polish messages. Keep the newest message in an easy to see field in JSON --- @types/src/request.d.ts | 2 ++ Docker/prepare-firefox.sh | 6 +++--- Docker/run-analysis.sh | 7 +++---- Docker/utils.sh | 4 ++-- src/request.ts | 18 ++++++++++++++++++ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/@types/src/request.d.ts b/@types/src/request.d.ts index 47dfb65..750b259 100644 --- a/@types/src/request.d.ts +++ b/@types/src/request.d.ts @@ -19,6 +19,7 @@ export default class ScreenshotRequest { processing_took: number | null; waiting_took: number | null; process: ChildProcessWithoutNullStreams; + current_action: string; constructor(url: string, domains: string[]); getJobsAhead(): number; getJSON(): Promise<{ @@ -36,6 +37,7 @@ export default class ScreenshotRequest { waiting_took: number | null; elapsed_time_s: number; zip_url: string | null; + current_action: string; }>; getGoodImages(): Image[]; setFinished(): void; diff --git a/Docker/prepare-firefox.sh b/Docker/prepare-firefox.sh index 7aa7b35..7763a30 100755 --- a/Docker/prepare-firefox.sh +++ b/Docker/prepare-firefox.sh @@ -4,13 +4,13 @@ source ./ephemeral-x.sh source ./annotate_header.sh source ./utils.sh -echo "{\"current_action\": \"Setting up X environment...\"}" +echo "{\"current_action\": \"Uruchamianie serwera X\"}" -echo "{\"current_action\": \"Starting firefox...\"}" +echo "{\"current_action\": \"Wczytywanie Firefoxa....\"}" start_firefox grab start_firefox prepare_firefox grab prepare_firefox -echo "{\"current_action\": \"Firefox started, waiting for URL. Run run-analysis.sh in this container to continue\", \"code\": \"ready\"}" +echo "{\"current_action\": \"Oczekiwanie na URL do analizy...\", \"code\": \"ready\"}" ./eternal-sleep.sh & wait diff --git a/Docker/run-analysis.sh b/Docker/run-analysis.sh index 46a608a..885284d 100755 --- a/Docker/run-analysis.sh +++ b/Docker/run-analysis.sh @@ -14,7 +14,7 @@ DOMAINS=`node array-to-lines.js "$(echo $INPUT | jq .third_party_domains)"` source ./utils.sh -PREVIEW="FALSE" # set to "TRUE" in order to enable automatic screenshots kept in preview.png +PREVIEW="TRUE" # set to "TRUE" in order to enable automatic screenshots kept in preview.png if [ "$PREVIEW" = "TRUE" ]; then @@ -27,7 +27,6 @@ fi ORIGIN_DOMAIN=$(sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/' <<< "$URL") -echo "DOMAINS ARE $DOMAINS" while IFS= read -r DOMAIN; do load_website "$DOMAIN" @@ -56,7 +55,7 @@ while IFS= read -r DOMAIN; do if [ "$DOMAIN" = "" ]; then continue fi - echo "{\"current_action\": \"scanning for requests from $DOMAIN...\"}" + echo "{\"current_action\": \"Skanowanie skryptów z domeny $DOMAIN...\"}" network_inspector_search "domain:$DOMAIN " # can filter with more granularity: https://developer.mozilla.org/en-US/docs/Tools/Network_Monitor/request_list#filtering_by_properties # grab ni_search @@ -86,7 +85,7 @@ then kill $refresher_pid; fi -echo "{\"current_action\": \"awaiting al background processes...\"}" +echo "{\"current_action\": \"Kończenie...\"}" for PID in "${pids[@]}" do diff --git a/Docker/utils.sh b/Docker/utils.sh index 700f793..d413a1d 100644 --- a/Docker/utils.sh +++ b/Docker/utils.sh @@ -166,14 +166,14 @@ load_website(){ xdotool key Return grab enter sleep 1 - echo "{\"current_action\": \"waiting for $WEBSITE_URL to load...\"}" + echo "{\"current_action\": \"Wczytywanie strony $WEBSITE_URL...\"}" wait_for_website_to_load sleep 0.3 wait_for_website_to_load sleep 0.3 wait_for_website_to_load sleep 7 # sometimes the consent modal needs a little time to show up - echo "{\"current_action\": \"website loaded\"}" + echo "{\"current_action\": \"Strona wczytana\"}" } open_network_inspector(){ diff --git a/src/request.ts b/src/request.ts index 08e7179..eeca8a1 100644 --- a/src/request.ts +++ b/src/request.ts @@ -24,6 +24,7 @@ export default class ScreenshotRequest { public processing_took: number | null = null; public waiting_took: number | null = null; public process: ChildProcessWithoutNullStreams; + public current_action = "Inicjalizowaine..."; constructor(public url: string, public domains: string[]) { q.push(async () => { @@ -62,9 +63,12 @@ export default class ScreenshotRequest { waiting_took: number | null; elapsed_time_s: number; zip_url: string | null; + current_action: string; }> { return { url: this.url, + current_action: + this.status == "finished" ? "Zakończono!" : this.current_action, domains: this.domains, jobs_ahead: this.getJobsAhead(), id: this.id, @@ -89,6 +93,14 @@ export default class ScreenshotRequest { } getGoodImages(): Image[] { + /* find the best set of screenshots, that is: a set of screenshots that + contain all the header values that appear in the headers, but with as little + screenshots as possible. + + The current approach is to sort the screenshots by which ones contain the + most highlighted information, and then go through them one by one and check + off the data that they provide and stop once all the values are checked off + */ const result: Image[] = []; const domains = Array.from( new Set(this.images.map((image) => image.domain)) @@ -200,6 +212,12 @@ export default class ScreenshotRequest { ) { this.images.push(parsed.new_file as Image); } + if ( + is(parsed, predicates.object) && + is(parsed.current_action, predicates.string) + ) { + this.current_action = parsed.current_action; + } } catch (e) { //noop }