From e38daf52de501b74b622b8f7977e44659279f2d9 Mon Sep 17 00:00:00 2001 From: Kuba Orlik Date: Fri, 27 May 2022 15:49:56 +0200 Subject: [PATCH] Return images in chronological order --- Docker/annotate_header.sh | 8 ++++---- Docker/script3.sh | 29 +++++++++++++++++++---------- Docker/utils.sh | 8 ++++---- request.js | 21 +++++++-------------- 4 files changed, 34 insertions(+), 32 deletions(-) diff --git a/Docker/annotate_header.sh b/Docker/annotate_header.sh index 5097a24..e28dad2 100755 --- a/Docker/annotate_header.sh +++ b/Docker/annotate_header.sh @@ -40,7 +40,7 @@ annotate_header(){ all_positions=$(python ./get-text-position.py "$cropped_filename" ) END=$(date +%s.%N) DIFF=$(echo "$END - $START" | bc) - echo "{'ocr_took': '${DIFF}s'}" + echo "{\"ocr_took\": \"${DIFF}s\"}" while [ $# -gt 0 ] do @@ -87,16 +87,16 @@ annotate_header(){ shift END=$(date +%s.%N) DIFF=$(echo "$END - $START" | bc) - echo "{'annotation_took': '${DIFF}s'}" + echo "{\"annotation_took\": \"${DIFF}s\"}" done START=$(date +%s.%N) convert "$filename" "$overlay_filename" -compose Darken -composite "${annotated_filename}.step.png" convert "${annotated_filename}.step.png" "$hardoverlay_filename" -compose src-over -composite "$annotated_filename" rm "$overlay_filename" "$annotated_filename.step.png" "$hardoverlay_filename" "$cropped_filename" "$filename" - echo "{'new_file': \"${BASE_URL}/$(echo "$annotated_filename" | sed 's|/opt/||')\"}" + echo "{\"new_file\": \"${BASE_URL}/$(echo "$annotated_filename" | sed 's|/opt/||')\"}" END=$(date +%s.%N) DIFF=$(echo "$END - $START" | bc) - echo "{'composition_took': '${DIFF}s'}" + echo "{\"composition_took\": \"${DIFF}s\"}" } #annotate_header "set-cookie" "identyfikator internetowy z cookie" "Cookie" "identyfikator internetowy z cookie" diff --git a/Docker/script3.sh b/Docker/script3.sh index bd98b94..4890bd7 100755 --- a/Docker/script3.sh +++ b/Docker/script3.sh @@ -13,17 +13,23 @@ DOMAINS=`node array-to-lines.js "$(echo $INPUT | jq .third_party_domains)"` source ./utils.sh source ./annotate_header.sh -echo "{'current_action': 'Setting up X environment...'}" +echo "{\"current_action\": \"Setting up X environment...\"}" source ./ephemeral-x.sh -# (while true; do -# grab_screen_to_public $ID -# sleep 1 -# done) & -# refresher_pid=$!; + +PREVIEW="FALSE" # set to "TRUE" in order to enable automatic screenshots kept in preview.png + +if [ "$PREVIEW" = "TRUE" ]; +then + (while true; do + grab_screen_to_public $ID + sleep 1 + done) & + refresher_pid=$!; +fi -echo "{'current_action': 'Starting firefox...'}" +echo "{\"current_action\": \"Starting firefox...\"}" start_firefox grab start_firefox prepare_firefox @@ -45,7 +51,7 @@ while IFS= read -r DOMAIN; do if [ "$DOMAIN" = "" ]; then continue fi - echo "{'current_action': 'scanning for requests from $DOMAIN...'}" + echo "{\"current_action\": \"scanning for requests from $DOMAIN...\"}" network_inspector_search "domain:$DOMAIN" # can filter with more granularity: https://developer.mozilla.org/en-US/docs/Tools/Network_Monitor/request_list#filtering_by_properties grab ni_search @@ -70,9 +76,12 @@ while IFS= read -r DOMAIN; do done done <<< "$DOMAINS" -# kill $refresher_pid; +if [ "$PREVIEW" = "TRUE" ]; +then + kill $refresher_pid; +fi -echo "{'current_action': 'awaiting al background processes...'}" +echo "{\"current_action\": \"awaiting al background processes...\"}" for PID in "${pids[@]}" do diff --git a/Docker/utils.sh b/Docker/utils.sh index b430d15..548249f 100644 --- a/Docker/utils.sh +++ b/Docker/utils.sh @@ -14,7 +14,7 @@ _get_pixel_color(){ magick $output_path -format "%[hex:p{$x,$y}]" info: END=$(timestamp) DIFF=$(echo "$END - $START" | bc) - echo "{'getting_pixel_color_took': '${DIFF}ms'}" > /dev/stderr + echo "{\"getting_pixel_color_took\": \"${DIFF}ms\"}" > /dev/stderr } @@ -144,7 +144,7 @@ load_website(){ xdotool key Return grab enter sleep 1 - echo "{'current_action': 'waiting for $URL to load...'}" + echo "{\"current_action\": \"waiting for $URL to load...\"}" times=0 while [ $(get_pixel_color 143 122) = "#2e3436" ] # the center of the X icon that becomes a "refresh" icon once the website is finished loading do @@ -154,11 +154,11 @@ load_website(){ times=$((times + 1)) if [ $times -eq 30 ] then - echo "{'current_action': 'website load timeout, proceeding anyway...'}" + echo "{\"current_action\": \"website load timeout, proceeding anyway...\"}" break; fi done - echo "{'current_action': 'website loaded'}" + echo "{\"current_action\": \"website loaded\"}" } open_network_inspector(){ diff --git a/request.js b/request.js index db6c79d..4d9377a 100644 --- a/request.js +++ b/request.js @@ -1,9 +1,7 @@ const { q, requests } = require("./memory"); const DOCKER_ARGS = require("./docker-args"); const { v4: uuid } = require("uuid"); -const { promises: fs } = require("fs"); const { spawn } = require("child_process"); -const { resolve } = require("path"); module.exports = class ScreenshotRequest { constructor(url, domains) { @@ -24,17 +22,6 @@ module.exports = class ScreenshotRequest { requests[this.id] = this; } - async getImages() { - try { - const files = await fs.readdir(resolve(__dirname, "./static/" + this.id)); - return files - .filter((file) => file.match(/.final.png$/)) - .map((file) => `/static/${this.id}/${file}`); - } catch (e) { - return []; - } - } - async getJSON() { return { url: this.url, @@ -42,7 +29,7 @@ module.exports = class ScreenshotRequest { id: this.id, status: this.status, output: this.output, - files: await this.getImages(), + images: this.images, request_time: this.request_time, started_time: this.started_time, finished_time: this.finished_time, @@ -88,6 +75,12 @@ module.exports = class ScreenshotRequest { } }); this.process.stdout.on("data", (d) => { + try { + const parsed = JSON.parse(d.toString()); + if (parsed.new_file) { + this.images.push(parsed.new_file); + } + } catch (e) {} this.output += d.toString(); /* console.log("DATA!", d.toString()); */ });