Add a webservice API with polling

Docker/annotate_header.sh

@@ -12,15 +12,19 @@ get_width(){
 
 
 annotate_header(){
-  echo annotate $1
+  echo annotate $1 $2 $3
   filename=$1
   shift;
+  domain=$1;
+  shift;
+  count=$1;
+  shift;
   d=$(date "+%Y-%m-%d__%H_%M_%S")
   #filename="2022-03-10__19_33_55__set-cookie.png"
   cropped_filename="${filename}__cropped.png"
   overlay_filename="${cropped_filename}__overlay.png"
   hardoverlay_filename="${cropped_filename}__hardoverlay.png"
-  annotated_filename="${cropped_filename}__annotated.png"
+  annotated_filename="$(dirname "$filename")/${domain}__${count}.final.png" # the name is crucial, because the web app part filters files based on the name
 
   # crop 
   left=2056
@@ -78,10 +82,10 @@ annotate_header(){
     shift
     shift
   done
-  convert "$filename" "$overlay_filename" -compose Darken -composite "$annotated_filename.step.png"
+  convert "$filename" "$overlay_filename" -compose Darken -composite "${annotated_filename}.step.png"
-  convert "$annotated_filename.step.png" "$hardoverlay_filename" -compose src-over -composite "$annotated_filename"
+  convert "${annotated_filename}.step.png" "$hardoverlay_filename" -compose src-over -composite "$annotated_filename"
   rm "$overlay_filename" "$annotated_filename.step.png" "$hardoverlay_filename" "$cropped_filename" "$filename"
-  echo "SCREENSHOT: <img width=\"720\" height=\"405\" src=\"$BASE_URL/$(echo "$annotated_filename" | sed 's|/opt/||')\"/>"
+  echo "SCREENSHOT: <img width=\"720\" height=\"405\" src=\"${BASE_URL}/$(echo "$annotated_filename" | sed 's|/opt/||')\"/>"
 }
 
 #annotate_header "set-cookie" "identyfikator internetowy z cookie" "Cookie" "identyfikator internetowy z cookie"

Docker/script3.sh

@@ -14,11 +14,11 @@ source ./ephemeral-x.sh
 source ./annotate_header.sh
 source ./utils.sh
 
-(while true; do
+# (while true; do
-   grab_screen_to_public $ID
+#    grab_screen_to_public $ID
-   sleep 1
+#    sleep 1
- done) &
+#  done) &
-refresher_pid=$!;
+# refresher_pid=$!;
 
 start_firefox
 grab start_firefox
@@ -53,7 +53,7 @@ while IFS= read -r DOMAIN; do
     do
       filename="/opt/static/$ID/${index}.png"
       scrot "$filename"
-      annotate_header "$filename" \
+      annotate_header "$filename" "$DOMAIN" "$count" \
         "set-cookie" "identyfikator internetowy z cookie" \
         "Cookie" "identyfikator internetowy z cookie" \
         "Referer" "Część mojej historii przeglądania" &
@@ -67,7 +67,7 @@ while IFS= read -r DOMAIN; do
     done
 done <<< "$DOMAINS"
 
-kill $refresher_pid;
+# kill $refresher_pid;
 
 echo "starting wait..."
 

README.md

@@ -23,3 +23,13 @@ BASE_URL=http://localhost:3000 docker run -i -v $PWD/static:/opt/static  headles
 ```
 BASE_URL=http://localhost:3000 node .
 ```
+
+## Testing the API
+
+with httpie:
+
+```
+http POST localhost:3000/api/requests url==pearson.com 'domains[]==youtube.com' 'domains[]==facebook.com'
+```
+
+It returns a 303 response that redirects to a URL you can poll to see the status of the screenshot request.

docker-args.js

@@ -0,0 +1,10 @@
+const DOCKER_ARGS = [
+  "run",
+  "-i",
+  "-v",
+  `${process.cwd()}/static:/opt/static`,
+  "headless-fox",
+  "./script3.sh",
+];
+
+module.exports = DOCKER_ARGS;

index.js

@@ -1,4 +1,3 @@
-const { v4: uuid } = require("uuid");
 var serve = require("koa-static");
 const Koa = require("koa");
 const Router = require("@koa/router");
@@ -6,6 +5,9 @@ const mount = require("koa-mount");
 const qs = require("qs");
 const { Readable } = require("stream");
 const { spawn } = require("child_process");
+const { requests } = require("./memory");
+const ScreenshotRequest = require("./request");
+const DOCKER_ARGS = require("./docker-args");
 
 const router = new Router();
 
@@ -65,13 +67,7 @@ router.get("/", async (ctx) => {
   const starter = spawn(
     "docker",
     [
-      "run",
+      ...DOCKER_ARGS,
-      "-i",
-      "-d",
-      "-v",
-      `${process.cwd()}/static:/opt/static`,
-      "headless-fox",
-      "./script3.sh",
       `{"url": "${params.url}", "third_party_domains": ["hotjar.com", "cookielaw.org", "facebook.com", "gemius.pl"]}`,
       id,
     ],
@@ -87,5 +83,36 @@ router.get("/", async (ctx) => {
   });
 });
 
+router.post("/api/requests", async (ctx) => {
+  const params = qs.parse(ctx.querystring);
+  if (!params.url) {
+    ctx.body = "Specify url";
+    ctx.status = 422;
+    return;
+  }
+  if (!params.domains) {
+    ctx.body = "Specify domains";
+    ctx.status = 422;
+    return;
+  }
+  if (!Array.isArray(params.domains)) {
+    ctx.body = "'domains' should be an array of strings";
+    ctx.status = 422;
+    return;
+  }
+  const request = new ScreenshotRequest(params.url, params.domains);
+  ctx.status = 303;
+  ctx.redirect(`/api/requests/${request.id}`);
+});
+
+router.get("/api/requests/:id", async (ctx) => {
+  const request = requests[ctx.params.id];
+  if (!request) {
+    ctx.status = 404;
+    return;
+  }
+  ctx.body = await request.getJSON();
+});
+
 app.use(router.routes()).use(router.allowedMethods());
 app.listen(3000);

memory.js

@@ -0,0 +1,3 @@
+const queue = require("queue");
+const q = queue({ concurrency: 1, autostart: true, results: [] });
+b;

package-lock.json

@@ -15,6 +15,7 @@
         "koa-mount": "^4.0.0",
         "koa-static": "^5.0.0",
         "qs": "^6.10.3",
+        "queue": "^6.0.2",
         "uuid": "^8.3.2"
       }
     },
@@ -571,6 +572,14 @@
       "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
       "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
     },
+    "node_modules/queue": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz",
+      "integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==",
+      "dependencies": {
+        "inherits": "~2.0.3"
+      }
+    },
     "node_modules/readable-stream": {
       "version": "1.1.14",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
@@ -1167,6 +1176,14 @@
       "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
       "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
     },
+    "queue": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz",
+      "integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==",
+      "requires": {
+        "inherits": "~2.0.3"
+      }
+    },
     "readable-stream": {
       "version": "1.1.14",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",

package.json

@@ -19,6 +19,7 @@
     "koa-mount": "^4.0.0",
     "koa-static": "^5.0.0",
     "qs": "^6.10.3",
+    "queue": "^6.0.2",
     "uuid": "^8.3.2"
   }
 }

request.js

@@ -0,0 +1,75 @@
+const { q, requests } = require("./memory");
+const DOCKER_ARGS = require("./docker-args");
+const { v4: uuid } = require("uuid");
+const { promises: fs } = require("fs");
+const { spawn } = require("child_process");
+const { resolve } = require("path");
+
+module.exports = class ScreenshotRequest {
+  constructor(url, domains) {
+    this.url = url;
+    this.domains = domains;
+    this.id = uuid();
+    this.status = "waiting";
+    this.output = "";
+    this.images = [];
+    q.push(async () => {
+      return this.exec();
+    });
+    requests[this.id] = this;
+  }
+
+  async getImages() {
+    try {
+      const files = await fs.readdir(resolve(__dirname, "./static/" + this.id));
+      return files.filter((file) => file.match(/.final.png$/));
+    } catch (e) {
+      return [];
+    }
+  }
+
+  async getJSON() {
+    return {
+      url: this.url,
+      domains: this.domains,
+      id: this.id,
+      status: this.status,
+      /* output: this.output, */
+      files: await this.getImages(),
+    };
+  }
+
+  async exec() {
+    return new Promise((resolve, reject) => {
+      this.status = "running";
+      this.process = spawn(
+        "docker",
+        [
+          ...DOCKER_ARGS,
+          JSON.stringify({
+            url: this.url,
+            third_party_domains: this.domains,
+          }),
+          this.id,
+        ],
+        { cwd: process.cwd() }
+      );
+      this.process.on("close", (exitCode) => {
+        this.status = "finished";
+        if (exitCode === 0) {
+          resolve();
+        } else {
+          reject();
+        }
+      });
+      this.process.stdout.on("data", (d) => {
+        this.output += d.toString();
+        /* console.log("DATA!", d.toString()); */
+      });
+      this.process.stderr.on("data", (d) => {
+        this.output += d.toString();
+        /* console.log("STDERR!", d.toString()); */
+      });
+    });
+  }
+};