Implement container warmup and pooling

Docker/Dockerfile

@@ -47,5 +47,6 @@ RUN apk add clang
 RUN apk add freetype-dev
 RUN python3 -m pip install --upgrade Pillow
 COPY . /opt
+CMD /opt/prepare-firefox.sh
 
 WORKDIR /opt

Docker/eternal-sleep.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+
+while true; do
+  sleep 1;
+done

Docker/prepare-firefox.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+source ./ephemeral-x.sh
+source ./annotate_header.sh
+source ./utils.sh
+
+echo "{\"current_action\": \"Setting up X environment...\"}"
+
+echo "{\"current_action\": \"Starting firefox...\"}"
+start_firefox
+grab start_firefox
+prepare_firefox
+grab prepare_firefox
+echo "{\"current_action\": \"Firefox started, waiting for URL. Run run-analysis.sh in this container to continue\", \"code\": \"ready\"}"
+./eternal-sleep.sh &
+wait

Docker/run-analysis.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+export DISPLAY=:0
+
 INPUT="$1"
 ID=$2
 
@@ -11,11 +13,6 @@ URL=$(unquote $(echo $INPUT | jq .url))
 DOMAINS=`node array-to-lines.js "$(echo $INPUT | jq .third_party_domains)"`
 
 source ./utils.sh
-source ./annotate_header.sh
-
-echo "{\"current_action\": \"Setting up X environment...\"}"
-source ./ephemeral-x.sh
-
 
 PREVIEW="FALSE" # set to "TRUE" in order to enable automatic screenshots kept in preview.png 
 
@@ -29,11 +26,6 @@ then
 fi
 
 
-echo "{\"current_action\": \"Starting firefox...\"}"
-start_firefox
-grab start_firefox
-prepare_firefox
-grab prepare_firefox
 load_website "$URL"
 grab load_website
 open_network_inspector

config.json

@@ -0,0 +1,3 @@
+{
+  "concurrency": 2
+}

container-pool.js

@@ -0,0 +1,75 @@
+const { spawn, spawnSync } = require("child_process");
+const { IMAGE_NAME } = require("./docker-args");
+const { concurrency } = require("./config.json");
+
+class Container {
+  constructor() {
+    this.callbacks = [];
+    this.ready = false;
+    this.id = spawnSync(
+      "docker",
+      ["run", "-d", "-v", `${process.cwd()}/static:/opt/static`, IMAGE_NAME],
+      {
+        cwd: process.cwd(),
+      }
+    )
+      .stdout.toString()
+      .replace("\n", "");
+    this.output = "";
+    this.bg_process = spawn("docker", ["logs", "-f", this.id]);
+    this.bg_process.stdout.on("data", (d) => {
+      try {
+        const parsed = JSON.parse(d.toString());
+        if (parsed.code == "ready") {
+          this.ready = true;
+          this.signalReady();
+        }
+      } catch (e) {}
+      this.output += d.toString();
+    });
+  }
+
+  signalReady() {
+    this.callbacks.forEach((callback) => callback(this));
+  }
+
+  onReady(callback) {
+    this.ready ? callback() : this.callbacks.push(callback);
+  }
+
+  async waitReady() {
+    if (this.ready) {
+      return;
+    }
+    return new Promise((resolve) => {
+      this.onReady(resolve);
+    });
+  }
+
+  close() {
+    spawn("docker", ["rm", "-f", this.id]);
+  }
+}
+
+module.exports = new (class ContainerPool {
+  constructor(concurrency) {
+    this.concurrency = concurrency;
+    this.pool = [];
+    for (let i = 1; i <= this.concurrency; i++) {
+      this.generateContainer();
+    }
+  }
+
+  generateContainer() {
+    this.pool.push(new Container());
+  }
+
+  getContainer() {
+    if (!this.pool.length) {
+      throw new Error("pool is empty, try again!");
+    }
+    const container = this.pool.shift(); // get and remove from pool the oldest container
+    this.generateContainer();
+    return container;
+  }
+})(concurrency);

docker-args.js

@@ -1,10 +1,12 @@
+const IMAGE_NAME = "headless-fox";
+
 const DOCKER_ARGS = [
   "run",
   "-i",
   "-v",
   `${process.cwd()}/static:/opt/static`,
-  "headless-fox",
+  IMAGE_NAME,
   "./script3.sh",
 ];
 
-module.exports = DOCKER_ARGS;
+module.exports = { DOCKER_ARGS, IMAGE_NAME };

index.js

@@ -8,7 +8,7 @@ const { Readable } = require("stream");
 const { spawn } = require("child_process");
 const { requests } = require("./memory");
 const ScreenshotRequest = require("./request");
-const DOCKER_ARGS = require("./docker-args");
+const { DOCKER_ARGS } = require("./docker-args");
 
 const router = new Router();
 

memory.js

@@ -1,5 +1,6 @@
 const queue = require("queue");
-const q = queue({ concurrency: 1, autostart: true, results: [] });
+const { concurrency } = require("./config.json");
+const q = queue({ concurrency, autostart: true, results: [] });
 const requests = {};
 
 module.exports = { q, requests };

request.js

@@ -1,7 +1,9 @@
 const { q, requests } = require("./memory");
+
 const DOCKER_ARGS = require("./docker-args");
 const { v4: uuid } = require("uuid");
 const { spawn } = require("child_process");
+const containerPool = require("./container-pool");
 
 module.exports = class ScreenshotRequest {
   constructor(url, domains) {
@@ -52,12 +54,16 @@ module.exports = class ScreenshotRequest {
 
   async exec() {
     this.started_time = Date.now();
-    return new Promise((resolve, reject) => {
+    return new Promise(async (resolve, reject) => {
       this.status = "running";
+      const container = containerPool.getContainer();
+      await container.waitReady();
       this.process = spawn(
         "docker",
         [
-          ...DOCKER_ARGS,
+          "exec",
+          container.id,
+          "/opt/run-analysis.sh",
           JSON.stringify({
             url: this.url,
             third_party_domains: this.domains,
@@ -68,6 +74,7 @@ module.exports = class ScreenshotRequest {
       );
       this.process.on("close", (exitCode) => {
         this.setFinished();
+        container.close();
         if (exitCode === 0) {
           resolve();
         } else {