diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 96c4e5d..ed77327 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -47,5 +47,6 @@ RUN apk add clang RUN apk add freetype-dev RUN python3 -m pip install --upgrade Pillow COPY . /opt +CMD /opt/prepare-firefox.sh WORKDIR /opt diff --git a/Docker/eternal-sleep.sh b/Docker/eternal-sleep.sh new file mode 100755 index 0000000..3021cd7 --- /dev/null +++ b/Docker/eternal-sleep.sh @@ -0,0 +1,6 @@ +#!/bin/bash + + +while true; do + sleep 1; +done diff --git a/Docker/prepare-firefox.sh b/Docker/prepare-firefox.sh new file mode 100755 index 0000000..7aa7b35 --- /dev/null +++ b/Docker/prepare-firefox.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +source ./ephemeral-x.sh +source ./annotate_header.sh +source ./utils.sh + +echo "{\"current_action\": \"Setting up X environment...\"}" + +echo "{\"current_action\": \"Starting firefox...\"}" +start_firefox +grab start_firefox +prepare_firefox +grab prepare_firefox +echo "{\"current_action\": \"Firefox started, waiting for URL. Run run-analysis.sh in this container to continue\", \"code\": \"ready\"}" +./eternal-sleep.sh & +wait diff --git a/Docker/script3.sh b/Docker/run-analysis.sh similarity index 88% rename from Docker/script3.sh rename to Docker/run-analysis.sh index 7715007..c6c3729 100755 --- a/Docker/script3.sh +++ b/Docker/run-analysis.sh @@ -1,5 +1,7 @@ #!/bin/bash +export DISPLAY=:0 + INPUT="$1" ID=$2 @@ -11,11 +13,6 @@ URL=$(unquote $(echo $INPUT | jq .url)) DOMAINS=`node array-to-lines.js "$(echo $INPUT | jq .third_party_domains)"` source ./utils.sh -source ./annotate_header.sh - -echo "{\"current_action\": \"Setting up X environment...\"}" -source ./ephemeral-x.sh - PREVIEW="FALSE" # set to "TRUE" in order to enable automatic screenshots kept in preview.png @@ -29,11 +26,6 @@ then fi -echo "{\"current_action\": \"Starting firefox...\"}" -start_firefox -grab start_firefox -prepare_firefox -grab prepare_firefox load_website "$URL" grab load_website open_network_inspector diff --git a/config.json b/config.json new file mode 100644 index 0000000..881f3fd --- /dev/null +++ b/config.json @@ -0,0 +1,3 @@ +{ + "concurrency": 2 +} diff --git a/container-pool.js b/container-pool.js new file mode 100644 index 0000000..15be89f --- /dev/null +++ b/container-pool.js @@ -0,0 +1,75 @@ +const { spawn, spawnSync } = require("child_process"); +const { IMAGE_NAME } = require("./docker-args"); +const { concurrency } = require("./config.json"); + +class Container { + constructor() { + this.callbacks = []; + this.ready = false; + this.id = spawnSync( + "docker", + ["run", "-d", "-v", `${process.cwd()}/static:/opt/static`, IMAGE_NAME], + { + cwd: process.cwd(), + } + ) + .stdout.toString() + .replace("\n", ""); + this.output = ""; + this.bg_process = spawn("docker", ["logs", "-f", this.id]); + this.bg_process.stdout.on("data", (d) => { + try { + const parsed = JSON.parse(d.toString()); + if (parsed.code == "ready") { + this.ready = true; + this.signalReady(); + } + } catch (e) {} + this.output += d.toString(); + }); + } + + signalReady() { + this.callbacks.forEach((callback) => callback(this)); + } + + onReady(callback) { + this.ready ? callback() : this.callbacks.push(callback); + } + + async waitReady() { + if (this.ready) { + return; + } + return new Promise((resolve) => { + this.onReady(resolve); + }); + } + + close() { + spawn("docker", ["rm", "-f", this.id]); + } +} + +module.exports = new (class ContainerPool { + constructor(concurrency) { + this.concurrency = concurrency; + this.pool = []; + for (let i = 1; i <= this.concurrency; i++) { + this.generateContainer(); + } + } + + generateContainer() { + this.pool.push(new Container()); + } + + getContainer() { + if (!this.pool.length) { + throw new Error("pool is empty, try again!"); + } + const container = this.pool.shift(); // get and remove from pool the oldest container + this.generateContainer(); + return container; + } +})(concurrency); diff --git a/docker-args.js b/docker-args.js index b7eeffa..56ef69b 100644 --- a/docker-args.js +++ b/docker-args.js @@ -1,10 +1,12 @@ +const IMAGE_NAME = "headless-fox"; + const DOCKER_ARGS = [ "run", "-i", "-v", `${process.cwd()}/static:/opt/static`, - "headless-fox", + IMAGE_NAME, "./script3.sh", ]; -module.exports = DOCKER_ARGS; +module.exports = { DOCKER_ARGS, IMAGE_NAME }; diff --git a/index.js b/index.js index 48b28bd..2bbde07 100644 --- a/index.js +++ b/index.js @@ -8,7 +8,7 @@ const { Readable } = require("stream"); const { spawn } = require("child_process"); const { requests } = require("./memory"); const ScreenshotRequest = require("./request"); -const DOCKER_ARGS = require("./docker-args"); +const { DOCKER_ARGS } = require("./docker-args"); const router = new Router(); diff --git a/memory.js b/memory.js index da3fdee..9852a76 100644 --- a/memory.js +++ b/memory.js @@ -1,5 +1,6 @@ const queue = require("queue"); -const q = queue({ concurrency: 1, autostart: true, results: [] }); +const { concurrency } = require("./config.json"); +const q = queue({ concurrency, autostart: true, results: [] }); const requests = {}; module.exports = { q, requests }; diff --git a/request.js b/request.js index 4d9377a..a5ee4d7 100644 --- a/request.js +++ b/request.js @@ -1,7 +1,9 @@ const { q, requests } = require("./memory"); + const DOCKER_ARGS = require("./docker-args"); const { v4: uuid } = require("uuid"); const { spawn } = require("child_process"); +const containerPool = require("./container-pool"); module.exports = class ScreenshotRequest { constructor(url, domains) { @@ -52,12 +54,16 @@ module.exports = class ScreenshotRequest { async exec() { this.started_time = Date.now(); - return new Promise((resolve, reject) => { + return new Promise(async (resolve, reject) => { this.status = "running"; + const container = containerPool.getContainer(); + await container.waitReady(); this.process = spawn( "docker", [ - ...DOCKER_ARGS, + "exec", + container.id, + "/opt/run-analysis.sh", JSON.stringify({ url: this.url, third_party_domains: this.domains, @@ -68,6 +74,7 @@ module.exports = class ScreenshotRequest { ); this.process.on("close", (exitCode) => { this.setFinished(); + container.close(); if (exitCode === 0) { resolve(); } else {