((resolve) => {
this.onReady(resolve);
});
}
- close() {
+ close(): void {
spawn("docker", ["rm", "-f", this.id]);
}
- closeSync() {
+ closeSync(): void {
spawnSync("docker", ["rm", "-f", this.id]);
console.log("doker rm done", this.id);
}
}
-module.exports = new (class ContainerPool {
- constructor(concurrency) {
+export default new (class ContainerPool {
+ pool: Container[] = [];
+ constructor(public concurrency: number) {
this.concurrency = concurrency;
- this.pool = [];
for (let i = 1; i <= this.concurrency; i++) {
this.generateContainer();
}
@@ -79,6 +92,9 @@ module.exports = new (class ContainerPool {
throw new Error("pool is empty, try again!");
}
const container = this.pool.shift(); // get and remove from pool the oldest container
+ if (!container) {
+ throw new Error("Pool was somehow empty!");
+ }
this.generateContainer();
return container;
}
diff --git a/docker-args.js b/src/docker-args.js
similarity index 100%
rename from docker-args.js
rename to src/docker-args.js
diff --git a/index.js b/src/index.ts
similarity index 74%
rename from index.js
rename to src/index.ts
index 2bbde07..6c47f94 100644
--- a/index.js
+++ b/src/index.ts
@@ -1,33 +1,34 @@
-var serve = require("koa-static");
-const { v4: uuid } = require("uuid");
-const Koa = require("koa");
-const Router = require("@koa/router");
-const mount = require("koa-mount");
-const qs = require("qs");
-const { Readable } = require("stream");
-const { spawn } = require("child_process");
-const { requests } = require("./memory");
-const ScreenshotRequest = require("./request");
-const { DOCKER_ARGS } = require("./docker-args");
+import Router from "@koa/router";
+import { hasShape, predicates } from "@sealcode/ts-predicates";
+import { spawn } from "child_process";
+import Koa from "koa";
+import mount from "koa-mount";
+import serve from "koa-static";
+import qs from "qs";
+import { Readable } from "stream";
+import { v4 as uuid } from "uuid";
+import { DOCKER_ARGS } from "./docker-args";
+import { requests } from "./memory";
+import ScreenshotRequest from "./request";
const router = new Router();
// response
const app = new Koa();
-const static = new Koa();
-static.use(serve("./static"));
+const Static = new Koa();
+Static.use(serve("./static"));
-app.use(mount("/static", static));
+app.use(mount("/static", Static));
-function attach(docker_id, output_stream) {
+function attach(docker_id: string, output_stream: Readable) {
// to prevent browser timeout
const interval = setInterval(() => output_stream.push(""), 500);
const task = spawn("docker", ["logs", "-f", docker_id]);
- task.stdout.on("data", (d) => {
+ task.stdout.on("data", (d: Buffer) => {
output_stream.push(d);
console.log("DATA!", d.toString());
});
- task.stderr.on("data", (d) => {
+ task.stderr.on("data", (d: Buffer) => {
/* output_stream.push(d); */
console.log("STDERR!", d.toString());
});
@@ -91,21 +92,28 @@ router.get("/", async (ctx) => {
});
router.get("/preview", async (ctx) => {
- const s = new Readable({ read() {} });
+ const response = new Readable({
+ read() {
+ /*noop */
+ },
+ });
// stream data
ctx.response.set("content-type", "txt/html");
ctx.type = "html"; // <-- THIS is the important step!
- ctx.body = s;
- ctx.body.push("");
+ ctx.body = response;
+ response.push("");
const id = uuid();
- ctx.body.push(
+ response.push(
`
`
);
- ctx.body.push(/* HTML */ ``);
const params = qs.parse(ctx.querystring);
- s.push(`Got request to screenshot ${params.url}`);
+ if (!hasShape({ url: predicates.string }, params)) {
+ throw new Error("MISSING URL PARAM");
+ }
+ response.push(`Got request to screenshot ${params.url}`);
let docker_id = "";
if (!params.url) {
ctx.body = "specify url!";
@@ -120,25 +128,25 @@ router.get("/preview", async (ctx) => {
],
{ cwd: process.cwd() }
);
- starter.stdout.on("data", (data) => {
+ starter.stdout.on("data", (data: Buffer) => {
docker_id += data.toString().replace(/\n/g, "");
});
starter.on("close", () => {
- ctx.body.push("spawned " + docker_id);
+ response.push("spawned " + docker_id);
attach(docker_id, ctx.body);
});
});
router.post("/api/requests", async (ctx) => {
const params = qs.parse(ctx.querystring);
- if (!params.url) {
+ if (!hasShape({ url: predicates.string }, params)) {
ctx.body = "Specify url";
ctx.status = 422;
return;
}
- if (!params.domains) {
- ctx.body = "Specify domains";
+ if (!hasShape({ domains: predicates.array(predicates.string) }, params)) {
+ ctx.body = "Specify domains as an array of strings";
ctx.status = 422;
return;
}
diff --git a/src/memory.ts b/src/memory.ts
new file mode 100644
index 0000000..e60e8a2
--- /dev/null
+++ b/src/memory.ts
@@ -0,0 +1,7 @@
+import queue from "queue";
+import { concurrency } from "../config.json";
+import ScreenshotRequest from "./request";
+const q = queue({ concurrency, autostart: true, results: [] });
+const requests: Record = {};
+
+export { requests, q };
diff --git a/src/request.ts b/src/request.ts
new file mode 100644
index 0000000..13819d8
--- /dev/null
+++ b/src/request.ts
@@ -0,0 +1,139 @@
+import { is, predicates } from "@sealcode/ts-predicates";
+import { ChildProcessWithoutNullStreams, spawn } from "child_process";
+import { v4 as uuid } from "uuid";
+import containerPool from "./container-pool";
+import { q, requests } from "./memory";
+
+let queue_order: ScreenshotRequest[] = [];
+
+export default class ScreenshotRequest {
+ public id = uuid();
+ public status = "waiting";
+ public output = "";
+ public images: Record[] = [];
+ public request_time: number = Date.now();
+ public started_time: number | null = null;
+ public finished_time: number | null = null;
+ public processing_took: number | null = null;
+ public waiting_took: number | null = null;
+ public process: ChildProcessWithoutNullStreams;
+
+ constructor(public url: string, public domains: string[]) {
+ q.push(async () => {
+ return this.exec();
+ });
+ requests[this.id] = this;
+ queue_order.push(this);
+ }
+
+ getJobsAhead(): number {
+ if (this.status != "waiting") {
+ return 0;
+ }
+ let count = 0;
+ for (const request of queue_order) {
+ if (request == this) {
+ break;
+ }
+ count++;
+ }
+ return count;
+ }
+
+ async getJSON(): Promise<{
+ url: string;
+ domains: string[];
+ jobs_ahead: number;
+ id: string;
+ status: string;
+ output: string;
+ images: Record[];
+ request_time: number;
+ started_time: number | null;
+ finished_time: number | null;
+ processing_took: number | null;
+ waiting_took: number | null;
+ elapsed_time_s: number;
+ }> {
+ return {
+ url: this.url,
+ domains: this.domains,
+ jobs_ahead: this.getJobsAhead(),
+ id: this.id,
+ status: this.status,
+ output: this.output,
+ images: this.images,
+ request_time: this.request_time,
+ started_time: this.started_time,
+ finished_time: this.finished_time,
+ processing_took: this.processing_took,
+ waiting_took: this.waiting_took,
+ elapsed_time_s: Math.round(
+ ((this.status === "finished" ? this.finished_time || -1 : Date.now()) -
+ this.request_time) /
+ 1000
+ ),
+ };
+ }
+
+ setFinished(): void {
+ this.status = "finished";
+ this.finished_time = Date.now();
+ if (this.started_time) {
+ this.processing_took = this.finished_time - this.started_time;
+ this.waiting_took = this.started_time - this.request_time;
+ }
+ }
+
+ async exec(): Promise {
+ this.started_time = Date.now();
+ this.status = "running";
+ const container = containerPool.getContainer();
+ await container.waitReady();
+ return new Promise((resolve, reject) => {
+ this.process = spawn(
+ "docker",
+ [
+ "exec",
+ container.id,
+ "/opt/run-analysis.sh",
+ JSON.stringify({
+ url: this.url,
+ third_party_domains: this.domains,
+ }),
+ this.id,
+ ],
+ { cwd: process.cwd() }
+ );
+ this.process.on("close", (exitCode) => {
+ this.setFinished();
+ container.close();
+ queue_order = queue_order.filter((request) => request != this);
+ if (exitCode === 0) {
+ resolve();
+ } else {
+ reject();
+ }
+ });
+ this.process.stdout.on("data", (d: Buffer) => {
+ try {
+ const parsed = JSON.parse(d.toString()) as unknown;
+ if (
+ is(parsed, predicates.object) &&
+ is(parsed.new_file, predicates.object)
+ ) {
+ this.images.push(parsed.new_file);
+ }
+ } catch (e) {
+ //noop
+ }
+ this.output += d.toString();
+ /* console.log("DATA!", d.toString()); */
+ });
+ this.process.stderr.on("data", (d: Buffer) => {
+ this.output += d.toString();
+ /* console.log("STDERR!", d.toString()); */
+ });
+ });
+ }
+}
diff --git a/tsconfig.json b/tsconfig.json
new file mode 100644
index 0000000..1905f87
--- /dev/null
+++ b/tsconfig.json
@@ -0,0 +1,28 @@
+ {
+ "compilerOptions": {
+ "module": "commonjs",
+ "moduleResolution": "node",
+ "noImplicitAny": true,
+ "noImplicitThis": true,
+ "strictNullChecks": true,
+ "target": "ES6",
+ "declaration": true,
+ "esModuleInterop": true,
+ "lib": [
+ "ES6",
+ "ESNext"
+ ],
+ "outDir": "lib",
+ "checkJs": true,
+ "allowJs": true,
+ "declarationDir": "@types",
+ "resolveJsonModule": true,
+ "sourceMap": true
+ },
+ "include": [
+ "src/**/*"
+ ],
+ "exclude": [
+ "src/**/*.test.ts"
+ ]
+ }
\ No newline at end of file