Switch to Typescript

.eslintrc.js

@@ -0,0 +1,37 @@
+module.exports = {
+  env: { node: true },
+  parser: "@typescript-eslint/parser",
+  plugins: ["@typescript-eslint", "prettier"],
+  extends: [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended",
+    "plugin:@typescript-eslint/recommended-requiring-type-checking",
+    "plugin:prettier/recommended",
+  ],
+  ignorePatterns: [".eslintrc.js", "src/**/*.test.ts", "@types/*"],
+  parserOptions: {
+    sourceType: "module",
+    ecmaFeatures: {
+      modules: true,
+    },
+    project: ["./tsconfig.json"],
+  },
+  rules: {
+    "@typescript-eslint/require-await": 0,
+    "no-await-in-loop": 2,
+  },
+  settings: { jsdoc: { mode: "typescript" } },
+  overrides: [
+    {
+      files: ["*.subtest.ts", "*.test.ts"],
+      rules: {
+        "@typescript-eslint/no-unsafe-member-access": 0,
+        "prefer-const": 0,
+        "@typescript-eslint/no-unsafe-call": 0,
+        "@typescript-eslint/no-unsafe-return": 0,
+        "@typescript-eslint/no-unsafe-assignment": 0,
+        "no-await-in-loop": 1, // sometimes it's easier to debug when requests run sequentially
+      },
+    },
+  ],
+};

.gitignore

@@ -1,4 +1,6 @@
+!@types
 /node_modules/
 /static/
 /.log/
 /yarn-error.log
+/lib/

@types/src/container-pool.d.ts

@@ -0,0 +1,23 @@
+/// <reference types="node" />
+import { ChildProcessWithoutNullStreams } from "child_process";
+export declare class Container {
+    callbacks: Array<() => void>;
+    ready: boolean;
+    id: string;
+    output: "";
+    bg_process: ChildProcessWithoutNullStreams;
+    constructor();
+    signalReady(): void;
+    onReady(callback: () => void): void;
+    waitReady(): Promise<void>;
+    close(): void;
+    closeSync(): void;
+}
+declare const _default: {
+    pool: Container[];
+    concurrency: number;
+    generateContainer(): void;
+    getContainer(): Container;
+    clear(): void;
+};
+export default _default;

@types/src/docker-args.d.ts

@@ -0,0 +1,2 @@
+export const DOCKER_ARGS: string[];
+export const IMAGE_NAME: "headless-fox";

@types/src/index.d.ts

@@ -0,0 +1 @@
+export {};

@types/src/memory.d.ts

@@ -0,0 +1,5 @@
+import queue from "queue";
+import ScreenshotRequest from "./request";
+declare const q: queue;
+declare const requests: Record<string, ScreenshotRequest | undefined>;
+export { requests, q };

@types/src/request.d.ts

@@ -0,0 +1,35 @@
+/// <reference types="node" />
+import { ChildProcessWithoutNullStreams } from "child_process";
+export default class ScreenshotRequest {
+    url: string;
+    domains: string[];
+    id: string;
+    status: string;
+    output: string;
+    images: Record<string, unknown>[];
+    request_time: number;
+    started_time: number | null;
+    finished_time: number | null;
+    processing_took: number | null;
+    waiting_took: number | null;
+    process: ChildProcessWithoutNullStreams;
+    constructor(url: string, domains: string[]);
+    getJobsAhead(): number;
+    getJSON(): Promise<{
+        url: string;
+        domains: string[];
+        jobs_ahead: number;
+        id: string;
+        status: string;
+        output: string;
+        images: Record<string, unknown>[];
+        request_time: number;
+        started_time: number | null;
+        finished_time: number | null;
+        processing_took: number | null;
+        waiting_took: number | null;
+        elapsed_time_s: number;
+    }>;
+    setFinished(): void;
+    exec(): Promise<void>;
+}

memory.js

@@ -1,6 +0,0 @@
-const queue = require("queue");
-const { concurrency } = require("./config.json");
-const q = queue({ concurrency, autostart: true, results: [] });
-const requests = {};
-
-module.exports = { q, requests };

package.json

@@ -2,10 +2,12 @@
   "name": "screenshot-service",
   "version": "1.0.0",
   "description": "## Dependencies",
-  "main": "index.js",
+  "main": "lib/src/index.js",
   "scripts": {
     "install-daemon": "cp screenshot-service.service /etc/systemd/system/ && systemctl start screenshot-service && systemctl enable screenshot-service",
-    "start": "docker image build -t headless-fox Docker && BASE_URL=http://localhost:3000 node ."
+    "start": "npm run build && docker image build -t headless-fox Docker && BASE_URL=http://localhost:3000 node .",
+    "build": "tsc",
+    "prepare": "npm run build"
   },
   "repository": {
     "type": "git",
@@ -15,6 +17,8 @@
   "license": "ISC",
   "dependencies": {
     "@koa/router": "^10.1.1",
+    "@sealcode/ts-predicates": "^0.2.2",
+    "@types/node": "^18.0.0",
     "amqplib": "^0.8.0",
     "koa": "^2.13.4",
     "koa-mount": "^4.0.0",
@@ -22,5 +26,21 @@
     "qs": "^6.10.3",
     "queue": "^6.0.2",
     "uuid": "^8.3.2"
+  },
+  "devDependencies": {
+    "@types/koa": "^2.13.4",
+    "@types/koa__router": "^8.0.11",
+    "@types/koa-mount": "^4.0.1",
+    "@types/koa-static": "^4.0.2",
+    "@types/uuid": "^8.3.4",
+    "@typescript-eslint/eslint-plugin": "^4.29.3",
+    "@typescript-eslint/parser": "^4.29.3",
+    "eslint": "^7.32.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-prettier": "^3.4.1",
+    "eslint-plugin-with-tsc-error": "^0.0.7",
+    "prettier": "^2.3.2",
+    "source-map-support": "^0.5.19",
+    "typescript": "^4.3.5"
   }
 }

request.js

@@ -1,119 +0,0 @@
-const { q, requests } = require("./memory");
-
-const DOCKER_ARGS = require("./docker-args");
-const { v4: uuid } = require("uuid");
-const { spawn } = require("child_process");
-const containerPool = require("./container-pool");
-
-let queue_order = [];
-
-module.exports = class ScreenshotRequest {
-  constructor(url, domains) {
-    this.url = url;
-    this.domains = domains;
-    this.id = uuid();
-    this.status = "waiting";
-    this.output = "";
-    this.images = [];
-    this.request_time = Date.now();
-    this.started_time = null;
-    this.finished_time = null;
-    this.processing_took = null;
-    this.waiting_took = null;
-    q.push(async () => {
-      return this.exec();
-    });
-    requests[this.id] = this;
-    queue_order.push(this);
-  }
-
-  getJobsAhead() {
-    if (this.status != "waiting") {
-      return 0;
-    }
-    let count = 0;
-    for (const request of queue_order) {
-      if (request == this) {
-        break;
-      }
-      count++;
-    }
-    return count;
-  }
-
-  async getJSON() {
-    return {
-      url: this.url,
-      domains: this.domains,
-      jobs_ahead: this.getJobsAhead(),
-      id: this.id,
-      status: this.status,
-      output: this.output,
-      images: this.images,
-      request_time: this.request_time,
-      started_time: this.started_time,
-      finished_time: this.finished_time,
-      processing_took: this.processing_took,
-      waiting_took: this.waiting_took,
-      elapsed_time_s: Math.round(
-        ((this.status === "finished" ? this.finished_time : Date.now()) -
-          this.request_time) /
-          1000
-      ),
-    };
-  }
-
-  setFinished() {
-    this.status = "finished";
-    this.finished_time = Date.now();
-    this.processing_took = this.finished_time - this.started_time;
-    this.waiting_took = this.started_time - this.request_time;
-  }
-
-  async exec() {
-    this.started_time = Date.now();
-    return new Promise(async (resolve, reject) => {
-      this.status = "running";
-      const container = containerPool.getContainer();
-      await container.waitReady();
-      this.process = spawn(
-        "docker",
-        [
-          "exec",
-          container.id,
-          "/opt/run-analysis.sh",
-          JSON.stringify({
-            url: this.url,
-            third_party_domains: this.domains,
-          }),
-          this.id,
-        ],
-        { cwd: process.cwd() }
-      );
-      this.process.on("close", (exitCode) => {
-        this.setFinished();
-        container.close();
-        queue_order = queue_order.filter((request) => request != this);
-        if (exitCode === 0) {
-          resolve();
-        } else {
-          reject();
-        }
-      });
-      this.process.stdout.on("data", (d) => {
-        try {
-          const parsed = JSON.parse(d.toString());
-          if (parsed.new_file) {
-            this.images.push(parsed.new_file);
-          }
-        } catch (e) {}
-        this.output += d.toString();
-        /* console.log("DATA!", d.toString()); */
-      });
-      this.process.stderr.on("data", (d) => {
-        this.output += d.toString();
-        /* console.log("STDERR!", d.toString()); */
-      });
-    });
-  }
-};

src/container-pool.ts

@@ -1,11 +1,19 @@
-const { spawn, spawnSync } = require("child_process");
+import { hasShape, predicates, is } from "@sealcode/ts-predicates";
-const { IMAGE_NAME } = require("./docker-args");
+import {
-const { concurrency } = require("./config.json");
+  ChildProcessWithoutNullStreams,
+  spawn,
+  spawnSync,
+} from "child_process";
+import { concurrency } from "../config.json";
+import { IMAGE_NAME } from "./docker-args";
 
-class Container {
+export class Container {
+  callbacks: Array<() => void> = [];
+  ready = false;
+  id: string;
+  output: "";
+  bg_process: ChildProcessWithoutNullStreams;
   constructor() {
-    this.callbacks = [];
-    this.ready = false;
     this.id = spawnSync(
       "docker",
       ["run", "-d", "-v", `${process.cwd()}/static:/opt/static`, IMAGE_NAME],
@@ -15,51 +23,56 @@ class Container {
     )
       .stdout.toString()
       .replace("\n", "");
-    this.output = "";
     this.bg_process = spawn("docker", ["logs", "-f", this.id]);
-    this.bg_process.stdout.on("data", (d) => {
+    this.bg_process.stdout.on("data", (d: Buffer) => {
       try {
-        const parsed = JSON.parse(d.toString());
+        const parsed = JSON.parse(d.toString()) as unknown;
-        if (parsed.code == "ready") {
+        if (
+          is(parsed, predicates.object) &&
+          hasShape({ code: predicates.string }, parsed) &&
+          parsed.code == "ready"
+        ) {
           this.ready = true;
           this.signalReady();
         }
-      } catch (e) {}
+      } catch (e) {
+        // noop
+      }
       this.output += d.toString();
     });
   }
 
-  signalReady() {
+  signalReady(): void {
-    this.callbacks.forEach((callback) => callback(this));
+    this.callbacks.forEach((callback) => callback());
   }
 
-  onReady(callback) {
+  onReady(callback: () => void): void {
     this.ready ? callback() : this.callbacks.push(callback);
   }
 
-  async waitReady() {
+  async waitReady(): Promise<void> {
     if (this.ready) {
       return;
     }
-    return new Promise((resolve) => {
+    return new Promise<void>((resolve) => {
       this.onReady(resolve);
     });
   }
 
-  close() {
+  close(): void {
     spawn("docker", ["rm", "-f", this.id]);
   }
 
-  closeSync() {
+  closeSync(): void {
     spawnSync("docker", ["rm", "-f", this.id]);
     console.log("doker rm done", this.id);
   }
 }
 
-module.exports = new (class ContainerPool {
+export default new (class ContainerPool {
-  constructor(concurrency) {
+  pool: Container[] = [];
+  constructor(public concurrency: number) {
     this.concurrency = concurrency;
-    this.pool = [];
     for (let i = 1; i <= this.concurrency; i++) {
       this.generateContainer();
     }
@@ -79,6 +92,9 @@ module.exports = new (class ContainerPool {
       throw new Error("pool is empty, try again!");
     }
     const container = this.pool.shift(); // get and remove from pool the oldest container
+    if (!container) {
+      throw new Error("Pool was somehow empty!");
+    }
     this.generateContainer();
     return container;
   }

src/index.ts

@@ -1,33 +1,34 @@
-var serve = require("koa-static");
+import Router from "@koa/router";
-const { v4: uuid } = require("uuid");
+import { hasShape, predicates } from "@sealcode/ts-predicates";
-const Koa = require("koa");
+import { spawn } from "child_process";
-const Router = require("@koa/router");
+import Koa from "koa";
-const mount = require("koa-mount");
+import mount from "koa-mount";
-const qs = require("qs");
+import serve from "koa-static";
-const { Readable } = require("stream");
+import qs from "qs";
-const { spawn } = require("child_process");
+import { Readable } from "stream";
-const { requests } = require("./memory");
+import { v4 as uuid } from "uuid";
-const ScreenshotRequest = require("./request");
+import { DOCKER_ARGS } from "./docker-args";
-const { DOCKER_ARGS } = require("./docker-args");
+import { requests } from "./memory";
+import ScreenshotRequest from "./request";
 
 const router = new Router();
 
 // response
 const app = new Koa();
-const static = new Koa();
+const Static = new Koa();
-static.use(serve("./static"));
+Static.use(serve("./static"));
 
-app.use(mount("/static", static));
+app.use(mount("/static", Static));
 
-function attach(docker_id, output_stream) {
+function attach(docker_id: string, output_stream: Readable) {
   // to prevent browser timeout
   const interval = setInterval(() => output_stream.push("<span></span>"), 500);
   const task = spawn("docker", ["logs", "-f", docker_id]);
-  task.stdout.on("data", (d) => {
+  task.stdout.on("data", (d: Buffer) => {
     output_stream.push(d);
     console.log("DATA!", d.toString());
   });
-  task.stderr.on("data", (d) => {
+  task.stderr.on("data", (d: Buffer) => {
     /* output_stream.push(d); */
     console.log("STDERR!", d.toString());
   });
@@ -91,21 +92,28 @@ router.get("/", async (ctx) => {
 });
 
 router.get("/preview", async (ctx) => {
-  const s = new Readable({ read() {} });
+  const response = new Readable({
+    read() {
+      /*noop */
+    },
+  });
   // stream data
   ctx.response.set("content-type", "txt/html");
   ctx.type = "html"; // <-- THIS is the important step!
-  ctx.body = s;
+  ctx.body = response;
-  ctx.body.push("<!doctype html>");
+  response.push("<!doctype html>");
   const id = uuid();
-  ctx.body.push(
+  response.push(
     `<img id="preview" width="1080" height="608" src="/static/${id}/preview.png?id=0"/><br/>`
   );
-  ctx.body.push(/* HTML */ `<script>
+  response.push(/* HTML */ `<script>
     window.interval = setInterval(() => (preview.src = preview.src + "0"), 500);
   </script>`);
   const params = qs.parse(ctx.querystring);
-  s.push(`Got request to screenshot ${params.url}<pre>`);
+  if (!hasShape({ url: predicates.string }, params)) {
+    throw new Error("MISSING URL PARAM");
+  }
+  response.push(`Got request to screenshot ${params.url}<pre>`);
   let docker_id = "";
   if (!params.url) {
     ctx.body = "specify url!";
@@ -120,25 +128,25 @@ router.get("/preview", async (ctx) => {
     ],
     { cwd: process.cwd() }
   );
-  starter.stdout.on("data", (data) => {
+  starter.stdout.on("data", (data: Buffer) => {
     docker_id += data.toString().replace(/\n/g, "");
   });
 
   starter.on("close", () => {
-    ctx.body.push("spawned " + docker_id);
+    response.push("spawned " + docker_id);
     attach(docker_id, ctx.body);
   });
 });
 
 router.post("/api/requests", async (ctx) => {
   const params = qs.parse(ctx.querystring);
-  if (!params.url) {
+  if (!hasShape({ url: predicates.string }, params)) {
     ctx.body = "Specify url";
     ctx.status = 422;
     return;
   }
-  if (!params.domains) {
+  if (!hasShape({ domains: predicates.array(predicates.string) }, params)) {
-    ctx.body = "Specify domains";
+    ctx.body = "Specify domains as an array of strings";
     ctx.status = 422;
     return;
   }

src/memory.ts

@@ -0,0 +1,7 @@
+import queue from "queue";
+import { concurrency } from "../config.json";
+import ScreenshotRequest from "./request";
+const q = queue({ concurrency, autostart: true, results: [] });
+const requests: Record<string, ScreenshotRequest | undefined> = {};
+
+export { requests, q };

src/request.ts

@@ -0,0 +1,139 @@
+import { is, predicates } from "@sealcode/ts-predicates";
+import { ChildProcessWithoutNullStreams, spawn } from "child_process";
+import { v4 as uuid } from "uuid";
+import containerPool from "./container-pool";
+import { q, requests } from "./memory";
+
+let queue_order: ScreenshotRequest[] = [];
+
+export default class ScreenshotRequest {
+  public id = uuid();
+  public status = "waiting";
+  public output = "";
+  public images: Record<string, unknown>[] = [];
+  public request_time: number = Date.now();
+  public started_time: number | null = null;
+  public finished_time: number | null = null;
+  public processing_took: number | null = null;
+  public waiting_took: number | null = null;
+  public process: ChildProcessWithoutNullStreams;
+
+  constructor(public url: string, public domains: string[]) {
+    q.push(async () => {
+      return this.exec();
+    });
+    requests[this.id] = this;
+    queue_order.push(this);
+  }
+
+  getJobsAhead(): number {
+    if (this.status != "waiting") {
+      return 0;
+    }
+    let count = 0;
+    for (const request of queue_order) {
+      if (request == this) {
+        break;
+      }
+      count++;
+    }
+    return count;
+  }
+
+  async getJSON(): Promise<{
+    url: string;
+    domains: string[];
+    jobs_ahead: number;
+    id: string;
+    status: string;
+    output: string;
+    images: Record<string, unknown>[];
+    request_time: number;
+    started_time: number | null;
+    finished_time: number | null;
+    processing_took: number | null;
+    waiting_took: number | null;
+    elapsed_time_s: number;
+  }> {
+    return {
+      url: this.url,
+      domains: this.domains,
+      jobs_ahead: this.getJobsAhead(),
+      id: this.id,
+      status: this.status,
+      output: this.output,
+      images: this.images,
+      request_time: this.request_time,
+      started_time: this.started_time,
+      finished_time: this.finished_time,
+      processing_took: this.processing_took,
+      waiting_took: this.waiting_took,
+      elapsed_time_s: Math.round(
+        ((this.status === "finished" ? this.finished_time || -1 : Date.now()) -
+          this.request_time) /
+          1000
+      ),
+    };
+  }
+
+  setFinished(): void {
+    this.status = "finished";
+    this.finished_time = Date.now();
+    if (this.started_time) {
+      this.processing_took = this.finished_time - this.started_time;
+      this.waiting_took = this.started_time - this.request_time;
+    }
+  }
+
+  async exec(): Promise<void> {
+    this.started_time = Date.now();
+    this.status = "running";
+    const container = containerPool.getContainer();
+    await container.waitReady();
+    return new Promise((resolve, reject) => {
+      this.process = spawn(
+        "docker",
+        [
+          "exec",
+          container.id,
+          "/opt/run-analysis.sh",
+          JSON.stringify({
+            url: this.url,
+            third_party_domains: this.domains,
+          }),
+          this.id,
+        ],
+        { cwd: process.cwd() }
+      );
+      this.process.on("close", (exitCode) => {
+        this.setFinished();
+        container.close();
+        queue_order = queue_order.filter((request) => request != this);
+        if (exitCode === 0) {
+          resolve();
+        } else {
+          reject();
+        }
+      });
+      this.process.stdout.on("data", (d: Buffer) => {
+        try {
+          const parsed = JSON.parse(d.toString()) as unknown;
+          if (
+            is(parsed, predicates.object) &&
+            is(parsed.new_file, predicates.object)
+          ) {
+            this.images.push(parsed.new_file);
+          }
+        } catch (e) {
+          //noop
+        }
+        this.output += d.toString();
+        /* console.log("DATA!", d.toString()); */
+      });
+      this.process.stderr.on("data", (d: Buffer) => {
+        this.output += d.toString();
+        /* console.log("STDERR!", d.toString()); */
+      });
+    });
+  }
+}

tsconfig.json

@@ -0,0 +1,28 @@
+	{
+	  "compilerOptions": {
+	    "module": "commonjs",
+	    "moduleResolution": "node",
+	    "noImplicitAny": true,
+	    "noImplicitThis": true,
+	    "strictNullChecks": true,
+	    "target": "ES6",
+	    "declaration": true,
+	    "esModuleInterop": true,
+	    "lib": [
+	      "ES6",
+	      "ESNext"
+	    ],
+	    "outDir": "lib",
+	    "checkJs": true,
+	    "allowJs": true,
+	    "declarationDir": "@types",
+	    "resolveJsonModule": true,
+	    "sourceMap": true
+	  },
+	  "include": [
+	    "src/**/*"
+	  ],
+	  "exclude": [
+	    "src/**/*.test.ts"
+	  ]
+	}