` ); - ctx.body.push(/* HTML */ ``); const params = qs.parse(ctx.querystring); - s.push(`Got request to screenshot ${params.url}
`);
+ if (!hasShape({ url: predicates.string }, params)) {
+ throw new Error("MISSING URL PARAM");
+ }
+ response.push(`Got request to screenshot ${params.url}`);
let docker_id = "";
if (!params.url) {
ctx.body = "specify url!";
@@ -120,25 +128,25 @@ router.get("/preview", async (ctx) => {
],
{ cwd: process.cwd() }
);
- starter.stdout.on("data", (data) => {
+ starter.stdout.on("data", (data: Buffer) => {
docker_id += data.toString().replace(/\n/g, "");
});
starter.on("close", () => {
- ctx.body.push("spawned " + docker_id);
+ response.push("spawned " + docker_id);
attach(docker_id, ctx.body);
});
});
router.post("/api/requests", async (ctx) => {
const params = qs.parse(ctx.querystring);
- if (!params.url) {
+ if (!hasShape({ url: predicates.string }, params)) {
ctx.body = "Specify url";
ctx.status = 422;
return;
}
- if (!params.domains) {
- ctx.body = "Specify domains";
+ if (!hasShape({ domains: predicates.array(predicates.string) }, params)) {
+ ctx.body = "Specify domains as an array of strings";
ctx.status = 422;
return;
}
diff --git a/src/memory.ts b/src/memory.ts
new file mode 100644
index 0000000..e60e8a2
--- /dev/null
+++ b/src/memory.ts
@@ -0,0 +1,7 @@
+import queue from "queue";
+import { concurrency } from "../config.json";
+import ScreenshotRequest from "./request";
+const q = queue({ concurrency, autostart: true, results: [] });
+const requests: Record = {};
+
+export { requests, q };
diff --git a/src/request.ts b/src/request.ts
new file mode 100644
index 0000000..13819d8
--- /dev/null
+++ b/src/request.ts
@@ -0,0 +1,139 @@
+import { is, predicates } from "@sealcode/ts-predicates";
+import { ChildProcessWithoutNullStreams, spawn } from "child_process";
+import { v4 as uuid } from "uuid";
+import containerPool from "./container-pool";
+import { q, requests } from "./memory";
+
+let queue_order: ScreenshotRequest[] = [];
+
+export default class ScreenshotRequest {
+ public id = uuid();
+ public status = "waiting";
+ public output = "";
+ public images: Record[] = [];
+ public request_time: number = Date.now();
+ public started_time: number | null = null;
+ public finished_time: number | null = null;
+ public processing_took: number | null = null;
+ public waiting_took: number | null = null;
+ public process: ChildProcessWithoutNullStreams;
+
+ constructor(public url: string, public domains: string[]) {
+ q.push(async () => {
+ return this.exec();
+ });
+ requests[this.id] = this;
+ queue_order.push(this);
+ }
+
+ getJobsAhead(): number {
+ if (this.status != "waiting") {
+ return 0;
+ }
+ let count = 0;
+ for (const request of queue_order) {
+ if (request == this) {
+ break;
+ }
+ count++;
+ }
+ return count;
+ }
+
+ async getJSON(): Promise<{
+ url: string;
+ domains: string[];
+ jobs_ahead: number;
+ id: string;
+ status: string;
+ output: string;
+ images: Record[];
+ request_time: number;
+ started_time: number | null;
+ finished_time: number | null;
+ processing_took: number | null;
+ waiting_took: number | null;
+ elapsed_time_s: number;
+ }> {
+ return {
+ url: this.url,
+ domains: this.domains,
+ jobs_ahead: this.getJobsAhead(),
+ id: this.id,
+ status: this.status,
+ output: this.output,
+ images: this.images,
+ request_time: this.request_time,
+ started_time: this.started_time,
+ finished_time: this.finished_time,
+ processing_took: this.processing_took,
+ waiting_took: this.waiting_took,
+ elapsed_time_s: Math.round(
+ ((this.status === "finished" ? this.finished_time || -1 : Date.now()) -
+ this.request_time) /
+ 1000
+ ),
+ };
+ }
+
+ setFinished(): void {
+ this.status = "finished";
+ this.finished_time = Date.now();
+ if (this.started_time) {
+ this.processing_took = this.finished_time - this.started_time;
+ this.waiting_took = this.started_time - this.request_time;
+ }
+ }
+
+ async exec(): Promise {
+ this.started_time = Date.now();
+ this.status = "running";
+ const container = containerPool.getContainer();
+ await container.waitReady();
+ return new Promise((resolve, reject) => {
+ this.process = spawn(
+ "docker",
+ [
+ "exec",
+ container.id,
+ "/opt/run-analysis.sh",
+ JSON.stringify({
+ url: this.url,
+ third_party_domains: this.domains,
+ }),
+ this.id,
+ ],
+ { cwd: process.cwd() }
+ );
+ this.process.on("close", (exitCode) => {
+ this.setFinished();
+ container.close();
+ queue_order = queue_order.filter((request) => request != this);
+ if (exitCode === 0) {
+ resolve();
+ } else {
+ reject();
+ }
+ });
+ this.process.stdout.on("data", (d: Buffer) => {
+ try {
+ const parsed = JSON.parse(d.toString()) as unknown;
+ if (
+ is(parsed, predicates.object) &&
+ is(parsed.new_file, predicates.object)
+ ) {
+ this.images.push(parsed.new_file);
+ }
+ } catch (e) {
+ //noop
+ }
+ this.output += d.toString();
+ /* console.log("DATA!", d.toString()); */
+ });
+ this.process.stderr.on("data", (d: Buffer) => {
+ this.output += d.toString();
+ /* console.log("STDERR!", d.toString()); */
+ });
+ });
+ }
+}
diff --git a/tsconfig.json b/tsconfig.json
new file mode 100644
index 0000000..1905f87
--- /dev/null
+++ b/tsconfig.json
@@ -0,0 +1,28 @@
+ {
+ "compilerOptions": {
+ "module": "commonjs",
+ "moduleResolution": "node",
+ "noImplicitAny": true,
+ "noImplicitThis": true,
+ "strictNullChecks": true,
+ "target": "ES6",
+ "declaration": true,
+ "esModuleInterop": true,
+ "lib": [
+ "ES6",
+ "ESNext"
+ ],
+ "outDir": "lib",
+ "checkJs": true,
+ "allowJs": true,
+ "declarationDir": "@types",
+ "resolveJsonModule": true,
+ "sourceMap": true
+ },
+ "include": [
+ "src/**/*"
+ ],
+ "exclude": [
+ "src/**/*.test.ts"
+ ]
+ }
\ No newline at end of file