icd
/
screenshot-service
2
0
Fork 0
Code Issues 18 Pull Requests Projects 1 Releases Wiki Activity

Switch to Typescript

This commit is contained in:
Kuba Orlik 2022-06-19 13:33:25 +02:00
parent 34cb00139f
commit 535535efd3
17 changed files with 3848 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
.eslintrc.js Normal file
View File

@ -0,0 +1,37 @@
module.exports = {
env: { node: true },
parser: "@typescript-eslint/parser",
plugins: ["@typescript-eslint", "prettier"],
extends: [
"eslint:recommended",
"plugin:@typescript-eslint/recommended",
"plugin:@typescript-eslint/recommended-requiring-type-checking",
"plugin:prettier/recommended",
],
ignorePatterns: [".eslintrc.js", "src/**/*.test.ts", "@types/*"],
parserOptions: {
sourceType: "module",
ecmaFeatures: {
modules: true,
},
project: ["./tsconfig.json"],
},
rules: {
"@typescript-eslint/require-await": 0,
"no-await-in-loop": 2,
},
settings: { jsdoc: { mode: "typescript" } },
overrides: [
{
files: ["*.subtest.ts", "*.test.ts"],
rules: {
"@typescript-eslint/no-unsafe-member-access": 0,
"prefer-const": 0,
"@typescript-eslint/no-unsafe-call": 0,
"@typescript-eslint/no-unsafe-return": 0,
"@typescript-eslint/no-unsafe-assignment": 0,
"no-await-in-loop": 1, // sometimes it's easier to debug when requests run sequentially
},
},
],
};

2
.gitignore vendored
View File

@ -1,4 +1,6 @@
!@types
/node_modules/
/static/
/.log/
/yarn-error.log
/lib/

23
@types/src/container-pool.d.ts vendored Normal file
View File

@ -0,0 +1,23 @@
/// <reference types="node" />
import { ChildProcessWithoutNullStreams } from "child_process";
export declare class Container {
callbacks: Array<() => void>;
ready: boolean;
id: string;
output: "";
bg_process: ChildProcessWithoutNullStreams;
constructor();
signalReady(): void;
onReady(callback: () => void): void;
waitReady(): Promise<void>;
close(): void;
closeSync(): void;
}
declare const _default: {
pool: Container[];
concurrency: number;
generateContainer(): void;
getContainer(): Container;
clear(): void;
};
export default _default;

2
@types/src/docker-args.d.ts vendored Normal file
View File

@ -0,0 +1,2 @@
export const DOCKER_ARGS: string[];
export const IMAGE_NAME: "headless-fox";

1
@types/src/index.d.ts vendored Normal file
View File

@ -0,0 +1 @@
export {};

5
@types/src/memory.d.ts vendored Normal file
View File

@ -0,0 +1,5 @@
import queue from "queue";
import ScreenshotRequest from "./request";
declare const q: queue;
declare const requests: Record<string, ScreenshotRequest | undefined>;
export { requests, q };

35
@types/src/request.d.ts vendored Normal file
View File

@ -0,0 +1,35 @@
/// <reference types="node" />
import { ChildProcessWithoutNullStreams } from "child_process";
export default class ScreenshotRequest {
url: string;
domains: string[];
id: string;
status: string;
output: string;
images: Record<string, unknown>[];
request_time: number;
started_time: number | null;
finished_time: number | null;
processing_took: number | null;
waiting_took: number | null;
process: ChildProcessWithoutNullStreams;
constructor(url: string, domains: string[]);
getJobsAhead(): number;
getJSON(): Promise<{
url: string;
domains: string[];
jobs_ahead: number;
id: string;
status: string;
output: string;
images: Record<string, unknown>[];
request_time: number;
started_time: number | null;
finished_time: number | null;
processing_took: number | null;
waiting_took: number | null;
elapsed_time_s: number;
}>;
setFinished(): void;
exec(): Promise<void>;
}

6
memory.js
View File

@ -1,6 +0,0 @@
const queue = require("queue");
const { concurrency } = require("./config.json");
const q = queue({ concurrency, autostart: true, results: [] });
const requests = {};
module.exports = { q, requests };

3474
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

24
package.json
View File

@ -2,10 +2,12 @@
"name": "screenshot-service",
"version": "1.0.0",
"description": "## Dependencies",
"main": "index.js",
"main": "lib/src/index.js",
"scripts": {
"install-daemon": "cp screenshot-service.service /etc/systemd/system/ && systemctl start screenshot-service && systemctl enable screenshot-service",
"start": "docker image build -t headless-fox Docker && BASE_URL=http://localhost:3000 node ."
"start": "npm run build && docker image build -t headless-fox Docker && BASE_URL=http://localhost:3000 node .",
"build": "tsc",
"prepare": "npm run build"
},
"repository": {
"type": "git",
@ -15,6 +17,8 @@
"license": "ISC",
"dependencies": {
"@koa/router": "^10.1.1",
"@sealcode/ts-predicates": "^0.2.2",
"@types/node": "^18.0.0",
"amqplib": "^0.8.0",
"koa": "^2.13.4",
"koa-mount": "^4.0.0",
@ -22,5 +26,21 @@
"qs": "^6.10.3",
"queue": "^6.0.2",
"uuid": "^8.3.2"
},
"devDependencies": {
"@types/koa": "^2.13.4",
"@types/koa__router": "^8.0.11",
"@types/koa-mount": "^4.0.1",
"@types/koa-static": "^4.0.2",
"@types/uuid": "^8.3.4",
"@typescript-eslint/eslint-plugin": "^4.29.3",
"@typescript-eslint/parser": "^4.29.3",
"eslint": "^7.32.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-prettier": "^3.4.1",
"eslint-plugin-with-tsc-error": "^0.0.7",
"prettier": "^2.3.2",
"source-map-support": "^0.5.19",
"typescript": "^4.3.5"
}
}

119
request.js
View File

@ -1,119 +0,0 @@
const { q, requests } = require("./memory");
const DOCKER_ARGS = require("./docker-args");
const { v4: uuid } = require("uuid");
const { spawn } = require("child_process");
const containerPool = require("./container-pool");
let queue_order = [];
module.exports = class ScreenshotRequest {
constructor(url, domains) {
this.url = url;
this.domains = domains;
this.id = uuid();
this.status = "waiting";
this.output = "";
this.images = [];
this.request_time = Date.now();
this.started_time = null;
this.finished_time = null;
this.processing_took = null;
this.waiting_took = null;
q.push(async () => {
return this.exec();
});
requests[this.id] = this;
queue_order.push(this);
}
getJobsAhead() {
if (this.status != "waiting") {
return 0;
}
let count = 0;
for (const request of queue_order) {
if (request == this) {
break;
}
count++;
}
return count;
}
async getJSON() {
return {
url: this.url,
domains: this.domains,
jobs_ahead: this.getJobsAhead(),
id: this.id,
status: this.status,
output: this.output,
images: this.images,
request_time: this.request_time,
started_time: this.started_time,
finished_time: this.finished_time,
processing_took: this.processing_took,
waiting_took: this.waiting_took,
elapsed_time_s: Math.round(
((this.status === "finished" ? this.finished_time : Date.now()) -
this.request_time) /
1000
),
};
}
setFinished() {
this.status = "finished";
this.finished_time = Date.now();
this.processing_took = this.finished_time - this.started_time;
this.waiting_took = this.started_time - this.request_time;
}
async exec() {
this.started_time = Date.now();
return new Promise(async (resolve, reject) => {
this.status = "running";
const container = containerPool.getContainer();
await container.waitReady();
this.process = spawn(
"docker",
[
"exec",
container.id,
"/opt/run-analysis.sh",
JSON.stringify({
url: this.url,
third_party_domains: this.domains,
}),
this.id,
],
{ cwd: process.cwd() }
);
this.process.on("close", (exitCode) => {
this.setFinished();
container.close();
queue_order = queue_order.filter((request) => request != this);
if (exitCode === 0) {
resolve();
} else {
reject();
}
});
this.process.stdout.on("data", (d) => {
try {
const parsed = JSON.parse(d.toString());
if (parsed.new_file) {
this.images.push(parsed.new_file);
}
} catch (e) {}
this.output += d.toString();
/* console.log("DATA!", d.toString()); */
});
this.process.stderr.on("data", (d) => {
this.output += d.toString();
/* console.log("STDERR!", d.toString()); */
});
});
}
};

58
container-pool.js → src/container-pool.ts
View File

@ -1,11 +1,19 @@
const { spawn, spawnSync } = require("child_process");
const { IMAGE_NAME } = require("./docker-args");
const { concurrency } = require("./config.json");
import { hasShape, predicates, is } from "@sealcode/ts-predicates";
import {
ChildProcessWithoutNullStreams,
spawn,
spawnSync,
} from "child_process";
import { concurrency } from "../config.json";
import { IMAGE_NAME } from "./docker-args";
class Container {
export class Container {
callbacks: Array<() => void> = [];
ready = false;
id: string;
output: "";
bg_process: ChildProcessWithoutNullStreams;
constructor() {
this.callbacks = [];
this.ready = false;
this.id = spawnSync(
"docker",
["run", "-d", "-v", `${process.cwd()}/static:/opt/static`, IMAGE_NAME],
@ -15,51 +23,56 @@ class Container {
)
.stdout.toString()
.replace("\n", "");
this.output = "";
this.bg_process = spawn("docker", ["logs", "-f", this.id]);
this.bg_process.stdout.on("data", (d) => {
this.bg_process.stdout.on("data", (d: Buffer) => {
try {
const parsed = JSON.parse(d.toString());
if (parsed.code == "ready") {
const parsed = JSON.parse(d.toString()) as unknown;
if (
is(parsed, predicates.object) &&
hasShape({ code: predicates.string }, parsed) &&
parsed.code == "ready"
) {
this.ready = true;
this.signalReady();
}
} catch (e) {}
} catch (e) {
// noop
}
this.output += d.toString();
});
}
signalReady() {
this.callbacks.forEach((callback) => callback(this));
signalReady(): void {
this.callbacks.forEach((callback) => callback());
}
onReady(callback) {
onReady(callback: () => void): void {
this.ready ? callback() : this.callbacks.push(callback);
}
async waitReady() {
async waitReady(): Promise<void> {
if (this.ready) {
return;
}
return new Promise((resolve) => {
return new Promise<void>((resolve) => {
this.onReady(resolve);
});
}
close() {
close(): void {
spawn("docker", ["rm", "-f", this.id]);
}
closeSync() {
closeSync(): void {
spawnSync("docker", ["rm", "-f", this.id]);
console.log("doker rm done", this.id);
}
}
module.exports = new (class ContainerPool {
constructor(concurrency) {
export default new (class ContainerPool {
pool: Container[] = [];
constructor(public concurrency: number) {
this.concurrency = concurrency;
this.pool = [];
for (let i = 1; i <= this.concurrency; i++) {
this.generateContainer();
}
@ -79,6 +92,9 @@ module.exports = new (class ContainerPool {
throw new Error("pool is empty, try again!");
}
const container = this.pool.shift(); // get and remove from pool the oldest container
if (!container) {
throw new Error("Pool was somehow empty!");
}
this.generateContainer();
return container;
}

0
docker-args.js → src/docker-args.js
View File

64
index.js → src/index.ts
View File

@ -1,33 +1,34 @@
var serve = require("koa-static");
const { v4: uuid } = require("uuid");
const Koa = require("koa");
const Router = require("@koa/router");
const mount = require("koa-mount");
const qs = require("qs");
const { Readable } = require("stream");
const { spawn } = require("child_process");
const { requests } = require("./memory");
const ScreenshotRequest = require("./request");
const { DOCKER_ARGS } = require("./docker-args");
import Router from "@koa/router";
import { hasShape, predicates } from "@sealcode/ts-predicates";
import { spawn } from "child_process";
import Koa from "koa";
import mount from "koa-mount";
import serve from "koa-static";
import qs from "qs";
import { Readable } from "stream";
import { v4 as uuid } from "uuid";
import { DOCKER_ARGS } from "./docker-args";
import { requests } from "./memory";
import ScreenshotRequest from "./request";
const router = new Router();
// response
const app = new Koa();
const static = new Koa();
static.use(serve("./static"));
const Static = new Koa();
Static.use(serve("./static"));
app.use(mount("/static", static));
app.use(mount("/static", Static));
function attach(docker_id, output_stream) {
function attach(docker_id: string, output_stream: Readable) {
// to prevent browser timeout
const interval = setInterval(() => output_stream.push("<span></span>"), 500);
const task = spawn("docker", ["logs", "-f", docker_id]);
task.stdout.on("data", (d) => {
task.stdout.on("data", (d: Buffer) => {
output_stream.push(d);
console.log("DATA!", d.toString());
});
task.stderr.on("data", (d) => {
task.stderr.on("data", (d: Buffer) => {
/* output_stream.push(d); */
console.log("STDERR!", d.toString());
});
@ -91,21 +92,28 @@ router.get("/", async (ctx) => {
});
router.get("/preview", async (ctx) => {
const s = new Readable({ read() {} });
const response = new Readable({
read() {
/*noop */
},
});
// stream data
ctx.response.set("content-type", "txt/html");
ctx.type = "html"; // <-- THIS is the important step!
ctx.body = s;
ctx.body.push("<!doctype html>");
ctx.body = response;
response.push("<!doctype html>");
const id = uuid();
ctx.body.push(
response.push(
`<img id="preview" width="1080" height="608" src="/static/${id}/preview.png?id=0"/><br/>`
);
ctx.body.push(/* HTML */ `<script>
response.push(/* HTML */ `<script>
window.interval = setInterval(() => (preview.src = preview.src + "0"), 500);
</script>`);
const params = qs.parse(ctx.querystring);
s.push(`Got request to screenshot ${params.url}<pre>`);
if (!hasShape({ url: predicates.string }, params)) {
throw new Error("MISSING URL PARAM");
}
response.push(`Got request to screenshot ${params.url}<pre>`);
let docker_id = "";
if (!params.url) {
ctx.body = "specify url!";
@ -120,25 +128,25 @@ router.get("/preview", async (ctx) => {
],
{ cwd: process.cwd() }
);
starter.stdout.on("data", (data) => {
starter.stdout.on("data", (data: Buffer) => {
docker_id += data.toString().replace(/\n/g, "");
});
starter.on("close", () => {
ctx.body.push("spawned " + docker_id);
response.push("spawned " + docker_id);
attach(docker_id, ctx.body);
});
});
router.post("/api/requests", async (ctx) => {
const params = qs.parse(ctx.querystring);
if (!params.url) {
if (!hasShape({ url: predicates.string }, params)) {
ctx.body = "Specify url";
ctx.status = 422;
return;
}
if (!params.domains) {
ctx.body = "Specify domains";
if (!hasShape({ domains: predicates.array(predicates.string) }, params)) {
ctx.body = "Specify domains as an array of strings";
ctx.status = 422;
return;
}

7
src/memory.ts Normal file
View File

@ -0,0 +1,7 @@
import queue from "queue";
import { concurrency } from "../config.json";
import ScreenshotRequest from "./request";
const q = queue({ concurrency, autostart: true, results: [] });
const requests: Record<string, ScreenshotRequest | undefined> = {};
export { requests, q };

139
src/request.ts Normal file
View File

@ -0,0 +1,139 @@
import { is, predicates } from "@sealcode/ts-predicates";
import { ChildProcessWithoutNullStreams, spawn } from "child_process";
import { v4 as uuid } from "uuid";
import containerPool from "./container-pool";
import { q, requests } from "./memory";
let queue_order: ScreenshotRequest[] = [];
export default class ScreenshotRequest {
public id = uuid();
public status = "waiting";
public output = "";
public images: Record<string, unknown>[] = [];
public request_time: number = Date.now();
public started_time: number | null = null;
public finished_time: number | null = null;
public processing_took: number | null = null;
public waiting_took: number | null = null;
public process: ChildProcessWithoutNullStreams;
constructor(public url: string, public domains: string[]) {
q.push(async () => {
return this.exec();
});
requests[this.id] = this;
queue_order.push(this);
}
getJobsAhead(): number {
if (this.status != "waiting") {
return 0;
}
let count = 0;
for (const request of queue_order) {
if (request == this) {
break;
}
count++;
}
return count;
}
async getJSON(): Promise<{
url: string;
domains: string[];
jobs_ahead: number;
id: string;
status: string;
output: string;
images: Record<string, unknown>[];
request_time: number;
started_time: number | null;
finished_time: number | null;
processing_took: number | null;
waiting_took: number | null;
elapsed_time_s: number;
}> {
return {
url: this.url,
domains: this.domains,
jobs_ahead: this.getJobsAhead(),
id: this.id,
status: this.status,
output: this.output,
images: this.images,
request_time: this.request_time,
started_time: this.started_time,
finished_time: this.finished_time,
processing_took: this.processing_took,
waiting_took: this.waiting_took,
elapsed_time_s: Math.round(
((this.status === "finished" ? this.finished_time || -1 : Date.now()) -
this.request_time) /
1000
),
};
}
setFinished(): void {
this.status = "finished";
this.finished_time = Date.now();
if (this.started_time) {
this.processing_took = this.finished_time - this.started_time;
this.waiting_took = this.started_time - this.request_time;
}
}
async exec(): Promise<void> {
this.started_time = Date.now();
this.status = "running";
const container = containerPool.getContainer();
await container.waitReady();
return new Promise((resolve, reject) => {
this.process = spawn(
"docker",
[
"exec",
container.id,
"/opt/run-analysis.sh",
JSON.stringify({
url: this.url,
third_party_domains: this.domains,
}),
this.id,
],
{ cwd: process.cwd() }
);
this.process.on("close", (exitCode) => {
this.setFinished();
container.close();
queue_order = queue_order.filter((request) => request != this);
if (exitCode === 0) {
resolve();
} else {
reject();
}
});
this.process.stdout.on("data", (d: Buffer) => {
try {
const parsed = JSON.parse(d.toString()) as unknown;
if (
is(parsed, predicates.object) &&
is(parsed.new_file, predicates.object)
) {
this.images.push(parsed.new_file);
}
} catch (e) {
//noop
}
this.output += d.toString();
/* console.log("DATA!", d.toString()); */
});
this.process.stderr.on("data", (d: Buffer) => {
this.output += d.toString();
/* console.log("STDERR!", d.toString()); */
});
});
}
}

28
tsconfig.json Normal file
View File

@ -0,0 +1,28 @@
{
"compilerOptions": {
"module": "commonjs",
"moduleResolution": "node",
"noImplicitAny": true,
"noImplicitThis": true,
"strictNullChecks": true,
"target": "ES6",
"declaration": true,
"esModuleInterop": true,
"lib": [
"ES6",
"ESNext"
],
"outDir": "lib",
"checkJs": true,
"allowJs": true,
"declarationDir": "@types",
"resolveJsonModule": true,
"sourceMap": true
},
"include": [
"src/**/*"
],
"exclude": [
"src/**/*.test.ts"
]
}