Add a webservice API with polling

This commit is contained in:
Kuba Orlik 2022-05-05 21:54:34 +02:00
parent 54ed7308a8
commit 4c34db1af9
11 changed files with 169 additions and 22 deletions

View File

@ -12,15 +12,19 @@ get_width(){
annotate_header(){
echo annotate $1
echo annotate $1 $2 $3
filename=$1
shift;
domain=$1;
shift;
count=$1;
shift;
d=$(date "+%Y-%m-%d__%H_%M_%S")
#filename="2022-03-10__19_33_55__set-cookie.png"
cropped_filename="${filename}__cropped.png"
overlay_filename="${cropped_filename}__overlay.png"
hardoverlay_filename="${cropped_filename}__hardoverlay.png"
annotated_filename="${cropped_filename}__annotated.png"
annotated_filename="$(dirname "$filename")/${domain}__${count}.final.png" # the name is crucial, because the web app part filters files based on the name
# crop
left=2056
@ -78,10 +82,10 @@ annotate_header(){
shift
shift
done
convert "$filename" "$overlay_filename" -compose Darken -composite "$annotated_filename.step.png"
convert "$annotated_filename.step.png" "$hardoverlay_filename" -compose src-over -composite "$annotated_filename"
convert "$filename" "$overlay_filename" -compose Darken -composite "${annotated_filename}.step.png"
convert "${annotated_filename}.step.png" "$hardoverlay_filename" -compose src-over -composite "$annotated_filename"
rm "$overlay_filename" "$annotated_filename.step.png" "$hardoverlay_filename" "$cropped_filename" "$filename"
echo "SCREENSHOT: <img width=\"720\" height=\"405\" src=\"$BASE_URL/$(echo "$annotated_filename" | sed 's|/opt/||')\"/>"
echo "SCREENSHOT: <img width=\"720\" height=\"405\" src=\"${BASE_URL}/$(echo "$annotated_filename" | sed 's|/opt/||')\"/>"
}
#annotate_header "set-cookie" "identyfikator internetowy z cookie" "Cookie" "identyfikator internetowy z cookie"

View File

@ -14,11 +14,11 @@ source ./ephemeral-x.sh
source ./annotate_header.sh
source ./utils.sh
(while true; do
grab_screen_to_public $ID
sleep 1
done) &
refresher_pid=$!;
# (while true; do
# grab_screen_to_public $ID
# sleep 1
# done) &
# refresher_pid=$!;
start_firefox
grab start_firefox
@ -53,7 +53,7 @@ while IFS= read -r DOMAIN; do
do
filename="/opt/static/$ID/${index}.png"
scrot "$filename"
annotate_header "$filename" \
annotate_header "$filename" "$DOMAIN" "$count" \
"set-cookie" "identyfikator internetowy z cookie" \
"Cookie" "identyfikator internetowy z cookie" \
"Referer" "Część mojej historii przeglądania" &
@ -67,7 +67,7 @@ while IFS= read -r DOMAIN; do
done
done <<< "$DOMAINS"
kill $refresher_pid;
# kill $refresher_pid;
echo "starting wait..."

View File

@ -28,7 +28,7 @@ click (){
grab_no=0
grab (){
grab(){
NOTE="$grab_no $@"
#echo $@
((grab_no++))

View File

@ -23,3 +23,13 @@ BASE_URL=http://localhost:3000 docker run -i -v $PWD/static:/opt/static headles
```
BASE_URL=http://localhost:3000 node .
```
## Testing the API
with httpie:
```
http POST localhost:3000/api/requests url==pearson.com 'domains[]==youtube.com' 'domains[]==facebook.com'
```
It returns a 303 response that redirects to a URL you can poll to see the status of the screenshot request.

10
docker-args.js Normal file
View File

@ -0,0 +1,10 @@
const DOCKER_ARGS = [
"run",
"-i",
"-v",
`${process.cwd()}/static:/opt/static`,
"headless-fox",
"./script3.sh",
];
module.exports = DOCKER_ARGS;

View File

@ -1,4 +1,3 @@
const { v4: uuid } = require("uuid");
var serve = require("koa-static");
const Koa = require("koa");
const Router = require("@koa/router");
@ -6,6 +5,9 @@ const mount = require("koa-mount");
const qs = require("qs");
const { Readable } = require("stream");
const { spawn } = require("child_process");
const { requests } = require("./memory");
const ScreenshotRequest = require("./request");
const DOCKER_ARGS = require("./docker-args");
const router = new Router();
@ -65,13 +67,7 @@ router.get("/", async (ctx) => {
const starter = spawn(
"docker",
[
"run",
"-i",
"-d",
"-v",
`${process.cwd()}/static:/opt/static`,
"headless-fox",
"./script3.sh",
...DOCKER_ARGS,
`{"url": "${params.url}", "third_party_domains": ["hotjar.com", "cookielaw.org", "facebook.com", "gemius.pl"]}`,
id,
],
@ -87,5 +83,36 @@ router.get("/", async (ctx) => {
});
});
router.post("/api/requests", async (ctx) => {
const params = qs.parse(ctx.querystring);
if (!params.url) {
ctx.body = "Specify url";
ctx.status = 422;
return;
}
if (!params.domains) {
ctx.body = "Specify domains";
ctx.status = 422;
return;
}
if (!Array.isArray(params.domains)) {
ctx.body = "'domains' should be an array of strings";
ctx.status = 422;
return;
}
const request = new ScreenshotRequest(params.url, params.domains);
ctx.status = 303;
ctx.redirect(`/api/requests/${request.id}`);
});
router.get("/api/requests/:id", async (ctx) => {
const request = requests[ctx.params.id];
if (!request) {
ctx.status = 404;
return;
}
ctx.body = await request.getJSON();
});
app.use(router.routes()).use(router.allowedMethods());
app.listen(3000);

3
memory.js Normal file
View File

@ -0,0 +1,3 @@
const queue = require("queue");
const q = queue({ concurrency: 1, autostart: true, results: [] });
b;

17
package-lock.json generated
View File

@ -15,6 +15,7 @@
"koa-mount": "^4.0.0",
"koa-static": "^5.0.0",
"qs": "^6.10.3",
"queue": "^6.0.2",
"uuid": "^8.3.2"
}
},
@ -571,6 +572,14 @@
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
},
"node_modules/queue": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz",
"integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==",
"dependencies": {
"inherits": "~2.0.3"
}
},
"node_modules/readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
@ -1167,6 +1176,14 @@
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
},
"queue": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz",
"integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==",
"requires": {
"inherits": "~2.0.3"
}
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",

View File

@ -19,6 +19,7 @@
"koa-mount": "^4.0.0",
"koa-static": "^5.0.0",
"qs": "^6.10.3",
"queue": "^6.0.2",
"uuid": "^8.3.2"
}
}

75
request.js Normal file
View File

@ -0,0 +1,75 @@
const { q, requests } = require("./memory");
const DOCKER_ARGS = require("./docker-args");
const { v4: uuid } = require("uuid");
const { promises: fs } = require("fs");
const { spawn } = require("child_process");
const { resolve } = require("path");
module.exports = class ScreenshotRequest {
constructor(url, domains) {
this.url = url;
this.domains = domains;
this.id = uuid();
this.status = "waiting";
this.output = "";
this.images = [];
q.push(async () => {
return this.exec();
});
requests[this.id] = this;
}
async getImages() {
try {
const files = await fs.readdir(resolve(__dirname, "./static/" + this.id));
return files.filter((file) => file.match(/.final.png$/));
} catch (e) {
return [];
}
}
async getJSON() {
return {
url: this.url,
domains: this.domains,
id: this.id,
status: this.status,
/* output: this.output, */
files: await this.getImages(),
};
}
async exec() {
return new Promise((resolve, reject) => {
this.status = "running";
this.process = spawn(
"docker",
[
...DOCKER_ARGS,
JSON.stringify({
url: this.url,
third_party_domains: this.domains,
}),
this.id,
],
{ cwd: process.cwd() }
);
this.process.on("close", (exitCode) => {
this.status = "finished";
if (exitCode === 0) {
resolve();
} else {
reject();
}
});
this.process.stdout.on("data", (d) => {
this.output += d.toString();
/* console.log("DATA!", d.toString()); */
});
this.process.stderr.on("data", (d) => {
this.output += d.toString();
/* console.log("STDERR!", d.toString()); */
});
});
}
};