simulate screen dragging
Summary: Ref T2890 Reviewers: #reviewers Subscribers: kuba-orlik Maniphest Tasks: T2890 Differential Revision: https://hub.sealcode.org/D1410
This commit is contained in:
parent
2e0ea198f1
commit
0a6e04c5ba
@ -5,46 +5,51 @@ const fs = require("fs");
|
||||
const server = net.createServer();
|
||||
|
||||
async function spawnPromise(program, args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const process = child_process.spawn(program, args);
|
||||
process.on("close", (_) => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
const process = child_process.spawn(program, args);
|
||||
process.on("close", (_) => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
//maybe check output of child processe and send errors in some way
|
||||
server.on("connection", (socket) => {
|
||||
socket.on("data", async (dataBuf) => {
|
||||
data = dataBuf.toString();
|
||||
if (data === "screenshot") {
|
||||
socket.write("start");
|
||||
await spawnPromise("bash", ["/conf/screenshot.sh"]);
|
||||
socket.write(fs.readFileSync("/screenshot.png"));
|
||||
socket.write("ENDOFMSG");
|
||||
} else if (data.includes("touch")) {
|
||||
dataSplit = data.split(" ");
|
||||
await spawnPromise("bash", [
|
||||
"/conf/touch.sh",
|
||||
dataSplit[1],
|
||||
dataSplit[2],
|
||||
]);
|
||||
} else if (data === "back") {
|
||||
await spawnPromise("bash", [
|
||||
"/conf/back.sh",
|
||||
]);
|
||||
} else if (data === "home") {
|
||||
await spawnPromise("bash", [
|
||||
"/conf/home.sh",
|
||||
]);
|
||||
}
|
||||
socket.on("data", async (dataBuf) => {
|
||||
data = dataBuf.toString();
|
||||
if (data === "screenshot") {
|
||||
socket.write("start");
|
||||
await spawnPromise("bash", ["/conf/screenshot.sh"]);
|
||||
socket.write(fs.readFileSync("/screenshot.png"));
|
||||
socket.write("ENDOFMSG");
|
||||
} else if (data.includes("touch")) {
|
||||
const dataSplit = data.split(" ");
|
||||
await spawnPromise("bash", [
|
||||
"/conf/touch.sh",
|
||||
dataSplit[1],
|
||||
dataSplit[2],
|
||||
]);
|
||||
} else if (data === "back") {
|
||||
await spawnPromise("bash", ["/conf/back.sh"]);
|
||||
} else if (data === "home") {
|
||||
await spawnPromise("bash", ["/conf/home.sh"]);
|
||||
} else if (data.includes("drag")) {
|
||||
const dataSplit = data.split(" ");
|
||||
|
||||
});
|
||||
socket.on("close", (_) => {
|
||||
socket.end();
|
||||
});
|
||||
await spawnPromise("bash", [
|
||||
"/conf/drag.sh",
|
||||
dataSplit[1],
|
||||
dataSplit[2],
|
||||
dataSplit[3],
|
||||
dataSplit[4],
|
||||
]);
|
||||
}
|
||||
});
|
||||
socket.on("close", (_) => {
|
||||
socket.end();
|
||||
});
|
||||
});
|
||||
|
||||
server.listen(3000, () => {
|
||||
console.log("listening on 3000");
|
||||
console.log("listening on 3000");
|
||||
});
|
||||
|
||||
1
android/conf/drag.sh
Normal file
1
android/conf/drag.sh
Normal file
@ -0,0 +1 @@
|
||||
/opt/android-sdk-linux/platform-tools/adb shell input swipe $1 $2 $3 $4 1000
|
||||
@ -1,151 +1,186 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Rentgen android</title>
|
||||
<style>
|
||||
main{
|
||||
display: flex;
|
||||
}
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Rentgen android</title>
|
||||
<style>
|
||||
main {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.log-section{
|
||||
height: 600px;
|
||||
width: 300px;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
margin-left: 20px;
|
||||
}
|
||||
.log-section {
|
||||
height: 600px;
|
||||
width: 480px;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
margin-left: 20px;
|
||||
}
|
||||
|
||||
.screen{
|
||||
display: inline-block
|
||||
}
|
||||
.screen {
|
||||
display: inline-block;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.screen-buttons{
|
||||
display: flex;
|
||||
justify-content: space-around;
|
||||
margin-top: 5px;
|
||||
gap: 10px;
|
||||
}
|
||||
.screen-buttons {
|
||||
display: flex;
|
||||
justify-content: space-around;
|
||||
margin-top: 5px;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.screen-buttons button{
|
||||
font-size: 1.1rem;
|
||||
padding: 10px 20px;
|
||||
width: 100%;
|
||||
cursor: pointer;
|
||||
background-color: transparent;
|
||||
}
|
||||
.screen-buttons button {
|
||||
font-size: 1.1rem;
|
||||
padding: 10px 20px;
|
||||
width: 100%;
|
||||
cursor: pointer;
|
||||
background-color: transparent;
|
||||
}
|
||||
|
||||
.screen-buttons button:hover{
|
||||
background-color: aqua;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<section class="screen-section">
|
||||
<img id="screen" alt="android screen" src="" class="screen"/>
|
||||
<div class="screen-buttons">
|
||||
<button class="screen-buttons-home">home</button>
|
||||
<button class="screen-buttons-back">back</button>
|
||||
</div>
|
||||
</section>
|
||||
<p
|
||||
id="clicks-log"
|
||||
class="log-section"
|
||||
></p>
|
||||
<p
|
||||
id="traffic-log"
|
||||
class="log-section"
|
||||
></p>
|
||||
</main>
|
||||
<script>
|
||||
var screen = document.getElementById("screen");
|
||||
var clicksLog = document.getElementById("clicks-log");
|
||||
const homeButton = document.querySelector(".screen-buttons-home");
|
||||
const backButton = document.querySelector(".screen-buttons-back");
|
||||
.screen-buttons button:hover {
|
||||
background-color: aqua;
|
||||
}
|
||||
|
||||
let lastTouch = new Date().getTime();
|
||||
#clicks-log {
|
||||
font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console,
|
||||
monospace;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<section class="screen-section">
|
||||
<img
|
||||
id="screen"
|
||||
alt="android screen"
|
||||
src=""
|
||||
draggable="false"
|
||||
class="screen"
|
||||
/>
|
||||
<div class="screen-buttons">
|
||||
<button class="screen-buttons-home">home</button>
|
||||
<button class="screen-buttons-back">back</button>
|
||||
</div>
|
||||
</section>
|
||||
<p id="clicks-log" class="log-section"></p>
|
||||
<p id="traffic-log" class="log-section"></p>
|
||||
</main>
|
||||
<script>
|
||||
var screen = document.getElementById("screen");
|
||||
var clicksLog = document.getElementById("clicks-log");
|
||||
const homeButton = document.querySelector(".screen-buttons-home");
|
||||
const backButton = document.querySelector(".screen-buttons-back");
|
||||
|
||||
const calculateElapsedTime = (last) => {
|
||||
const currentTouch = new Date().getTime();
|
||||
const elapsedTime = currentTouch - lastTouch;
|
||||
const elapsedSec = Math.round(elapsedTime / 1000);
|
||||
lastTouch = currentTouch;
|
||||
return elapsedSec;
|
||||
};
|
||||
let lastTouch = new Date().getTime();
|
||||
|
||||
const waitToLog = (clickInfoText) => {
|
||||
const clickInfo = document.createElement("span");
|
||||
const waitInfo = document.createElement("span");
|
||||
waitInfo.textContent = `await wait(${calculateElapsedTime(lastTouch)});`
|
||||
clicksLog.appendChild(waitInfo);
|
||||
clickInfo.textContent = clickInfoText;
|
||||
clicksLog.appendChild(clickInfo);
|
||||
}
|
||||
const calculateElapsedTime = (last) => {
|
||||
const currentTouch = new Date().getTime();
|
||||
const elapsedTime = currentTouch - lastTouch;
|
||||
const elapsedSec = Math.round(elapsedTime / 1000);
|
||||
lastTouch = currentTouch;
|
||||
return elapsedSec;
|
||||
};
|
||||
|
||||
const registerClick = ({path, logText}) =>{
|
||||
const clicksLog = document.getElementById("clicks-log");
|
||||
const span = document.createElement("span");
|
||||
const waitToLog = (clickInfoText) => {
|
||||
const clickInfo = document.createElement("span");
|
||||
const waitInfo = document.createElement("span");
|
||||
waitInfo.textContent = `await wait(${calculateElapsedTime(
|
||||
lastTouch
|
||||
)});`;
|
||||
clicksLog.appendChild(waitInfo);
|
||||
clickInfo.textContent = clickInfoText;
|
||||
clicksLog.appendChild(clickInfo);
|
||||
};
|
||||
|
||||
waitToLog(logText);
|
||||
const registerClick = ({ path, logText, body }) => {
|
||||
const clicksLog = document.getElementById("clicks-log");
|
||||
const span = document.createElement("span");
|
||||
|
||||
fetch(path, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
},
|
||||
});
|
||||
}
|
||||
waitToLog(logText);
|
||||
|
||||
homeButton.addEventListener("click", () => registerClick({path: "home", logText: "await homeButton();"}));
|
||||
fetch(path, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
},
|
||||
...(body ? { body } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
backButton.addEventListener("click", () => registerClick({path: "back", logText: "await backButton();"}));
|
||||
homeButton.addEventListener("click", () =>
|
||||
registerClick({ path: "home", logText: "await homeButton();" })
|
||||
);
|
||||
|
||||
async function displayImage() {
|
||||
try {
|
||||
const response = await fetch("screen");
|
||||
const blob = await response.blob();
|
||||
screen.src = URL.createObjectURL(blob);
|
||||
} catch (error) {
|
||||
console.error("Error fetching image: ", error);
|
||||
}
|
||||
}
|
||||
backButton.addEventListener("click", () =>
|
||||
registerClick({ path: "back", logText: "await backButton();" })
|
||||
);
|
||||
|
||||
async function handleTouchEvent(event) {
|
||||
const phoneX = event.offsetX;
|
||||
const phoneY = event.offsetY;
|
||||
async function displayImage() {
|
||||
try {
|
||||
const response = await fetch("screen");
|
||||
const blob = await response.blob();
|
||||
screen.src = URL.createObjectURL(blob);
|
||||
} catch (error) {
|
||||
console.error("Error fetching image: ", error);
|
||||
}
|
||||
}
|
||||
|
||||
waitToLog(`await click(${phoneX}, ${phoneY});`);
|
||||
let isDragging = false;
|
||||
let startDraggingPosX = 0;
|
||||
let endDraggingPosX = 0;
|
||||
let startDraggingPosY = 0;
|
||||
let endDraggingPosY = 0;
|
||||
|
||||
await fetch("touch", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
},
|
||||
body: `x=${phoneX}&y=${phoneY}`,
|
||||
});
|
||||
}
|
||||
const handleDraggStart = (e) => {
|
||||
e.preventDefault();
|
||||
isDragging = true;
|
||||
startDraggingPosX = e.offsetX;
|
||||
startDraggingPosY = e.offsetY;
|
||||
};
|
||||
|
||||
async function sleep(time) {
|
||||
return new Promise((resolve) => setTimeout(resolve, time));
|
||||
}
|
||||
screen.addEventListener("mousedown", handleDraggStart);
|
||||
|
||||
async function screenshot_loop() {
|
||||
var before;
|
||||
document.addEventListener("mouseup", (e) => {
|
||||
endDraggingPosX = e.offsetX;
|
||||
endDraggingPosY = e.offsetY;
|
||||
if (
|
||||
(isDragging && Math.abs(endDraggingPosY - startDraggingPosY) > 10) ||
|
||||
Math.abs(endDraggingPosX - startDraggingPosX) > 10
|
||||
) {
|
||||
registerClick({
|
||||
path: "drag",
|
||||
logText: `await drag({x:${startDraggingPosX},y:${startDraggingPosY}},{x:${e.offsetX},y:${e.offsetY}});`,
|
||||
body: `startX=${startDraggingPosX}&startY=${startDraggingPosY}&endX=${e.offsetX}&endY=${e.offsetY}`,
|
||||
});
|
||||
isDragging = false;
|
||||
} else {
|
||||
const phoneX = event.offsetX;
|
||||
const phoneY = event.offsetY;
|
||||
registerClick({
|
||||
path: "touch",
|
||||
logText: `await click(${phoneX}, ${phoneY});`,
|
||||
body: `x=${phoneX}&y=${phoneY}`,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
while (true) {
|
||||
before = performance.now();
|
||||
await displayImage();
|
||||
while (performance.now() - before < ___screenshotDelayMs___)
|
||||
await sleep(50);
|
||||
}
|
||||
}
|
||||
async function sleep(time) {
|
||||
return new Promise((resolve) => setTimeout(resolve, time));
|
||||
}
|
||||
|
||||
screen.addEventListener("click", handleTouchEvent);
|
||||
async function screenshot_loop() {
|
||||
var before;
|
||||
|
||||
screenshot_loop();
|
||||
</script>
|
||||
<script src="/trafficLog.js"></script>
|
||||
</body>
|
||||
while (true) {
|
||||
before = performance.now();
|
||||
await displayImage();
|
||||
while (performance.now() - before < ___screenshotDelayMs___)
|
||||
await sleep(50);
|
||||
}
|
||||
}
|
||||
|
||||
screenshot_loop();
|
||||
</script>
|
||||
<script src="/trafficLog.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@ -11,55 +11,64 @@ app.use(express.urlencoded({ extended: false }));
|
||||
const socket_client = net.createConnection({ port: 3000, host: "android" });
|
||||
|
||||
async function sleep(time) {
|
||||
return new Promise((resolve) => setTimeout(resolve, time));
|
||||
return new Promise((resolve) => setTimeout(resolve, time));
|
||||
}
|
||||
|
||||
let doneWrite = 0;
|
||||
let screenshotPromise = null;
|
||||
|
||||
async function screenshot() {
|
||||
socket_client.write("screenshot");
|
||||
while (!doneWrite) await sleep(15);
|
||||
doneWrite = 0;
|
||||
screenshotPromise = null;
|
||||
const time_start = Date.now();
|
||||
socket_client.write("screenshot");
|
||||
while (!doneWrite) {
|
||||
await sleep(15);
|
||||
if (Date.now() - time_start > 2000) {
|
||||
console.error("Screenshot timed out after 2s");
|
||||
break; // timeout
|
||||
}
|
||||
}
|
||||
doneWrite = 0;
|
||||
screenshotPromise = null;
|
||||
}
|
||||
|
||||
async function guardedScreenshot() {
|
||||
if (!screenshotPromise) {
|
||||
screenshotPromise = screenshot();
|
||||
}
|
||||
return screenshotPromise;
|
||||
console.log("Requesting a screenshot");
|
||||
if (!screenshotPromise) {
|
||||
console.log("no ongoing promise, starting a new one");
|
||||
screenshotPromise = screenshot();
|
||||
}
|
||||
return screenshotPromise;
|
||||
}
|
||||
|
||||
async function waitFullBoot() {
|
||||
var start = performance.now();
|
||||
var counter = 0;
|
||||
var start = performance.now();
|
||||
var counter = 0;
|
||||
|
||||
//will timeout after 10 min
|
||||
while (performance.now() - start < 600 * 1000) {
|
||||
var before = performance.now();
|
||||
await screenshot();
|
||||
var after = performance.now();
|
||||
if (after - before < process.env.screenshotDelayMs) counter++;
|
||||
else counter = 0;
|
||||
//will timeout after 10 min
|
||||
while (performance.now() - start < 600 * 1000) {
|
||||
var before = performance.now();
|
||||
await screenshot();
|
||||
var after = performance.now();
|
||||
if (after - before < process.env.screenshotDelayMs) counter++;
|
||||
else counter = 0;
|
||||
|
||||
if (counter === 10) return;
|
||||
}
|
||||
if (counter === 10) return;
|
||||
}
|
||||
|
||||
throw new Error("wait for screenshot time to be less than 0.5s timed out");
|
||||
throw new Error("wait for screenshot time to be less than 0.5s timed out");
|
||||
}
|
||||
|
||||
let fd;
|
||||
socket_client.on("data", (dataBuf) => {
|
||||
if (dataBuf.toString() === "start")
|
||||
fd = fs.openSync("/code/screenshot.png", "w");
|
||||
else {
|
||||
if (dataBuf.toString().includes("ENDOFMSG")) {
|
||||
fs.writeSync(fd, dataBuf);
|
||||
fs.close(fd);
|
||||
doneWrite = 1;
|
||||
} else fs.writeSync(fd, dataBuf);
|
||||
}
|
||||
if (dataBuf.toString() === "start")
|
||||
fd = fs.openSync("/code/screenshot.png", "w");
|
||||
else {
|
||||
if (dataBuf.toString().includes("ENDOFMSG")) {
|
||||
fs.writeSync(fd, dataBuf);
|
||||
fs.close(fd);
|
||||
doneWrite = 1;
|
||||
} else fs.writeSync(fd, dataBuf);
|
||||
}
|
||||
});
|
||||
|
||||
console.log("Waiting for full boot...");
|
||||
@ -67,58 +76,68 @@ await waitFullBoot();
|
||||
console.log("Boot detected! activating endpoints");
|
||||
|
||||
app.get("/screen", async function (req, res) {
|
||||
await guardedScreenshot();
|
||||
res.sendFile("/code/screenshot.png");
|
||||
await guardedScreenshot();
|
||||
res.sendFile("/code/screenshot.png");
|
||||
});
|
||||
|
||||
app.get("/favicon.ico", function (req, res) {
|
||||
res.sendFile("/code/favicon.ico");
|
||||
res.sendFile("/code/favicon.ico");
|
||||
});
|
||||
|
||||
app.get("/trafficLog.js", function (req, res) {
|
||||
res.sendFile("/code/dist/trafficLog.js");
|
||||
res.sendFile("/code/dist/trafficLog.js");
|
||||
});
|
||||
|
||||
app.get("/trafficLog", async function (req, res) {
|
||||
res.sendFile("/log/trafficLog");
|
||||
res.sendFile("/log/trafficLog");
|
||||
});
|
||||
|
||||
app.post("/touch", function (req, res) {
|
||||
const x = parseInt(req.body.x);
|
||||
const y = parseInt(req.body.y);
|
||||
const x = parseInt(req.body.x);
|
||||
const y = parseInt(req.body.y);
|
||||
|
||||
if (isNaN(x) || isNaN(y) || x > device_size_x || y > device_size_y) {
|
||||
res.send(
|
||||
`the query params must be x <= ${device_size_x}, y <= ${device_size_y}\n`
|
||||
);
|
||||
} else {
|
||||
socket_client.write(`touch ${x} ${y}`);
|
||||
res.sendStatus(200);
|
||||
}
|
||||
if (isNaN(x) || isNaN(y) || x > device_size_x || y > device_size_y) {
|
||||
res.send(
|
||||
`the query params must be x <= ${device_size_x}, y <= ${device_size_y}\n`
|
||||
);
|
||||
} else {
|
||||
socket_client.write(`touch ${x} ${y}`);
|
||||
res.sendStatus(200);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/", async function (req, res) {
|
||||
let fileData = (await readFile("/code/index.html")).toString();
|
||||
let fileData = (await readFile("/code/index.html")).toString();
|
||||
|
||||
fileData = fileData.replace(
|
||||
"___screenshotDelayMs___",
|
||||
process.env.screenshotDelayMs
|
||||
);
|
||||
fileData = fileData.replace(
|
||||
"___screenshotDelayMs___",
|
||||
process.env.screenshotDelayMs
|
||||
);
|
||||
|
||||
res.setHeader("Content-Type", "text/html");
|
||||
res.setHeader("Content-Disposition", "inline");
|
||||
res.setHeader("Content-Type", "text/html");
|
||||
res.setHeader("Content-Disposition", "inline");
|
||||
|
||||
res.send(fileData);
|
||||
res.send(fileData);
|
||||
});
|
||||
|
||||
app.post("/back", function (req, res) {
|
||||
socket_client.write(`back`);
|
||||
res.sendStatus(200);
|
||||
socket_client.write(`back`);
|
||||
res.sendStatus(200);
|
||||
});
|
||||
|
||||
app.post("/home", function (req, res) {
|
||||
socket_client.write(`home`);
|
||||
res.sendStatus(200);
|
||||
socket_client.write(`home`);
|
||||
res.sendStatus(200);
|
||||
});
|
||||
|
||||
app.post("/drag", function (req, res) {
|
||||
const body = req.body;
|
||||
const startX = Number(body.startX);
|
||||
const startY = Number(body.startY);
|
||||
const endX = Number(body.endX);
|
||||
const endY = Number(body.endY);
|
||||
socket_client.write(`drag ${startX} ${startY} ${endX} ${endY}`);
|
||||
res.sendStatus(200);
|
||||
});
|
||||
|
||||
app.listen(8080, () => console.log("Listening in port 8080"));
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user