icd/rentgendroid
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

simulate screen dragging

Browse Source 
Summary: Ref T2890

Reviewers: #reviewers

Subscribers: kuba-orlik

Maniphest Tasks: T2890

Differential Revision: https://hub.sealcode.org/D1410
This commit is contained in:
Kuba Orlik 2025-01-18 16:43:36 +01:00
parent 2e0ea198f1
commit 0a6e04c5ba
4 changed files with 279 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
android/code/index.js
View File

@ -5,46 +5,51 @@ const fs = require("fs");
const server = net.createServer(); const server = net.createServer();
async function spawnPromise(program, args) { async function spawnPromise(program, args) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const process = child_process.spawn(program, args); const process = child_process.spawn(program, args);
process.on("close", (_) => { process.on("close", (_) => {
resolve(); resolve();
}); });
}); });
} }
//maybe check output of child processe and send errors in some way //maybe check output of child processe and send errors in some way
server.on("connection", (socket) => { server.on("connection", (socket) => {
socket.on("data", async (dataBuf) => { socket.on("data", async (dataBuf) => {
data = dataBuf.toString(); data = dataBuf.toString();
if (data === "screenshot") { if (data === "screenshot") {
socket.write("start"); socket.write("start");
await spawnPromise("bash", ["/conf/screenshot.sh"]); await spawnPromise("bash", ["/conf/screenshot.sh"]);
socket.write(fs.readFileSync("/screenshot.png")); socket.write(fs.readFileSync("/screenshot.png"));
socket.write("ENDOFMSG"); socket.write("ENDOFMSG");
} else if (data.includes("touch")) { } else if (data.includes("touch")) {
dataSplit = data.split(" "); const dataSplit = data.split(" ");
await spawnPromise("bash", [ await spawnPromise("bash", [
"/conf/touch.sh", "/conf/touch.sh",
dataSplit[1], dataSplit[1],
dataSplit[2], dataSplit[2],
]); ]);
} else if (data === "back") { } else if (data === "back") {
await spawnPromise("bash", [ await spawnPromise("bash", ["/conf/back.sh"]);
"/conf/back.sh", } else if (data === "home") {
]); await spawnPromise("bash", ["/conf/home.sh"]);
} else if (data === "home") { } else if (data.includes("drag")) {
await spawnPromise("bash", [ const dataSplit = data.split(" ");
"/conf/home.sh",
]); await spawnPromise("bash", [
} "/conf/drag.sh",
dataSplit[1],
}); dataSplit[2],
socket.on("close", (_) => { dataSplit[3],
socket.end(); dataSplit[4],
}); ]);
}
});
socket.on("close", (_) => {
socket.end();
});
}); });
server.listen(3000, () => { server.listen(3000, () => {
console.log("listening on 3000"); console.log("listening on 3000");
}); });

1
android/conf/drag.sh Normal file
View File

@ -0,0 +1 @@
/opt/android-sdk-linux/platform-tools/adb shell input swipe $1 $2 $3 $4 1000

289
http_server/code/index.html
View File

@ -1,151 +1,186 @@
<!DOCTYPE html> <!DOCTYPE html>
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<title>Rentgen android</title> <title>Rentgen android</title>
<style> <style>
main{ main {
display: flex; display: flex;
} }
.log-section{ .log-section {
height: 600px; height: 600px;
width: 300px; width: 480px;
overflow-y: auto; overflow-y: auto;
display: flex; display: flex;
flex-direction: column; flex-direction: column;
margin-left: 20px; margin-left: 20px;
} }
.screen{ .screen {
display: inline-block display: inline-block;
} cursor: pointer;
}
.screen-buttons{ .screen-buttons {
display: flex; display: flex;
justify-content: space-around; justify-content: space-around;
margin-top: 5px; margin-top: 5px;
gap: 10px; gap: 10px;
} }
.screen-buttons button{ .screen-buttons button {
font-size: 1.1rem; font-size: 1.1rem;
padding: 10px 20px; padding: 10px 20px;
width: 100%; width: 100%;
cursor: pointer; cursor: pointer;
background-color: transparent; background-color: transparent;
} }
.screen-buttons button:hover{ .screen-buttons button:hover {
background-color: aqua; background-color: aqua;
} }
</style>
</head>
<body>
<main>
<section class="screen-section">
<img id="screen" alt="android screen" src="" class="screen"/>
<div class="screen-buttons">
<button class="screen-buttons-home">home</button>
<button class="screen-buttons-back">back</button>
</div>
</section>
<p
id="clicks-log"
class="log-section"
></p>
<p
id="traffic-log"
class="log-section"
></p>
</main>
<script>
var screen = document.getElementById("screen");
var clicksLog = document.getElementById("clicks-log");
const homeButton = document.querySelector(".screen-buttons-home");
const backButton = document.querySelector(".screen-buttons-back");
let lastTouch = new Date().getTime(); #clicks-log {
font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console,
monospace;
}
</style>
</head>
<body>
<main>
<section class="screen-section">
<img
id="screen"
alt="android screen"
src=""
draggable="false"
class="screen"
/>
<div class="screen-buttons">
<button class="screen-buttons-home">home</button>
<button class="screen-buttons-back">back</button>
</div>
</section>
<p id="clicks-log" class="log-section"></p>
<p id="traffic-log" class="log-section"></p>
</main>
<script>
var screen = document.getElementById("screen");
var clicksLog = document.getElementById("clicks-log");
const homeButton = document.querySelector(".screen-buttons-home");
const backButton = document.querySelector(".screen-buttons-back");
const calculateElapsedTime = (last) => { let lastTouch = new Date().getTime();
const currentTouch = new Date().getTime();
const elapsedTime = currentTouch - lastTouch;
const elapsedSec = Math.round(elapsedTime / 1000);
lastTouch = currentTouch;
return elapsedSec;
};
const waitToLog = (clickInfoText) => { const calculateElapsedTime = (last) => {
const clickInfo = document.createElement("span"); const currentTouch = new Date().getTime();
const waitInfo = document.createElement("span"); const elapsedTime = currentTouch - lastTouch;
waitInfo.textContent = `await wait(${calculateElapsedTime(lastTouch)});` const elapsedSec = Math.round(elapsedTime / 1000);
clicksLog.appendChild(waitInfo); lastTouch = currentTouch;
clickInfo.textContent = clickInfoText; return elapsedSec;
clicksLog.appendChild(clickInfo); };
}
const registerClick = ({path, logText}) =>{ const waitToLog = (clickInfoText) => {
const clicksLog = document.getElementById("clicks-log"); const clickInfo = document.createElement("span");
const span = document.createElement("span"); const waitInfo = document.createElement("span");
waitInfo.textContent = `await wait(${calculateElapsedTime(
lastTouch
)});`;
clicksLog.appendChild(waitInfo);
clickInfo.textContent = clickInfoText;
clicksLog.appendChild(clickInfo);
};
waitToLog(logText); const registerClick = ({ path, logText, body }) => {
const clicksLog = document.getElementById("clicks-log");
const span = document.createElement("span");
fetch(path, { waitToLog(logText);
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
});
}
homeButton.addEventListener("click", () => registerClick({path: "home", logText: "await homeButton();"})); fetch(path, {
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
...(body ? { body } : {}),
});
};
backButton.addEventListener("click", () => registerClick({path: "back", logText: "await backButton();"})); homeButton.addEventListener("click", () =>
registerClick({ path: "home", logText: "await homeButton();" })
);
async function displayImage() { backButton.addEventListener("click", () =>
try { registerClick({ path: "back", logText: "await backButton();" })
const response = await fetch("screen"); );
const blob = await response.blob();
screen.src = URL.createObjectURL(blob);
} catch (error) {
console.error("Error fetching image: ", error);
}
}
async function handleTouchEvent(event) { async function displayImage() {
const phoneX = event.offsetX; try {
const phoneY = event.offsetY; const response = await fetch("screen");
const blob = await response.blob();
screen.src = URL.createObjectURL(blob);
} catch (error) {
console.error("Error fetching image: ", error);
}
}
waitToLog(`await click(${phoneX}, ${phoneY});`); let isDragging = false;
let startDraggingPosX = 0;
await fetch("touch", { let endDraggingPosX = 0;
method: "POST", let startDraggingPosY = 0;
headers: { let endDraggingPosY = 0;
"Content-Type": "application/x-www-form-urlencoded",
},
body: `x=${phoneX}&y=${phoneY}`,
});
}
async function sleep(time) { const handleDraggStart = (e) => {
return new Promise((resolve) => setTimeout(resolve, time)); e.preventDefault();
} isDragging = true;
startDraggingPosX = e.offsetX;
startDraggingPosY = e.offsetY;
};
async function screenshot_loop() { screen.addEventListener("mousedown", handleDraggStart);
var before;
while (true) { document.addEventListener("mouseup", (e) => {
before = performance.now(); endDraggingPosX = e.offsetX;
await displayImage(); endDraggingPosY = e.offsetY;
while (performance.now() - before < ___screenshotDelayMs___) if (
await sleep(50); (isDragging && Math.abs(endDraggingPosY - startDraggingPosY) > 10) ||
} Math.abs(endDraggingPosX - startDraggingPosX) > 10
} ) {
registerClick({
path: "drag",
logText: `await drag({x:${startDraggingPosX},y:${startDraggingPosY}},{x:${e.offsetX},y:${e.offsetY}});`,
body: `startX=${startDraggingPosX}&startY=${startDraggingPosY}&endX=${e.offsetX}&endY=${e.offsetY}`,
});
isDragging = false;
} else {
const phoneX = event.offsetX;
const phoneY = event.offsetY;
registerClick({
path: "touch",
logText: `await click(${phoneX}, ${phoneY});`,
body: `x=${phoneX}&y=${phoneY}`,
});
}
});
screen.addEventListener("click", handleTouchEvent); async function sleep(time) {
return new Promise((resolve) => setTimeout(resolve, time));
}
screenshot_loop(); async function screenshot_loop() {
</script> var before;
<script src="/trafficLog.js"></script>
</body> while (true) {
before = performance.now();
await displayImage();
while (performance.now() - before < ___screenshotDelayMs___)
await sleep(50);
}
}
screenshot_loop();
</script>
<script src="/trafficLog.js"></script>
</body>
</html> </html>

133
http_server/code/index.mjs
View File

@ -11,55 +11,64 @@ app.use(express.urlencoded({ extended: false }));
const socket_client = net.createConnection({ port: 3000, host: "android" }); const socket_client = net.createConnection({ port: 3000, host: "android" });
async function sleep(time) { async function sleep(time) {
return new Promise((resolve) => setTimeout(resolve, time)); return new Promise((resolve) => setTimeout(resolve, time));
} }
let doneWrite = 0; let doneWrite = 0;
let screenshotPromise = null; let screenshotPromise = null;
async function screenshot() { async function screenshot() {
socket_client.write("screenshot"); const time_start = Date.now();
while (!doneWrite) await sleep(15); socket_client.write("screenshot");
doneWrite = 0; while (!doneWrite) {
screenshotPromise = null; await sleep(15);
if (Date.now() - time_start > 2000) {
console.error("Screenshot timed out after 2s");
break; // timeout
}
}
doneWrite = 0;
screenshotPromise = null;
} }
async function guardedScreenshot() { async function guardedScreenshot() {
if (!screenshotPromise) { console.log("Requesting a screenshot");
screenshotPromise = screenshot(); if (!screenshotPromise) {
} console.log("no ongoing promise, starting a new one");
return screenshotPromise; screenshotPromise = screenshot();
}
return screenshotPromise;
} }
async function waitFullBoot() { async function waitFullBoot() {
var start = performance.now(); var start = performance.now();
var counter = 0; var counter = 0;
//will timeout after 10 min //will timeout after 10 min
while (performance.now() - start < 600 * 1000) { while (performance.now() - start < 600 * 1000) {
var before = performance.now(); var before = performance.now();
await screenshot(); await screenshot();
var after = performance.now(); var after = performance.now();
if (after - before < process.env.screenshotDelayMs) counter++; if (after - before < process.env.screenshotDelayMs) counter++;
else counter = 0; else counter = 0;
if (counter === 10) return; if (counter === 10) return;
} }
throw new Error("wait for screenshot time to be less than 0.5s timed out"); throw new Error("wait for screenshot time to be less than 0.5s timed out");
} }
let fd; let fd;
socket_client.on("data", (dataBuf) => { socket_client.on("data", (dataBuf) => {
if (dataBuf.toString() === "start") if (dataBuf.toString() === "start")
fd = fs.openSync("/code/screenshot.png", "w"); fd = fs.openSync("/code/screenshot.png", "w");
else { else {
if (dataBuf.toString().includes("ENDOFMSG")) { if (dataBuf.toString().includes("ENDOFMSG")) {
fs.writeSync(fd, dataBuf); fs.writeSync(fd, dataBuf);
fs.close(fd); fs.close(fd);
doneWrite = 1; doneWrite = 1;
} else fs.writeSync(fd, dataBuf); } else fs.writeSync(fd, dataBuf);
} }
}); });
console.log("Waiting for full boot..."); console.log("Waiting for full boot...");
@ -67,58 +76,68 @@ await waitFullBoot();
console.log("Boot detected! activating endpoints"); console.log("Boot detected! activating endpoints");
app.get("/screen", async function (req, res) { app.get("/screen", async function (req, res) {
await guardedScreenshot(); await guardedScreenshot();
res.sendFile("/code/screenshot.png"); res.sendFile("/code/screenshot.png");
}); });
app.get("/favicon.ico", function (req, res) { app.get("/favicon.ico", function (req, res) {
res.sendFile("/code/favicon.ico"); res.sendFile("/code/favicon.ico");
}); });
app.get("/trafficLog.js", function (req, res) { app.get("/trafficLog.js", function (req, res) {
res.sendFile("/code/dist/trafficLog.js"); res.sendFile("/code/dist/trafficLog.js");
}); });
app.get("/trafficLog", async function (req, res) { app.get("/trafficLog", async function (req, res) {
res.sendFile("/log/trafficLog"); res.sendFile("/log/trafficLog");
}); });
app.post("/touch", function (req, res) { app.post("/touch", function (req, res) {
const x = parseInt(req.body.x); const x = parseInt(req.body.x);
const y = parseInt(req.body.y); const y = parseInt(req.body.y);
if (isNaN(x) || isNaN(y) || x > device_size_x || y > device_size_y) { if (isNaN(x) || isNaN(y) || x > device_size_x || y > device_size_y) {
res.send( res.send(
`the query params must be x <= ${device_size_x}, y <= ${device_size_y}\n` `the query params must be x <= ${device_size_x}, y <= ${device_size_y}\n`
); );
} else { } else {
socket_client.write(`touch ${x} ${y}`); socket_client.write(`touch ${x} ${y}`);
res.sendStatus(200); res.sendStatus(200);
} }
}); });
app.get("/", async function (req, res) { app.get("/", async function (req, res) {
let fileData = (await readFile("/code/index.html")).toString(); let fileData = (await readFile("/code/index.html")).toString();
fileData = fileData.replace( fileData = fileData.replace(
"___screenshotDelayMs___", "___screenshotDelayMs___",
process.env.screenshotDelayMs process.env.screenshotDelayMs
); );
res.setHeader("Content-Type", "text/html"); res.setHeader("Content-Type", "text/html");
res.setHeader("Content-Disposition", "inline"); res.setHeader("Content-Disposition", "inline");
res.send(fileData); res.send(fileData);
}); });
app.post("/back", function (req, res) { app.post("/back", function (req, res) {
socket_client.write(`back`); socket_client.write(`back`);
res.sendStatus(200); res.sendStatus(200);
}); });
app.post("/home", function (req, res) { app.post("/home", function (req, res) {
socket_client.write(`home`); socket_client.write(`home`);
res.sendStatus(200); res.sendStatus(200);
});
app.post("/drag", function (req, res) {
const body = req.body;
const startX = Number(body.startX);
const startY = Number(body.startY);
const endX = Number(body.endX);
const endY = Number(body.endY);
socket_client.write(`drag ${startX} ${startY} ${endX} ${endY}`);
res.sendStatus(200);
}); });
app.listen(8080, () => console.log("Listening in port 8080")); app.listen(8080, () => console.log("Listening in port 8080"));