-
Notifications
You must be signed in to change notification settings - Fork 4
Expand file tree
/
Copy pathactions.js
More file actions
114 lines (106 loc) · 3.35 KB
/
actions.js
File metadata and controls
114 lines (106 loc) · 3.35 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
const keyMap = {
ENTER: "Enter",
ESC: "Escape",
ARROWLEFT: "ArrowLeft",
ARROWRIGHT: "ArrowRight",
ARROWUP: "ArrowUp",
ARROWDOWN: "ArrowDown",
ALT: "Alt",
CTRL: "Control",
SHIFT: "Shift",
CMD: "Meta" // macOS Command key
};
const modifierKeys = new Set(["Control", "Shift", "Alt", "Meta"]);
export async function handleModelAction(page, action) {
try {
const { x, y, button, path, scroll_x, scroll_y, text, keys, url } = action;
switch (action.type) {
case "click":
console.log(`Clicking at (${x}, ${y}), ${button} button`);
await page.mouse.click(x, y);
break;
case "double_click":
console.log(`Double clicking at (${x}, ${y})`);
await page.mouse.dblclick(x, y);
break;
case "move":
console.log(`Moving mouse to (${x}, ${y})`);
await page.mouse.move(x, y);
break;
case "drag":
console.log("Dragging along path", path);
if (Array.isArray(path) && path.length > 0) {
const [firstPoint, ...restPoints] = path;
await page.mouse.move(firstPoint.x, firstPoint.y);
await page.mouse.down();
for (const point of restPoints) {
await page.mouse.move(point.x, point.y);
}
await page.mouse.up();
} else {
console.log("Drag action missing a valid path");
}
break;
case "scroll":
console.log(`Scrolling by (${scroll_x}, ${scroll_y})`);
await page.mouse.wheel(scroll_x, scroll_y);
break;
case "type":
console.log(`Typing text: ${text}`);
await page.keyboard.type(text);
break;
case "keypress":
console.log(`Pressing key: ${keys}`);
const mappedKeys = keys.map(key => keyMap[key.toUpperCase()] || key);
const modifiers = mappedKeys.filter(key => modifierKeys.has(key));
const normalKeys = mappedKeys.filter(key => !modifierKeys.has(key));
if (
(mappedKeys[0] === "Meta" && mappedKeys[1] === "[") ||
(mappedKeys[0] === "Alt" && mappedKeys[1] === "ArrowLeft")
) {
await page.goBack();
break;
}
// Hold down modifier keys
for (const key of modifiers) {
await page.keyboard.down(key);
}
// Press normal keys
for (const key of normalKeys) {
await page.keyboard.press(key);
}
// Release modifier keys
for (const key of modifiers) {
await page.keyboard.up(key);
}
break;
case "wait":
console.log("Waiting for browser...");
await page.waitForTimeout(1000);
break;
case "goto":
console.log(`Navigating to ${url}`);
await page.goto(url);
break;
case "back":
console.log("Navigating back");
await page.goBack();
break;
case "forward":
console.log("Navigating forward");
await page.goForward();
break;
case "screenshot":
console.log("Taking a screenshot");
break;
default:
console.log("Unknown action:", action);
}
} catch (error) {
console.error("Error executing action:", action, error);
}
}
export async function getScreenshotAsBase64(page) {
const screenshotBuffer = await page.screenshot({ fullPage: true });
return screenshotBuffer.toString("base64");
}