src/domains/drag.ts (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
import { Type } from "typebox";
import { type Result, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
import { cdpCall } from "./cdp-call";
const DragArgs = Type.Object({
startX: Type.Number(),
startY: Type.Number(),
endX: Type.Number(),
endY: Type.Number(),
dataTransfer: Type.Optional(
Type.Record(Type.String(), Type.String(), {
description: "Optional MIME → data map for DataTransfer (e.g., { 'text/plain': 'hello' })",
}),
),
});
export const dragAndDropTool = defineBrowserTool({
name: "browser_drag_and_drop",
label: "Browser Drag & Drop",
description: "Drag from (startX, startY) to (endX, endY) using CDP Input.dispatchDragEvent.",
promptSnippet: "Drag and drop between two coordinates",
promptGuidelines: [
"Coordinates in CSS pixels.",
"dataTransfer is an optional MIME→data map (e.g., { 'text/plain': 'hello' }).",
],
parameters: DragArgs,
concurrency: "serialized",
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
const data = args.dataTransfer
? {
items: Object.entries(args.dataTransfer).map(([mimeType, payload]) => ({
mimeType,
data: Buffer.from(payload).toString("base64"),
})),
dragOperationsMask: 1,
}
: { items: [], dragOperationsMask: 1 };
type InputCall = readonly ["Input.dispatchMouseEvent" | "Input.dispatchDragEvent", Record<string, unknown>];
const calls: ReadonlyArray<InputCall> = [
["Input.dispatchMouseEvent", { type: "mousePressed", x: args.startX, y: args.startY, button: "left", clickCount: 1 }],
["Input.dispatchDragEvent", { type: "dragEnter", x: args.startX, y: args.startY, data, modifiers: 0 }],
...Array.from({ length: 5 }, (_, i): InputCall => {
const t = (i + 1) / 5;
return [
"Input.dispatchDragEvent",
{
type: "dragOver",
x: Math.round(args.startX + (args.endX - args.startX) * t),
y: Math.round(args.startY + (args.endY - args.startY) * t),
data,
modifiers: 0,
},
];
}),
["Input.dispatchDragEvent", { type: "drop", x: args.endX, y: args.endY, data, modifiers: 0 }],
["Input.dispatchMouseEvent", { type: "mouseReleased", x: args.endX, y: args.endY, button: "left", clickCount: 1 }],
];
for (const [method, params] of calls) {
const r = await cdpCall(client, method, params);
if (!r.success) return r;
}
return ok({
text: `Dragged from (${args.startX},${args.startY}) to (${args.endX},${args.endY})`,
});
},
});
|