src/domains/keyboard.ts (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 |
import { Type } from "typebox";
import { safeJs } from "../util/js-template";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
import { cdpCall, evalJs } from "./cdp-call";
import { resolveRefToObjectId } from "./ref-resolve";
const VIRTUAL_KEY_CODES: Readonly<Record<string, number>> = {
Enter: 13, Tab: 9, Backspace: 8, Escape: 27, Delete: 46, " ": 32,
ArrowLeft: 37, ArrowUp: 38, ArrowRight: 39, ArrowDown: 40,
Home: 36, End: 35, PageUp: 33, PageDown: 34,
};
const virtualKeyCode = (key: string): number =>
VIRTUAL_KEY_CODES[key] ?? (key.length === 1 ? key.charCodeAt(0) : 0);
const TypeArgs = Type.Object({
text: Type.String({ description: "Text to type" }),
});
type FocusProbe = { readonly ok: boolean; readonly tag: string | null };
const isFocusProbe = (v: unknown): v is FocusProbe =>
typeof v === "object" && v !== null
&& "ok" in v && typeof v.ok === "boolean"
&& "tag" in v && (typeof v.tag === "string" || v.tag === null);
export const typeTool = defineBrowserTool({
name: "browser_type",
label: "Browser Type",
description: "Type text into the currently focused element. Use browser_click first to focus an input field.",
promptSnippet: "Type text into the focused element",
promptGuidelines: [
"For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.",
],
parameters: TypeArgs,
concurrency: "serialized",
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
const focused = await evalJs(client, safeJs`
(() => {
const el = document.activeElement;
const ok = !!el && el !== document.body &&
(el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable);
return { ok, tag: el ? el.tagName : null };
})()
`);
if (focused.success) {
const f: FocusProbe = isFocusProbe(focused.data) ? focused.data : { ok: false, tag: null };
if (!f.ok) {
return err({
kind: "invalid_state",
message:
"No editable element is focused — typed text would be lost. Click the field with browser_click first, or use browser_fill with a selector.",
details: { activeElement: f.tag },
});
}
}
const r = await cdpCall(client, "Input.insertText", { text: args.text });
if (!r.success) return r;
return ok({ text: `Typed: "${args.text}"` });
},
});
const PressKeyArgs = Type.Object({
key: Type.String({
description:
'Key to press. Special keys: Enter, Tab, Backspace, Escape, Delete, ArrowLeft/Up/Right/Down, Home, End, PageUp, PageDown. Space as " "',
}),
modifiers: Type.Optional(
Type.Integer({
minimum: 0,
maximum: 15,
description: "Modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with OR.",
}),
),
});
export const pressKeyTool = defineBrowserTool({
name: "browser_press_key",
label: "Browser Press Key",
description:
"Press a keyboard key. Supports special keys (Enter, Tab, Backspace, Escape, Delete, arrows, Home, End, PageUp, PageDown, Space as ' ') and regular characters. Optional modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta/Cmd, 8=Shift.",
promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)",
promptGuidelines: [
"Use browser_press_key for keyboard shortcuts and navigation keys.",
"browser_press_key modifiers combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
],
parameters: PressKeyArgs,
concurrency: "serialized",
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
const k = args.key;
const code = virtualKeyCode(k);
const modifiers = args.modifiers ?? 0;
const isChar = k.length === 1;
const downParams: Record<string, unknown> = {
type: "keyDown",
key: k,
code: k,
modifiers,
...(code ? { windowsVirtualKeyCode: code, nativeVirtualKeyCode: code } : {}),
...(isChar ? { text: k, unmodifiedText: k } : {}),
};
const down = await cdpCall(client, "Input.dispatchKeyEvent", downParams);
if (!down.success) return down;
// A printable character needs a `char` event for the page to receive keypress/textInput and actually insert it; Shift (8) still counts as printable.
const hasCommandModifier = (modifiers & (1 | 2 | 4)) !== 0;
if (isChar && !hasCommandModifier) {
const charEv = await cdpCall(client, "Input.dispatchKeyEvent", { type: "char", text: k, key: k });
if (!charEv.success) return charEv;
}
const up = await cdpCall(client, "Input.dispatchKeyEvent", { type: "keyUp", key: k, code: k, modifiers });
if (!up.success) return up;
return ok({ text: `Pressed: ${k}${modifiers ? ` (modifiers=${modifiers})` : ""}` });
},
});
const DispatchKeyArgs = Type.Object({
ref: Type.Optional(
Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector." }),
),
selector: Type.Optional(Type.String({ description: "CSS selector of the target element (fallback when no ref)" })),
key: Type.String({ description: "Key value (e.g., 'Enter', 'a')" }),
eventType: Type.Optional(
Type.Union(
[Type.Literal("keydown"), Type.Literal("keyup"), Type.Literal("keypress")],
{ default: "keydown", description: "Event type to dispatch. Default: keydown" },
),
),
});
export const dispatchKeyTool = defineBrowserTool({
name: "browser_dispatch_key",
label: "Browser Dispatch Key",
description:
"Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.",
promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector",
promptGuidelines: [
"Does NOT insert text (use browser_type or browser_press_key for actual typing).",
"Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.",
"The event carries keyCode/which (e.g. 13 for Enter) for legacy handlers, but is untrusted (isTrusted === false) — a few libraries may still ignore it.",
],
parameters: DispatchKeyArgs,
concurrency: "serialized",
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
const eventType = args.eventType ?? "keydown";
const target = args.ref ?? args.selector ?? "";
// Legacy handlers branch on e.keyCode/e.which rather than e.key, so populate both.
const code = virtualKeyCode(args.key);
if (args.ref !== undefined) {
const objectId = await resolveRefToObjectId(client, args.ref);
if (!objectId.success) return objectId;
const r = await cdpCall(client, "Runtime.callFunctionOn", {
objectId: objectId.data,
functionDeclaration: `function (type, key, keyCode) { this.dispatchEvent(new KeyboardEvent(type, { key, keyCode, which: keyCode, bubbles: true, cancelable: true })); return 1; }`,
arguments: [{ value: eventType }, { value: args.key }, { value: code }],
returnByValue: true,
});
if (!r.success) return r;
return ok({
text: `Dispatched ${eventType}('${args.key}') on ${target}`,
details: { matched: 1, ref: args.ref },
});
}
if (args.selector === undefined) {
return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
}
const expr = safeJs`
(() => {
const els = document.querySelectorAll(${args.selector});
if (els.length === 0) return 0;
for (const el of els) {
el.dispatchEvent(new KeyboardEvent(${eventType}, { key: ${args.key}, keyCode: ${code}, which: ${code}, bubbles: true, cancelable: true }));
}
return els.length;
})()
`;
const r = await evalJs(client, expr);
if (!r.success) return r;
const matched = Number(r.data ?? 0);
if (matched === 0) {
return err({
kind: "invalid_state",
message: `Selector matched 0 elements: ${args.selector}`,
details: { matched: 0, selector: args.selector },
});
}
return ok({
text: `Dispatched ${eventType}('${args.key}') on ${matched} element(s)`,
details: { matched, selector: args.selector },
});
},
});
|