all repos — sz-chrome-remote-cdp @ master

pi-browser-harness, patched to drive a Chrome on another host over CDP, with castle_cdp_search web search

src/domains/keyboard.ts (view raw)

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
import { Type } from "typebox";
import { safeJs } from "../util/js-template";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
import { cdpCall, evalJs } from "./cdp-call";
import { resolveRefToObjectId } from "./ref-resolve";

const VIRTUAL_KEY_CODES: Readonly<Record<string, number>> = {
  Enter: 13, Tab: 9, Backspace: 8, Escape: 27, Delete: 46, " ": 32,
  ArrowLeft: 37, ArrowUp: 38, ArrowRight: 39, ArrowDown: 40,
  Home: 36, End: 35, PageUp: 33, PageDown: 34,
};

const virtualKeyCode = (key: string): number =>
  VIRTUAL_KEY_CODES[key] ?? (key.length === 1 ? key.charCodeAt(0) : 0);

const TypeArgs = Type.Object({
  text: Type.String({ description: "Text to type" }),
});

type FocusProbe = { readonly ok: boolean; readonly tag: string | null };

const isFocusProbe = (v: unknown): v is FocusProbe =>
  typeof v === "object" && v !== null
  && "ok" in v && typeof v.ok === "boolean"
  && "tag" in v && (typeof v.tag === "string" || v.tag === null);

export const typeTool = defineBrowserTool({
  name: "browser_type",
  label: "Browser Type",
  description: "Type text into the currently focused element. Use browser_click first to focus an input field.",
  promptSnippet: "Type text into the focused element",
  promptGuidelines: [
    "For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.",
  ],
  parameters: TypeArgs,
  concurrency: "serialized",
  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
    const focused = await evalJs(client, safeJs`
      (() => {
        const el = document.activeElement;
        const ok = !!el && el !== document.body &&
          (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable);
        return { ok, tag: el ? el.tagName : null };
      })()
    `);
    if (focused.success) {
      const f: FocusProbe = isFocusProbe(focused.data) ? focused.data : { ok: false, tag: null };
      if (!f.ok) {
        return err({
          kind: "invalid_state",
          message:
            "No editable element is focused — typed text would be lost. Click the field with browser_click first, or use browser_fill with a selector.",
          details: { activeElement: f.tag },
        });
      }
    }
    const r = await cdpCall(client, "Input.insertText", { text: args.text });
    if (!r.success) return r;
    return ok({ text: `Typed: "${args.text}"` });
  },
});

const PressKeyArgs = Type.Object({
  key: Type.String({
    description:
      'Key to press. Special keys: Enter, Tab, Backspace, Escape, Delete, ArrowLeft/Up/Right/Down, Home, End, PageUp, PageDown. Space as " "',
  }),
  modifiers: Type.Optional(
    Type.Integer({
      minimum: 0,
      maximum: 15,
      description: "Modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with OR.",
    }),
  ),
});

export const pressKeyTool = defineBrowserTool({
  name: "browser_press_key",
  label: "Browser Press Key",
  description:
    "Press a keyboard key. Supports special keys (Enter, Tab, Backspace, Escape, Delete, arrows, Home, End, PageUp, PageDown, Space as ' ') and regular characters. Optional modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta/Cmd, 8=Shift.",
  promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)",
  promptGuidelines: [
    "Use browser_press_key for keyboard shortcuts and navigation keys.",
    "browser_press_key modifiers combine with bitwise OR: Ctrl+Shift = 2|8 = 10.",
  ],
  parameters: PressKeyArgs,
  concurrency: "serialized",
  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
    const k = args.key;
    const code = virtualKeyCode(k);
    const modifiers = args.modifiers ?? 0;
    const isChar = k.length === 1;
    const downParams: Record<string, unknown> = {
      type: "keyDown",
      key: k,
      code: k,
      modifiers,
      ...(code ? { windowsVirtualKeyCode: code, nativeVirtualKeyCode: code } : {}),
      ...(isChar ? { text: k, unmodifiedText: k } : {}),
    };
    const down = await cdpCall(client, "Input.dispatchKeyEvent", downParams);
    if (!down.success) return down;
    // A printable character needs a `char` event for the page to receive keypress/textInput and actually insert it; Shift (8) still counts as printable.
    const hasCommandModifier = (modifiers & (1 | 2 | 4)) !== 0;
    if (isChar && !hasCommandModifier) {
      const charEv = await cdpCall(client, "Input.dispatchKeyEvent", { type: "char", text: k, key: k });
      if (!charEv.success) return charEv;
    }
    const up = await cdpCall(client, "Input.dispatchKeyEvent", { type: "keyUp", key: k, code: k, modifiers });
    if (!up.success) return up;
    return ok({ text: `Pressed: ${k}${modifiers ? ` (modifiers=${modifiers})` : ""}` });
  },
});

const DispatchKeyArgs = Type.Object({
  ref: Type.Optional(
    Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector." }),
  ),
  selector: Type.Optional(Type.String({ description: "CSS selector of the target element (fallback when no ref)" })),
  key: Type.String({ description: "Key value (e.g., 'Enter', 'a')" }),
  eventType: Type.Optional(
    Type.Union(
      [Type.Literal("keydown"), Type.Literal("keyup"), Type.Literal("keypress")],
      { default: "keydown", description: "Event type to dispatch. Default: keydown" },
    ),
  ),
});

export const dispatchKeyTool = defineBrowserTool({
  name: "browser_dispatch_key",
  label: "Browser Dispatch Key",
  description:
    "Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.",
  promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector",
  promptGuidelines: [
    "Does NOT insert text (use browser_type or browser_press_key for actual typing).",
    "Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.",
    "The event carries keyCode/which (e.g. 13 for Enter) for legacy handlers, but is untrusted (isTrusted === false) — a few libraries may still ignore it.",
  ],
  parameters: DispatchKeyArgs,
  concurrency: "serialized",
  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
    const eventType = args.eventType ?? "keydown";
    const target = args.ref ?? args.selector ?? "";
    // Legacy handlers branch on e.keyCode/e.which rather than e.key, so populate both.
    const code = virtualKeyCode(args.key);
    if (args.ref !== undefined) {
      const objectId = await resolveRefToObjectId(client, args.ref);
      if (!objectId.success) return objectId;
      const r = await cdpCall(client, "Runtime.callFunctionOn", {
        objectId: objectId.data,
        functionDeclaration: `function (type, key, keyCode) { this.dispatchEvent(new KeyboardEvent(type, { key, keyCode, which: keyCode, bubbles: true, cancelable: true })); return 1; }`,
        arguments: [{ value: eventType }, { value: args.key }, { value: code }],
        returnByValue: true,
      });
      if (!r.success) return r;
      return ok({
        text: `Dispatched ${eventType}('${args.key}') on ${target}`,
        details: { matched: 1, ref: args.ref },
      });
    }
    if (args.selector === undefined) {
      return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
    }
    const expr = safeJs`
      (() => {
        const els = document.querySelectorAll(${args.selector});
        if (els.length === 0) return 0;
        for (const el of els) {
          el.dispatchEvent(new KeyboardEvent(${eventType}, { key: ${args.key}, keyCode: ${code}, which: ${code}, bubbles: true, cancelable: true }));
        }
        return els.length;
      })()
    `;
    const r = await evalJs(client, expr);
    if (!r.success) return r;
    const matched = Number(r.data ?? 0);
    if (matched === 0) {
      return err({
        kind: "invalid_state",
        message: `Selector matched 0 elements: ${args.selector}`,
        details: { matched: 0, selector: args.selector },
      });
    }
    return ok({
      text: `Dispatched ${eventType}('${args.key}') on ${matched} element(s)`,
      details: { matched, selector: args.selector },
    });
  },
});