all repos — sz-chrome-remote-cdp @ 68bf7fd2988e42343833be09e4115cd7175029d0

pi-browser-harness, patched to drive a Chrome on another host over CDP, with castle_cdp_search web search

feat: add browser_read_page reader-mode tool

URL (or owned targetId) → clean main-article text with nav/ads/footer
boilerplate stripped, via a dependency-free readability heuristic. DOM walk
lives in an in-page capture expression; scoring/selection is a pure, fixture-
tested function (link-density + boilerplate-ancestor filtering, body-text
fallback for structure-less pages). Reuses the isolated-attach lifecycle;
registered unserialized (safe concurrent read). 10 extractor unit scenarios.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
amankumarsingh77 amankumarsingh7702@gmail.com
Wed, 08 Jul 2026 21:24:32 +0530
commit

68bf7fd2988e42343833be09e4115cd7175029d0

parent

c5c42df5976ed63faac2adcf3b04a4ad2f519b32

A src/domains/readpage/capture.ts

@@ -0,0 +1,46 @@

+/** + * The JavaScript source evaluated in a loaded page to capture candidate content + * blocks + metadata for the pure readability extractor (readability.ts). Crosses + * the CDP boundary via Runtime.evaluate (returnByValue) and returns a JSON + * string shaped like PageCapture. + * + * The DOM walk lives here (unavoidable); all scoring/selection is pure and lives + * in readability.ts, so the decision logic stays unit-testable. + */ + +/** Max bytes of body-text fallback captured for thin/structure-less pages. */ +const BODY_TEXT_LIMIT = 20_000; + +/** Build the capture expression. No interpolated values ⇒ no injection surface. */ +export const buildPageCaptureExpr = (): string => ` + (() => { + const BOILERPLATE = new Set(['NAV', 'HEADER', 'FOOTER', 'ASIDE']); + const inBoilerplate = (el) => { + for (let n = el; n; n = n.parentElement) { + if (BOILERPLATE.has(n.tagName)) return true; + const role = n.getAttribute && n.getAttribute('role'); + if (role === 'navigation' || role === 'banner' || role === 'contentinfo') return true; + } + return false; + }; + const kindOf = (tag) => { + if (/^H[1-6]$/.test(tag)) return 'heading'; + if (tag === 'LI') return 'listitem'; + if (tag === 'BLOCKQUOTE') return 'blockquote'; + if (tag === 'P') return 'paragraph'; + return 'other'; + }; + const selector = 'p, h1, h2, h3, h4, h5, h6, li, blockquote'; + const root = document.querySelector('article, main, [role=main]') || document.body; + const blocks = []; + for (const el of root.querySelectorAll(selector)) { + const text = (el.innerText || '').replace(/\\s+/g, ' ').trim(); + if (!text) continue; + let linkTextLength = 0; + for (const a of el.querySelectorAll('a')) linkTextLength += (a.innerText || '').length; + blocks.push({ kind: kindOf(el.tagName), text, linkTextLength, inBoilerplate: inBoilerplate(el) }); + } + const bodyText = (document.body.innerText || '').replace(/\\n{3,}/g, '\\n\\n').slice(0, ${BODY_TEXT_LIMIT}); + return JSON.stringify({ url: location.href, title: document.title, blocks, bodyText }); + })() +`;
A src/domains/readpage/read-page.ts

@@ -0,0 +1,109 @@

+import { Type } from "typebox"; +import { type Result, err, ok } from "../../util/result"; +import { applyTruncation } from "../../util/truncate"; +import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool"; +import { + closeIsolatedTab, + evalInIsolatedTab, + type IsolatedTab, + navigateIsolatedTab, + openIsolatedTab, + waitForIsolatedLoad, +} from "../isolated-tab"; +import { buildPageCaptureExpr } from "./capture"; +import { extractReadable, type PageCapture, type ReadablePage } from "./readability"; + +const LOAD_TIMEOUT_MS = 15_000; + +const ReadPageArgs = Type.Object({ + url: Type.Optional(Type.String({ description: "URL to open in an isolated tab, read, then close." })), + targetId: Type.Optional( + Type.String({ description: "Read an already-open tab this session owns instead of opening a URL." }), + ), +}); + +const isPageCapture = (value: unknown): value is PageCapture => { + if (typeof value !== "object" || value === null) return false; + const v = value as Record<string, unknown>; + return typeof v["url"] === "string" && typeof v["title"] === "string" && Array.isArray(v["blocks"]) && typeof v["bodyText"] === "string"; +}; + +const safeParse = (text: string): unknown => { + try { + return JSON.parse(text) as unknown; + } catch { + return undefined; + } +}; + +const captureToResult = async (raw: unknown): Promise<Result<ToolOk, ToolErr>> => { + const parsed = typeof raw === "string" ? safeParse(raw) : raw; + if (!isPageCapture(parsed)) return err({ kind: "internal", message: "page capture returned an unexpected shape" }); + const page: ReadablePage = extractReadable(parsed); + const header = `# ${page.title}\n${page.url}\n(${page.wordCount} words)\n\n`; + const truncated = await applyTruncation(header + page.text, "readpage"); + return ok({ + text: truncated.text, + details: { + title: page.title, + url: page.url, + wordCount: page.wordCount, + ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}), + }, + }); +}; + +const readOpenedUrl = async ( + client: Parameters<typeof openIsolatedTab>[0], + url: string, + signal: AbortSignal | undefined, +): Promise<Result<ToolOk, ToolErr>> => { + const opened = await openIsolatedTab(client); + if (!opened.success) return err(opened.error); + const tab: IsolatedTab = opened.data; + try { + const navigated = await navigateIsolatedTab(client, tab, url); + if (!navigated.success) return err(navigated.error); + const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal); + if (!loaded.success) return err(loaded.error); + const captured = await evalInIsolatedTab(client, tab, buildPageCaptureExpr()); + if (!captured.success) return err(captured.error); + return captureToResult(captured.data); + } finally { + await closeIsolatedTab(client, tab); + } +}; + +const readOwnedTab = async ( + client: Parameters<typeof openIsolatedTab>[0], + targetId: string, +): Promise<Result<ToolOk, ToolErr>> => { + if (!client.owns(targetId)) { + return err({ kind: "invalid_state", message: `Tab ${targetId} is not owned by this session.` }); + } + const attached = await client.session().callBrowser("Target.attachToTarget", { targetId, flatten: true }); + if (!attached.success) return err({ kind: "cdp_error", message: attached.error.message }); + const sessionId = (attached.data as { sessionId: string }).sessionId; + const captured = await client.evaluateJs(buildPageCaptureExpr(), sessionId); + if (!captured.success) return err({ kind: "cdp_error", message: captured.error.message }); + return captureToResult(captured.data); +}; + +export const readPageTool = defineBrowserTool({ + name: "browser_read_page", + label: "Browser Read Page", + description: + "Read a page as clean article text — main content with nav/ads/boilerplate stripped. Pass a url (opened in an isolated tab, read, then closed) or a targetId of an owned tab. Reader-mode counterpart to browser_web_search.", + promptSnippet: "Read a page's main content as clean text", + promptGuidelines: [ + "Pass url to read an arbitrary page (opened + closed in its own tab, never disturbing your current tab), or targetId to read an already-open owned tab.", + "Returns readable main-article text — use this over browser_snapshot/browser_execute_js when you want an article's content for reading or research.", + "Boilerplate-heavy or structure-less pages fall back to bounded body text rather than erroring.", + ], + parameters: ReadPageArgs, + async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> { + if (args.url !== undefined && args.url.length > 0) return readOpenedUrl(client, args.url, signal); + if (args.targetId !== undefined && args.targetId.length > 0) return readOwnedTab(client, args.targetId); + return err({ kind: "invalid_state", message: "provide either url or targetId" }); + }, +});
A src/domains/readpage/readability.ts

@@ -0,0 +1,89 @@

+/** + * Pure reader-mode extraction — no CDP, no DOM. The in-page script + * (see capture.ts) serializes candidate content blocks; these pure functions + * score them and assemble clean article text, so the selection logic is + * unit-testable against saved fixtures (test/deep-research/readability-test.ts). + * + * Dependency-free by design (no jsdom / Readability): the harness ships no new + * runtime deps. The heuristic favors blocks with high text density and low link + * density, which is what separates article body from nav/ads/boilerplate. + */ + +/** One candidate content block captured in-page. */ +export type ContentBlock = { + /** Semantic tag: paragraph | heading | listitem | blockquote | other. */ + readonly kind: "paragraph" | "heading" | "listitem" | "blockquote" | "other"; + readonly text: string; + /** Chars of text that sit inside <a> tags — high ratio ⇒ nav/menu, not prose. */ + readonly linkTextLength: number; + /** True when an ancestor is nav/header/footer/aside — structural boilerplate. */ + readonly inBoilerplate: boolean; +}; + +/** The JSON payload capture.ts returns from the page. */ +export type PageCapture = { + readonly url: string; + readonly title: string; + readonly blocks: ReadonlyArray<ContentBlock>; + /** Bounded innerText fallback when block extraction finds nothing usable. */ + readonly bodyText: string; +}; + +/** The tool's public output shape. */ +export type ReadablePage = { + readonly title: string; + readonly url: string; + readonly text: string; + readonly wordCount: number; +}; + +const MIN_PARAGRAPH_CHARS = 25; +const MAX_LINK_DENSITY = 0.5; +const MIN_ARTICLE_WORDS = 40; + +const linkDensity = (block: ContentBlock): number => { + if (block.text.length === 0) return 1; + return block.linkTextLength / block.text.length; +}; + +/** A block is article prose if it is substantive, text-dense, and not chrome. */ +const isArticleBlock = (block: ContentBlock): boolean => { + if (block.inBoilerplate) return false; + if (linkDensity(block) > MAX_LINK_DENSITY) return false; + if (block.kind === "heading" || block.kind === "blockquote" || block.kind === "listitem") { + return block.text.trim().length > 0; + } + return block.text.trim().length >= MIN_PARAGRAPH_CHARS; +}; + +const render = (block: ContentBlock): string => { + const text = block.text.trim(); + if (block.kind === "heading") return `\n## ${text}\n`; + if (block.kind === "listitem") return `- ${text}`; + if (block.kind === "blockquote") return `> ${text}`; + return text; +}; + +const wordCountOf = (text: string): number => { + const trimmed = text.trim(); + return trimmed.length === 0 ? 0 : trimmed.split(/\s+/).length; +}; + +/** + * Assemble readable article text from a page capture. Keeps text-dense, + * non-boilerplate blocks; falls back to the bounded body text when the article + * is too thin (e.g. a page with no clear content structure). + */ +export const extractReadable = (capture: PageCapture): ReadablePage => { + const articleText = capture.blocks.filter(isArticleBlock).map(render).join("\n\n").replace(/\n{3,}/g, "\n\n").trim(); + + const useFallback = wordCountOf(articleText) < MIN_ARTICLE_WORDS; + const text = useFallback ? capture.bodyText.trim() : articleText; + + return { + title: capture.title.trim(), + url: capture.url, + text, + wordCount: wordCountOf(text), + }; +};
M src/registry.ts → src/registry.ts

@@ -20,6 +20,7 @@ import { snapshotTool } from "./domains/snapshot";

import { executeJsTool, runScriptTool } from "./domains/js"; import { setupTool } from "./domains/setup"; import { webSearchTool } from "./domains/search/web-search"; +import { readPageTool } from "./domains/readpage/read-page"; // Mutation tools that modify page / browser state are marked `serialized: true` // so they run through a shared async mutex. Observation / read-only tools are

@@ -64,6 +65,8 @@ executeJsTool,

runScriptTool, // Opens and owns an isolated tab to scrape a SERP — same class as openUrlsTool. { ...webSearchTool, serialized: true }, + // Pure isolated read (own sessionId, own tab) — safe concurrent, like executeJsTool. + readPageTool, ]; export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => {
A test/deep-research/fixtures/article-capture.json

@@ -0,0 +1,17 @@

+{ + "url": "https://blog.example.com/how-cdp-works", + "title": "How the Chrome DevTools Protocol Works", + "blocks": [ + { "kind": "other", "text": "Home Products Blog About Contact", "linkTextLength": 33, "inBoilerplate": true }, + { "kind": "heading", "text": "How the Chrome DevTools Protocol Works", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "paragraph", "text": "The Chrome DevTools Protocol lets tools instrument, inspect, and control Chromium-based browsers over a WebSocket connection.", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "paragraph", "text": "Each command targets a domain such as Page, Runtime, or Network, and receives a structured JSON response that the client correlates by id.", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "heading", "text": "Sessions and Targets", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "paragraph", "text": "A target is a page or worker; attaching to one yields a session id that scopes subsequent commands to that target without disturbing others.", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "listitem", "text": "Attach with Target.attachToTarget", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "listitem", "text": "Drive with Runtime.evaluate", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "paragraph", "text": "Subscribe here for our newsletter and follow us on social media for updates", "linkTextLength": 55, "inBoilerplate": true }, + { "kind": "other", "text": "Copyright 2026 Example Inc. Privacy Terms Cookies", "linkTextLength": 24, "inBoilerplate": true } + ], + "bodyText": "Home Products Blog How the Chrome DevTools Protocol Works The Chrome DevTools Protocol lets tools instrument inspect and control browsers Copyright 2026" +}
A test/deep-research/fixtures/no-article-capture.json

@@ -0,0 +1,10 @@

+{ + "url": "https://app.example.com/dashboard", + "title": "Dashboard", + "blocks": [ + { "kind": "other", "text": "Menu", "linkTextLength": 4, "inBoilerplate": true }, + { "kind": "listitem", "text": "Widget A", "linkTextLength": 8, "inBoilerplate": true }, + { "kind": "listitem", "text": "Widget B", "linkTextLength": 8, "inBoilerplate": true } + ], + "bodyText": "Dashboard Widget A Widget B Your account is active. Usage this month is within limits and no action is required at this time from you." +}
A test/deep-research/readability-test.ts

@@ -0,0 +1,46 @@

+/** + * Unit tests for the pure readability extractor — no browser required. + * + * Run: npx tsx test/deep-research/readability-test.ts + */ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { extractReadable, type PageCapture } from "../../src/domains/readpage/readability"; + +let passed = 0; +let failed = 0; +const check = (cond: boolean, label: string): void => { + if (cond) { + passed++; + console.log(` ✓ ${label}`); + } else { + failed++; + console.error(` ✗ ${label}`); + } +}; + +const fixture = (name: string): PageCapture => + JSON.parse(readFileSync(join(import.meta.dirname, "fixtures", name), "utf8")) as PageCapture; + +// S10: boilerplate stripped, article content preserved +{ + const page = extractReadable(fixture("article-capture.json")); + check(page.text.includes("Chrome DevTools Protocol lets tools instrument"), "S10: article body text is present"); + check(page.text.includes("Sessions and Targets"), "S10: article headings are preserved"); + check(!page.text.includes("Home Products Blog"), "S10: nav boilerplate is dropped"); + check(!page.text.includes("Copyright 2026"), "S10: footer boilerplate is dropped"); + check(!page.text.includes("Subscribe here for our newsletter"), "S10: high-link-density promo block is dropped"); + check(page.text.includes("## Sessions and Targets"), "S10: paragraph/heading structure preserved as markdown"); + check(page.title === "How the Chrome DevTools Protocol Works" && page.url.startsWith("https://"), "S10: title and url passed through"); + check(page.wordCount > 40, "S10: word count reflects the article body"); +} + +// S11: no clear article node falls back to bounded body text +{ + const page = extractReadable(fixture("no-article-capture.json")); + check(page.text.length > 0, "S11: structure-less page falls back to non-empty body text"); + check(page.text.includes("Your account is active"), "S11: fallback body text carries the real content"); +} + +console.log(`\n${passed} passed, ${failed} failed`); +if (failed > 0) process.exit(1);