castle-search/src/engines.ts (view raw)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 |
/**
* The search engines this extension knows how to drive.
*
* Every selector and every URL parameter below was read off the real SERP in
* castle's Chrome, not recalled or inferred. That is not pedantry: Google's own
* Tools menu writes `tbs=qdr:*`, which renders an *empty* page on that profile,
* while the undocumented-looking `as_qdr=*` works. Anything in here that was not
* observed is a bug waiting to be reported as "no results".
*
* Two extraction modes, because the engines genuinely differ:
*
* - "items" — the SERP has a clean per-result container (`li.b_algo`,
* `.result`, `.snippet[data-type=web]`). Straightforward.
* - "headings" — Google has no stable result container, so results are found
* from each <h3> outward. Kept as its own mode rather than
* forced into the item shape, because it is the one that has
* been through the most verification.
*/
export type EngineId = "google" | "duckduckgo" | "bing" | "brave";
export type Recency = "day" | "week" | "month" | "year";
export const RECENCY_VALUES = ["day", "week", "month", "year"] as const;
export const ENGINE_IDS: readonly EngineId[] = ["google", "duckduckgo", "bing", "brave"];
/** Aliases accepted from humans and from the model. */
const ENGINE_ALIASES: Record<string, EngineId> = {
google: "google",
g: "google",
duckduckgo: "duckduckgo",
ddg: "duckduckgo",
duck: "duckduckgo",
bing: "bing",
b: "bing",
brave: "brave",
};
/**
* How the in-page script should read one engine's results. Passed into the
* browser as JSON, so everything here must be plain data.
*/
export type ExtractionConfig = {
mode: "items" | "headings";
/** Results container candidates, first match wins. */
roots: string[];
/**
* Present only on the engine's own "nothing matched" page. Without it a page
* that genuinely has zero hits is indistinguishable from broken markup, and
* the error blames the extractor for what is just an over-narrow query.
*/
emptyMarker?: string;
/** "items" mode: one result. */
item?: string;
/** Anchor carrying the outbound link, relative to the item. */
link?: string;
/** Title element, relative to the item. Falls back to the anchor's text. */
title?: string;
/** Description element, relative to the item. */
snippet?: string;
/**
* "items" mode with no snippet selector: text from these is subtracted from
* the item's text to leave the description behind.
*/
subtract?: string[];
/** Items matching any of these are ads or non-web cards, and are skipped. */
exclude?: string[];
/** How outbound links are wrapped, if they are. */
unwrap: "none" | "ddg" | "bing" | "google";
/** Hosts belonging to the engine itself; links to them are not results. */
selfHostPattern: string;
};
export type EngineDefinition = {
readonly id: EngineId;
readonly label: string;
/** Human-facing note about what makes this engine worth choosing. */
readonly note: string;
/**
* Recency windows this engine can actually express, mapped to the parameter
* value. A window that is absent is one the engine does not support — never
* one that is silently dropped.
*/
readonly recency: Partial<Record<Recency, string>>;
readonly buildUrl: (query: string, numResults: number, recency: Recency | undefined) => string;
readonly extraction: ExtractionConfig;
};
const GOOGLE: EngineDefinition = {
id: "google",
label: "Google",
note: "best result quality; blocks aggressively under repeated automated queries",
recency: { day: "d", week: "w", month: "m", year: "y" },
buildUrl: (query, numResults, recency) => {
const params = new URLSearchParams({ q: query, num: String(numResults), hl: "en" });
// `udm=14` is the plain "Web" tab: no AI overview, no carousels. But any
// date restriction combined with it renders an empty page, and `tbs=qdr:*`
// renders an empty page on its own — both observed. So a time-limited
// search drops udm and uses the older as_qdr instead.
if (recency) params.set("as_qdr", GOOGLE.recency[recency] as string);
else params.set("udm", "14");
return `https://www.google.com/search?${params.toString()}`;
},
extraction: {
mode: "headings",
roots: ["#rso", "#search"],
// Every result is `google.com/goto?url=<opaque token>` (since at least
// 2026-11-06). The page cannot unwrap it; search.ts resolves it afterwards.
// It must not count as Google's own link, hence the lookahead.
unwrap: "google",
selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/(?!goto\?)`,
},
};
const DUCKDUCKGO: EngineDefinition = {
id: "duckduckgo",
label: "DuckDuckGo",
note: "no-JS endpoint, the most stable markup here and the least likely to challenge",
// Read off DuckDuckGo's own <select name="df">: "", d, w, m, y.
recency: { day: "d", week: "w", month: "m", year: "y" },
buildUrl: (query, _numResults, recency) => {
// The html endpoint renders server-side with no JavaScript, which makes it
// both faster and far less fragile than the app at duckduckgo.com. It has
// no result-count parameter — it returns a full page and the caller slices.
const params = new URLSearchParams({ q: query });
if (recency) params.set("df", DUCKDUCKGO.recency[recency] as string);
return `https://html.duckduckgo.com/html/?${params.toString()}`;
},
extraction: {
mode: "items",
roots: [".results", "#links"],
// Seen 2026-11-06: `<div class="result results_links no-results">` — it is
// itself a `.result`, which is why the extractor finds a container and no hits.
emptyMarker: ".no-results",
item: ".result",
link: "a.result__a[href]",
title: "a.result__a",
snippet: ".result__snippet",
exclude: [".result--ad", ".badge--ad"],
unwrap: "ddg",
selfHostPattern: String.raw`^https?://(html\.|www\.)?duckduckgo\.com/`,
},
};
const BING: EngineDefinition = {
id: "bing",
label: "Bing",
note: "good coverage; supports day/week/month only — it has no year filter",
// ez1/ez2/ez3 verified to filter (results carried "4 hours ago", "1 day ago").
// There is deliberately no year: Bing's UI offers no such window, and the
// ez5 custom-range form returned nothing when tried.
recency: { day: "ez1", week: "ez2", month: "ez3" },
buildUrl: (query, numResults, recency) => {
const params = new URLSearchParams({ q: query });
if (recency) {
// `count` silently cancels `filters` — with ez1 alone every result is
// hours old, and adding count in either order brings back months-old
// ones. Observed directly, and it fails silently: the results look
// perfectly plausible, just unfiltered. Exactly the same trap as Google's
// udm+as_qdr, so the same answer — drop the count parameter and let the
// caller slice the list it gets.
params.set("filters", `ex1:"${BING.recency[recency] as string}"`);
} else {
params.set("count", String(numResults));
}
return `https://www.bing.com/search?${params.toString()}`;
},
extraction: {
mode: "items",
// Organic results only. Bing's answer cards also contain <h2>s, which is
// why this targets li.b_algo rather than walking headings.
roots: ["#b_results"],
item: "li.b_algo",
link: "h2 a[href]",
title: "h2",
snippet: ".b_caption p, .b_algoSlug, p",
exclude: [".b_ad", ".b_adBottom"],
unwrap: "bing",
selfHostPattern: String.raw`^https?://(www\.)?bing\.com/`,
},
};
const BRAVE: EngineDefinition = {
id: "brave",
label: "Brave Search",
note: "independent index, direct links; challenges after 1-2 queries, clears in ~15-20 min",
// Left empty deliberately — see the note in README. Brave's filter UI is
// client-rendered and the `tf=` values could not be confirmed against a page
// that was not simultaneously serving a bot check, so no window is claimed
// rather than one being guessed at.
recency: {},
buildUrl: (query, _numResults, _recency) => {
const params = new URLSearchParams({ q: query });
return `https://search.brave.com/search?${params.toString()}`;
},
extraction: {
mode: "items",
// `#results` is gone from the live page (2026-11-06); `#main` and `main`
// wrap the results. Left first so the older layout still matches.
roots: ["#results", "#main", "main"],
// data-type="web" excludes the AI summariser, video and news cards, which
// share the .snippet class but are not ranked web results.
item: '.snippet[data-type="web"]',
link: "a[href]",
title: ".title",
// No dedicated description element. The item's text is
// "source | breadcrumb | title | description", so the extractor keeps only
// the lines after the title. These selectors are the fallback for when the
// title is not on a line of its own.
subtract: [".title", "cite"],
unwrap: "none",
selfHostPattern: String.raw`^https?://(search\.)?brave\.com/`,
},
};
const BY_ID: Record<EngineId, EngineDefinition> = {
google: GOOGLE,
duckduckgo: DUCKDUCKGO,
bing: BING,
brave: BRAVE,
};
export const getEngine = (id: EngineId): EngineDefinition => BY_ID[id];
export const allEngines = (): readonly EngineDefinition[] => ENGINE_IDS.map((id) => BY_ID[id]);
/** Parse a user- or model-supplied engine name. Null when unrecognised. */
export const parseEngineId = (raw: string): EngineId | null =>
ENGINE_ALIASES[raw.trim().toLowerCase()] ?? null;
/** The names accepted for an engine, for help text and error messages. */
export const engineAliases = (): readonly string[] => Object.keys(ENGINE_ALIASES);
/** Which engines can express a given recency window. */
export const enginesSupportingRecency = (recency: Recency): readonly EngineId[] =>
ENGINE_IDS.filter((id) => BY_ID[id].recency[recency] !== undefined);
/** Human-readable list of the windows an engine supports, or "none". */
export const describeRecencySupport = (engine: EngineDefinition): string => {
const windows = RECENCY_VALUES.filter((r) => engine.recency[r] !== undefined);
return windows.length > 0 ? windows.join(", ") : "none";
};
|