get visible dom for agentic web navigation

getVisibleDOMEvaluate.js

export function getVisibleDOMEvaluate(): object[] {
  const vw = window.innerWidth;
  const vh = window.innerHeight;
  const els = Array.from(document.querySelectorAll("*"));
  const visible: Record<string, unknown>[] = [];

  for (const el of els) {
    const style = getComputedStyle(el);
    if (style.display === "none") continue;
    if (style.visibility === "hidden") continue;
    if (parseFloat(style.opacity) === 0) continue;

    const rect = el.getBoundingClientRect();
    if (rect.width === 0 || rect.height === 0) continue;
    if (rect.bottom < 0 || rect.top > vh) continue;
    if (rect.right < 0 || rect.left > vw) continue;

    const tag = el.tagName.toLowerCase();
    if (
      [
        "html",
        "head",
        "body",
        "script",
        "style",
        "noscript",
        "br",
        "hr",
      ].includes(tag)
    )
      continue;

    const text = (el.textContent || "").trim().slice(0, 120);
    const isInteractive =
      [
        "a",
        "button",
        "input",
        "select",
        "textarea",
        "details",
        "summary",
      ].includes(tag) ||
      el.getAttribute("role") === "button" ||
      el.getAttribute("tabindex") != null ||
      (el as HTMLElement).onclick != null;
    const hasText = text.length > 0;
    if (!hasText && !isInteractive) continue;

    const parentText = (el.parentElement?.textContent || "")
      .trim()
      .slice(0, 120);
    const isLeafText =
      !hasText || el.children.length === 0 || text !== parentText;
    if (!isLeafText && !isInteractive) continue;

    const entry: Record<string, unknown> = { tag };
    const role = el.getAttribute("role");
    if (role) entry.role = role;
    const ariaLabel = el.getAttribute("aria-label");
    if (ariaLabel) entry.label = ariaLabel;
    if (tag === "a" && (el as HTMLAnchorElement).href)
      entry.href = (el as HTMLAnchorElement).href;
    if (tag === "img") entry.alt = (el as HTMLImageElement).alt || "";
    if (tag === "input" || tag === "textarea" || tag === "select") {
      entry.type = (el as HTMLInputElement).type || tag;
      entry.name = (el as HTMLInputElement).name || "";
      entry.value = ((el as HTMLInputElement).value || "").slice(0, 80);
      if ((el as HTMLInputElement).placeholder)
        entry.placeholder = (el as HTMLInputElement).placeholder;
    }
    if (hasText && el.children.length === 0) entry.text = text;
    if (isInteractive) entry.interactive = true;

    const zIndex = parseInt(style.zIndex);
    if (!isNaN(zIndex) && zIndex !== 0) entry.z = zIndex;

    entry.box = [
      Math.round(rect.left),
      Math.round(rect.top),
      Math.round(rect.width),
      Math.round(rect.height),
    ];

    visible.push(entry);
  }

  visible.sort(
    (a, b) =>
      ((b.z as number) || 0) - ((a.z as number) || 0) ||
      (a.box as number[])[1] - (b.box as number[])[1],
  );

  return visible;
}