Browser Automation

1234567891011121314151617181920212223242526272829import { Image, Snapshots } from "@opencomputer/sdk/node";

// Runtime deps Chromium links against on Ubuntu 22.04. Matches Playwright's
// published dependency list.
const CHROMIUM_DEPS = [
  "libnss3", "libnspr4", "libatk1.0-0", "libatk-bridge2.0-0", "libcups2",
  "libdrm2", "libxkbcommon0", "libxcomposite1", "libxdamage1", "libxfixes3",
  "libxrandr2", "libxext6", "libgbm1", "libpango-1.0-0", "libcairo2",
  "libasound2", "fonts-liberation",
  // libnss3-tools gives us `certutil` — needed at runtime to trust OC's
  // egress-proxy CA in Chromium (Chromium ignores SSL_CERT_FILE).
  "libnss3-tools",
];

const image = Image.base()
  .aptInstall(CHROMIUM_DEPS)
  .workdir("/home/sandbox")
  .runCommands(
    "cd /home/sandbox && npm init -y >/dev/null",
    // Install libretto + AI adapter locally so `npx libretto` resolves without
    // a cold npm fetch at runtime.
    "cd /home/sandbox && npm install --no-audit --no-fund libretto @ai-sdk/anthropic",
    // Playwright downloads its own Chromium build — we want headless-shell
    // for lightweight scraping, or full chromium for headed/VNC scenarios.
    "cd /home/sandbox && npx --yes playwright install chromium-headless-shell",
  );

const snapshots = new Snapshots();
await snapshots.create({ name: "browser", image });

12345678910111213141516171819202122232425262728293031323334import { Sandbox, SecretStore } from "@opencomputer/sdk/node";

const sandbox = await Sandbox.create({
  snapshot: "browser",
  envs: {
    ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY!,
    // Keep Playwright/Chromium off the small tmpfs /dev/shm and /tmp —
    // write profile dirs, cache, everything to the data disk.
    TMPDIR: "/home/sandbox/tmp",
  },
  secretStore: "browser-egress",  // see "Networking" section below
});

// Populate /etc/hosts — the guest kernel sets up /etc/resolv.conf but not
// /etc/hosts, and libretto's Playwright CDP client hardcodes http://localhost
// which will ENOTFOUND without this.
await sandbox.commands.run(
  "grep -q 'localhost' /etc/hosts || " +
    "(printf '127.0.0.1 localhost\\n::1 localhost\\n' | sudo tee -a /etc/hosts)",
);

// Trust OC's egress-proxy CA in Chromium's NSS store. Chromium ignores
// SSL_CERT_FILE / NODE_EXTRA_CA_CERTS — the env vars OC sets for libraries
// have no effect on the browser's TLS validation.
await sandbox.commands.run(
  [
    "mkdir -p /home/sandbox/.pki/nssdb",
    "certutil -d sql:/home/sandbox/.pki/nssdb -N --empty-password || true",
    "certutil -d sql:/home/sandbox/.pki/nssdb -A -n opensandbox-proxy -t 'TC,C,T' " +
      "-i /usr/local/share/ca-certificates/opensandbox-proxy.crt",
  ].join(" && "),
);

await sandbox.commands.run("mkdir -p /home/sandbox/tmp && chmod 700 /home/sandbox/tmp");

12345678910111213141516import { SecretStore } from "@opencomputer/sdk/node";

const stores = await SecretStore.list();
let store = stores.find((s) => s.name === "browser-egress");
if (!store) {
  store = await SecretStore.create({
    name: "browser-egress",
    egressAllowlist: ["*"],  // wildcard — scope this down for production
  });
}
await SecretStore.setSecret(store.id, "PLACEHOLDER", "not-used-just-triggers-session");

const sandbox = await Sandbox.create({
  snapshot: "browser",
  secretStore: "browser-egress",
});

123456789101112131415161718import { execFile } from "node:child_process";
import { promisify } from "node:util";
const run = promisify(execFile);

// Open a page in a named session — libretto persists cookies/localStorage
// per session name in .libretto/sessions/<name>/.
await run("npx", ["libretto", "open", "https://example.com", "--session", "demo", "--headless"], {
  cwd: "/home/sandbox",
});

// AI snapshot — Claude analyzes the page and returns a summary + selectors.
const { stdout } = await run("npx", [
  "libretto", "snapshot", "--session", "demo",
  "--objective", "Describe the main content and interactive elements.",
  "--context", "Freshly loaded page",
], { cwd: "/home/sandbox", maxBuffer: 8 * 1024 * 1024 });

console.log(stdout);

123456789101112131415161718await sandbox.commands.run(
  "sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq xvfb x11vnc novnc websockify",
);

// Start Xvfb → x11vnc → websockify as long-lived exec sessions so they
// persist across individual commands.
const xvfb = await sandbox.exec.start("Xvfb", {
  args: [":99", "-screen", "0", "1280x800x24", "-ac"],
});
await new Promise((r) => setTimeout(r, 1500));

const x11vnc = await sandbox.exec.start("x11vnc", {
  args: ["-display", ":99", "-forever", "-shared", "-nopw", "-rfbport", "5900", "-quiet"],
});

const websockify = await sandbox.exec.start("websockify", {
  args: ["--web=/usr/share/novnc/", "6080", "localhost:5900"],
});

1234await run("npx", ["libretto", "open", "https://app.example.com", "--session", "login", "--headed"], {
  cwd: "/home/sandbox",
  env: { ...process.env, DISPLAY: ":99" },
});

1234await run("npx", ["libretto", "save", "app.example.com", "--session", "login"], {
  cwd: "/home/sandbox",
});
// → writes /home/sandbox/.libretto/profiles/app.example.com.json