first

src/servers/api/scrape/scrape.ts

@@ -0,0 +1,161 @@
+import { createRouter } from '../../create-router';
+import { mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { chromium, type Browser } from 'playwright';
+import { getTmpAttachmentsDir, getAttachmentsDir } from '@@/data-path';
+
+const MAX_CONTENT_LENGTH = 100_000;
+
+let browserPromise: Promise<Browser> | null = null;
+
+function getBrowser(): Promise<Browser> {
+  if (!browserPromise) {
+    browserPromise = chromium.launch({ headless: true }).catch((err) => {
+      browserPromise = null;
+      throw err;
+    });
+  }
+  return browserPromise;
+}
+
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 80);
+}
+
+export const scrapeRouter = createRouter();
+
+scrapeRouter.post('/', async (ctx) => {
+  const user = ctx.get('user');
+  const { url, sessionId, provider } = ctx.get('body') as {
+    url: string;
+    sessionId?: string;
+    provider?: 'claude' | 'opencode';
+  };
+
+  if (!url) return ctx.json({ error: 'url is required' }, 400);
+
+  const browser = await getBrowser();
+  const page = await browser.newPage();
+
+  try {
+    await page.goto(url, { waitUntil: 'networkidle', timeout: 30_000 });
+
+    // Scroll through the page to capture all content, including virtualized/lazy-loaded
+    // content (e.g. ChatGPT shared links remove DOM nodes as you scroll past them).
+    // We capture text incrementally at each scroll position and merge it.
+    const { accumulatedText, scrollableFound } = await page.evaluate(async () => {
+      const delay = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+      // Find the deepest scrollable container (the one that actually scrolls content)
+      let bestScrollable: Element | null = null;
+      let bestOverflow = 0;
+      const elements = Array.from(document.querySelectorAll('*'));
+      for (const el of elements) {
+        const style = getComputedStyle(el);
+        const overflowY = style.overflowY;
+        if ((overflowY === 'auto' || overflowY === 'scroll') && el.scrollHeight > el.clientHeight + 10) {
+          const overflow = el.scrollHeight - el.clientHeight;
+          if (overflow > bestOverflow) {
+            bestOverflow = overflow;
+            bestScrollable = el;
+          }
+        }
+      }
+
+      const scrollable = bestScrollable ?? document.scrollingElement;
+      if (!scrollable || scrollable.scrollHeight <= scrollable.clientHeight + 10) {
+        return { accumulatedText: '', scrollableFound: false };
+      }
+
+      // Collect text chunks as we scroll through
+      const seenChunks = new Set<string>();
+      const orderedChunks: string[] = [];
+
+      const captureVisible = () => {
+        const text = document.body.innerText;
+        // Split into paragraphs and capture new ones
+        const paragraphs = text.split(/\n{2,}/);
+        for (const p of paragraphs) {
+          const trimmed = p.trim();
+          if (trimmed && !seenChunks.has(trimmed)) {
+            seenChunks.add(trimmed);
+            orderedChunks.push(trimmed);
+          }
+        }
+      };
+
+      // Start from the top
+      scrollable.scrollTop = 0;
+      await delay(500);
+      captureVisible();
+
+      // Scroll incrementally, capturing at each position
+      let stableCount = 0;
+      let lastChunkCount = orderedChunks.length;
+      for (let i = 0; i < 200; i++) {
+        scrollable.scrollTop += scrollable.clientHeight * 0.6;
+        await delay(300);
+        captureVisible();
+
+        // Check if we're at the bottom
+        const atBottom = scrollable.scrollTop + scrollable.clientHeight >= scrollable.scrollHeight - 5;
+        if (atBottom) {
+          // Wait a bit for potential dynamic loading
+          await delay(500);
+          captureVisible();
+          // If no new content appeared, we're done
+          if (orderedChunks.length === lastChunkCount) {
+            stableCount++;
+            if (stableCount >= 2) break;
+          } else {
+            stableCount = 0;
+            lastChunkCount = orderedChunks.length;
+          }
+        }
+      }
+
+      return { accumulatedText: orderedChunks.join('\n\n'), scrollableFound: true };
+    });
+
+    const title = await page.title();
+    const html = await page.evaluate(() => document.documentElement.outerHTML);
+
+    // Use accumulated text from scrolling if available, otherwise fall back to current innerText
+    const rawText =
+      scrollableFound && accumulatedText ? accumulatedText : await page.evaluate(() => document.body.innerText);
+
+    // Strip excessive whitespace and truncate
+    const content = rawText
+      .replace(/\n{3,}/g, '\n\n')
+      .replace(/[ \t]+/g, ' ')
+      .trim()
+      .slice(0, MAX_CONTENT_LENGTH);
+
+    // Determine save location
+    const slug = slugify(title || 'page');
+    let attachmentId: string;
+    let saveDir: string;
+
+    if (sessionId && provider) {
+      attachmentId = `${slug}.html`;
+      saveDir = getAttachmentsDir(user.email, provider, sessionId);
+    } else {
+      attachmentId = `${crypto.randomUUID()}.html`;
+      saveDir = getTmpAttachmentsDir(user.email);
+    }
+
+    await mkdir(saveDir, { recursive: true });
+    await Bun.write(join(saveDir, attachmentId), html);
+
+    return ctx.json({ url, title, content, attachmentId });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Scrape failed';
+    return ctx.json({ error: message }, 500);
+  } finally {
+    await page.close();
+  }
+});