platform/src/servers/api/scrape/scrape.ts

import { createRouter } from '../../create-router';
import { mkdir } from 'node:fs/promises';
import { join } from 'node:path';
import { chromium, type Browser } from 'playwright';
import { getTmpAttachmentsDir, getAttachmentsDir } from '@@/data-path';

const MAX_CONTENT_LENGTH = 100_000;

let browserPromise: Promise<Browser> | null = null;

function getBrowser(): Promise<Browser> {
  if (!browserPromise) {
    browserPromise = chromium.launch({ headless: true }).catch((err) => {
      browserPromise = null;
      throw err;
    });
  }
  return browserPromise;
}

function slugify(text: string): string {
  return text
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, '-')
    .replace(/^-+|-+$/g, '')
    .slice(0, 80);
}

export const scrapeRouter = createRouter();

scrapeRouter.post('/', async (ctx) => {
  const user = ctx.get('user');
  const { url, sessionId, provider } = ctx.get('body') as {
    url: string;
    sessionId?: string;
    provider?: 'claude' | 'opencode' | 'pi-mono';
  };

  if (!url) return ctx.json({ error: 'url is required' }, 400);

  const browser = await getBrowser();
  const page = await browser.newPage();

  try {
    await page.goto(url, { waitUntil: 'networkidle', timeout: 30_000 });

    // Scroll through the page to capture all content, including virtualized/lazy-loaded
    // content (e.g. ChatGPT shared links remove DOM nodes as you scroll past them).
    // We capture text incrementally at each scroll position and merge it.
    const { accumulatedText, scrollableFound } = await page.evaluate(async () => {
      const delay = (ms: number) => new Promise((r) => setTimeout(r, ms));

      // Find the deepest scrollable container (the one that actually scrolls content)
      let bestScrollable: Element | null = null;
      let bestOverflow = 0;
      const elements = Array.from(document.querySelectorAll('*'));
      for (const el of elements) {
        const style = getComputedStyle(el);
        const overflowY = style.overflowY;
        if ((overflowY === 'auto' || overflowY === 'scroll') && el.scrollHeight > el.clientHeight + 10) {
          const overflow = el.scrollHeight - el.clientHeight;
          if (overflow > bestOverflow) {
            bestOverflow = overflow;
            bestScrollable = el;
          }
        }
      }

      const scrollable = bestScrollable ?? document.scrollingElement;
      if (!scrollable || scrollable.scrollHeight <= scrollable.clientHeight + 10) {
        return { accumulatedText: '', scrollableFound: false };
      }

      // Collect text chunks as we scroll through
      const seenChunks = new Set<string>();
      const orderedChunks: string[] = [];

      const captureVisible = () => {
        const text = document.body.innerText;
        // Split into paragraphs and capture new ones
        const paragraphs = text.split(/\n{2,}/);
        for (const p of paragraphs) {
          const trimmed = p.trim();
          if (trimmed && !seenChunks.has(trimmed)) {
            seenChunks.add(trimmed);
            orderedChunks.push(trimmed);
          }
        }
      };

      // Start from the top
      scrollable.scrollTop = 0;
      await delay(500);
      captureVisible();

      // Scroll incrementally, capturing at each position
      let stableCount = 0;
      let lastChunkCount = orderedChunks.length;
      for (let i = 0; i < 200; i++) {
        scrollable.scrollTop += scrollable.clientHeight * 0.6;
        await delay(300);
        captureVisible();

        // Check if we're at the bottom
        const atBottom = scrollable.scrollTop + scrollable.clientHeight >= scrollable.scrollHeight - 5;
        if (atBottom) {
          // Wait a bit for potential dynamic loading
          await delay(500);
          captureVisible();
          // If no new content appeared, we're done
          if (orderedChunks.length === lastChunkCount) {
            stableCount++;
            if (stableCount >= 2) break;
          } else {
            stableCount = 0;
            lastChunkCount = orderedChunks.length;
          }
        }
      }

      return { accumulatedText: orderedChunks.join('\n\n'), scrollableFound: true };
    });

    const title = await page.title();
    const html = await page.evaluate(() => document.documentElement.outerHTML);

    // Use accumulated text from scrolling if available, otherwise fall back to current innerText
    const rawText =
      scrollableFound && accumulatedText ? accumulatedText : await page.evaluate(() => document.body.innerText);

    // Strip excessive whitespace and truncate
    const content = rawText
      .replace(/\n{3,}/g, '\n\n')
      .replace(/[ \t]+/g, ' ')
      .trim()
      .slice(0, MAX_CONTENT_LENGTH);

    // Determine save location
    const slug = slugify(title || 'page');
    let attachmentId: string;
    let saveDir: string;

    if (sessionId && provider) {
      attachmentId = `${slug}.html`;
      saveDir = getAttachmentsDir(user.email, provider, sessionId);
    } else {
      attachmentId = `${crypto.randomUUID()}.html`;
      saveDir = getTmpAttachmentsDir(user.email);
    }

    await mkdir(saveDir, { recursive: true });
    await Bun.write(join(saveDir, attachmentId), html);

    return ctx.json({ url, title, content, attachmentId });
  } catch (err) {
    const message = err instanceof Error ? err.message : 'Scrape failed';
    return ctx.json({ error: message }, 500);
  } finally {
    await page.close();
  }
});