import { EnqueueStrategy, PuppeteerCrawler } from "crawlee";
import express from "express";
import { Page } from "puppeteer-core";
import { getExecutablePath } from "./executable-path";

const app = express();
app.listen(process.env.PORT || 4100);

const startCrawl = async (props: { startUrl: string; onRequest: (page: Page) => Promise<void> }) => {
  const crawler = new PuppeteerCrawler({
    requestHandlerTimeoutSecs: 120,
    launchContext: { launchOptions: { executablePath: getExecutablePath() } },
    maxConcurrency: 10,
    async requestHandler({ enqueueLinks, log, request, page }) {
      await props.onRequest(page);
      log.info(`Added: ${request.url}`);
      await enqueueLinks({ strategy: EnqueueStrategy.SameDomain });
    },
  });
  await crawler.run([props.startUrl]);
};