Web Crawler with Puppeteer and React

   cd backend

   npm install

   node server.js

   cd frontend

   npm install

   npm start

const express = require("express");
const puppeteer = require("puppeteer");
const app = express();
const cors = require("cors");

app.use(express.json());
app.use(cors());

app.post("/crawl-site", async (req, res) => {
  const { url } = req.body;

  let browser;
  try {
    browser = await puppeteer.launch({
      headless: true,
      args: ["--no-sandbox", "--disable-setuid-sandbox"],
    });

    const page = await browser.newPage();
    await page.setUserAgent(
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
    );

    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60000 });
    await page.waitForSelector("body", { timeout: 10000 });

    const links = await page.evaluate(() => {
      const anchors = Array.from(document.querySelectorAll("a"));
      return anchors
        .map((anchor) => anchor.href)
        .filter((href) => href.includes(window.location.origin));
    });

    await browser.close();
    res.json({ pages: links });
  } catch (error) {
    console.error(`Navigation to ${url} failed:`, error);
    if (browser) await browser.close();
    res
      .status(500)
      .json({ error: `A server error has occurred: ${error.message}` });
  }
});

app.post("/start-crawl", async (req, res) => {
  const { pages } = req.body;
  let browser;
  try {
    browser = await puppeteer.launch({
      headless: true,
      args: ["--no-sandbox", "--disable-setuid-sandbox"],
    });

    const results = [];
    for (const pageUrl of pages) {
      const page = await browser.newPage();
      try {
        await page.goto(pageUrl, { waitUntil: "networkidle2", timeout: 60000 });
        const content = await page.content();
        results.push({ url: pageUrl, content });
      } catch (error) {
        console.error(`Navigation to ${pageUrl} failed:`, error);
        results.push({ url: pageUrl, error: `Failed to load ${pageUrl}` });
      }
      await page.close();
    }

    await browser.close();
    res.json({ data: results });
  } catch (error) {
    console.error("Failed to launch browser:", error);
    if (browser) await browser.close();
    res.status(500).json({ error: "Failed to launch browser" });
  }
});

const server = app.listen(8000, () =>
  console.log("Backend server started on port 8000")
);

import React from 'react';
import CrawlPage from './components/CrawlPage';

function App() {
  return (
    <div className="App">
      <CrawlPage />
    </div>
  );
}

export default App;

import React, { useState } from 'react';

const CrawlPage = () => {
  const [url, setUrl] = useState('');
  const [pages, setPages] = useState([]);
  const [selectedPages, setSelectedPages] = useState([]);
  const [crawledData, setCrawledData] = useState([]);
  const [loading, setLoading] = useState(false);
  const [error, setError] = useState('');

  // Function to fetch internal links from the backend
  const fetchPages = async () => {
    try {
      setLoading(true);
      const response = await fetch('http://localhost:8000/crawl-site', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({ url }),
      });
      const data = await response.json();
      if (response.ok) {
        setPages(data.pages);
        setError('');
      } else {
        setError(data.error || 'Failed to fetch pages');
      }
    } catch (err) {
      setError('Failed to fetch pages');
    } finally {
      setLoading(false);
    }
  };

  // Function to handle the selection of pages
  const handlePageSelection = (pageUrl) => {
    setSelectedPages((prevSelected) =>
      prevSelected.includes(pageUrl)
        ? prevSelected.filter((url) => url !== pageUrl)
        : [...prevSelected, pageUrl]
    );
  };

  // Function to start crawling the selected pages
  const startCrawling = async () => {
    try {
      setLoading(true);
      const response = await fetch('http://localhost:8000/start-crawl', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({ pages: selectedPages }),
      });
      const data = await response.json();
      if (response.ok) {
        setCrawledData(data.data);
        setError('');
      } else {
        setError(data.error || 'Failed to crawl pages');
      }
    } catch (err) {
      setError('Failed to crawl pages');
    } finally {
      setLoading(false);
    }
  };

  return (
    <div className="container mx-auto p-6">
      <h1 className="text-3xl font-bold mb-6">Web Crawler</h1>

      <div className="mb-4">
        <input
          type="text"
          value={url}
          onChange={(e) => setUrl(e.target.value)}
          placeholder="Enter website URL"
          className="p-2 border border-gray-300 rounded w-full"
        />
      </div>

      <button
        onClick={fetchPages}
        className="bg-blue-500 text-white p-2 rounded"
        disabled={!url || loading}
      >
        {loading ? 'Fetching...' : 'Fetch Pages'}
      </button>

      {error && <p className="text-red-500 mt-4">{error}</p>}

      {pages.length > 0 && (
        <div className="mt-6">
          <h2 className="text-xl font-semibold">Select Pages to Crawl</h2>
          <ul className="list-disc ml-5 mt-4">
            {pages.map((pageUrl) => (
              <li key={pageUrl}>
                <label>
                  <input
                    type="checkbox"
                    checked={selectedPages.includes(pageUrl)}
                    onChange={() => handlePageSelection(pageUrl)}
                  />
                  <span className="ml-2">{pageUrl}</span>
                </label>
              </li>
            ))}
          </ul>
          <button
            onClick={startCrawling}
            className="bg-green-500 text-white p-2 mt-4 rounded"
            disabled={selectedPages.length === 0 || loading}
          >
            {loading ? 'Crawling...' : 'Start Crawling'}
          </button>
        </div>
      )}

      {crawledData.length > 0 && (
        <div className="mt-8">
          <h2 className="text-xl font-semibold">Crawled Data</h2>
          {crawledData.map((data, index) => (
            <div key={index} className="bg-gray-100 p-4 mt-4 rounded">
              <h3 className="text-lg font-bold mb-2">{data.url}</h3>
              <pre className="whitespace-pre-wrap">
                {data.error || data.content}
              </pre>
            </div>
          ))}
        </div>
      )}
    </div>
  );
};

export default CrawlPage;

   cd frontend
   npm install

   npm start