playwright2

Running

App Files Files Community

deepak191z commited on May 26

Commit

d751bce

verified ·

1 Parent(s): 45bc5e7

Update index.js

Browse files

Files changed (1) hide show

index.js +82 -118

index.js CHANGED Viewed

@@ -1,132 +1,96 @@
-const express = require('express');
-const { chromium } = require('playwright');
-const bodyParser = require('body-parser');
-const cors = require('cors');
 const app = express();
-app.use(bodyParser.urlencoded({ extended: true }));
-app.use(bodyParser.json());
-app.use(cors());
-const html = `
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>YouTube Transcript Generator (Playwright)</title>
-    <style>
-        body {
-            font-family: Arial, sans-serif;
-            max-width: 800px;
-            margin: 0 auto;
-            padding: 20px;
-        }
-        form {
-            display: flex;
-            flex-direction: column;
-        }
-        input, button {
-            margin: 10px 0;
-            padding: 5px;
-        }
-        #result {
-            white-space: pre-wrap;
-            background-color: #f0f0f0;
-            padding: 10px;
-            border-radius: 5px;
-        }
-    </style>
-</head>
-<body>
-    <h1>YouTube Transcript Generator (Playwright)</h1>
-    <form id="transcriptForm">
-        <input type="text" id="videoUrl" name="videoUrl" placeholder="YouTube Video URL" required>
-        <input type="text" id="videoTitle" name="videoTitle" placeholder="Video Title" required>
-        <button type="submit">Generate Transcript</button>
-    </form>
-    <div id="result"></div>
-    <script>
-        document.getElementById('transcriptForm').addEventListener('submit', async (e) => {
-            e.preventDefault();
-            const videoUrl = document.getElementById('videoUrl').value;
-            const videoTitle = document.getElementById('videoTitle').value;
-            const resultDiv = document.getElementById('result');
-            resultDiv.textContent = 'Generating transcript...';
-            try {
-                const response = await fetch('/extract-transcript', {
-                    method: 'POST',
-                    headers: {
-                        'Content-Type': 'application/json',
-                    },
-                    body: JSON.stringify({ videoUrl, videoTitle }),
-                });
-                if (response.ok) {
-                    const data = await response.json();
-                    resultDiv.textContent = data.transcript;
-                } else {
-                    resultDiv.textContent = 'Error generating transcript. Please try again.';
-                }
-            } catch (error) {
-                console.error('Error:', error);
-                resultDiv.textContent = 'An error occurred. Please try again.';
-            }
-        });
-    </script>
-</body>
-</html>
-`;
-app.get('/', (req, res) => {
-    res.send(html);
 });
-app.post('/extract-transcript', async (req, res) => {
-    const { videoUrl, videoTitle } = req.body;
-    if (!videoUrl || !videoTitle) {
-        return res.status(400).send('videoUrl and videoTitle are required');
-    }
-    const browser = await chromium.launch();
-    const context = await browser.newContext();
-    const page = await context.newPage();
-    try {
-        await page.goto(videoUrl, { waitUntil: 'networkidle' });
-        // Set viewport size
-        await page.setViewportSize({ width: 1920, height: 1080 });
-        // Click the "Expand" button to expand the video description
-        await page.click('tp-yt-paper-button#expand');
-        // Wait for the "Show transcript" button and click it
-        await page.click('button[aria-label="Show transcript"]');
-        // Wait for the transcript container to appear
-        await page.waitForSelector('ytd-transcript-segment-list-renderer');
-        // Extract the transcript text
-        const transcript = await page.evaluate(() => {
-            const elements = Array.from(document.querySelectorAll('ytd-transcript-segment-renderer .segment-text'));
-            return elements.map(element => element.innerText).join('\n');
-        });
-        res.json({ transcript });
-    } catch (error) {
-        console.error('Error extracting transcript:', error);
-        res.status(500).send('Error extracting transcript');
-    } finally {
-        await browser.close();
-    }
 });
-const PORT = 7860;
 app.listen(PORT, () => {
-    console.log(`Server is running on port ${PORT}`);
-});

+const express = require("express");
+const { chromium } = require("playwright");
 const app = express();
+const PORT = process.env.PORT || 7860;
+let browser;
+// Launch Playwright browser once at startup
+(async () => {
+  browser = await chromium.launch({
+    headless: true,
+    args: [
+      "--no-sandbox",
+      "--disable-setuid-sandbox",
+      "--disable-dev-shm-usage"
+    ]
+  });
+})();
+// Core scraping function
+async function scrapeChannelVideos(channelName) {
+  const url = `https://www.youtube.com/@${channelName}/videos`;
+  const context = await browser.newContext({
+    userAgent:
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
+      "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+  });
+  const page = await context.newPage();
+  try {
+    await page.goto(url, { waitUntil: "networkidle", timeout: 20000 });
+    await page.waitForSelector('a#video-title-link', { timeout: 10000 });
+    const videos = await page.$$eval("#dismissible", (els) =>
+      els.slice(0, 3).map((el) => {
+        const anchor = el.querySelector('a#video-title-link');
+        const img = el.querySelector("img");
+        const meta = el.querySelector("#metadata-line span");
+        const vidUrl = anchor?.href || "";
+        const vidIdMatch = vidUrl.match(/v=([^&]+)/);
+        return {
+          title: anchor?.title || anchor?.textContent.trim() || "",
+          videoId: vidIdMatch?.[1] || null,
+          url: vidUrl,
+          thumbnail:
+            vidIdMatch && `https://img.youtube.com/vi/${vidIdMatch[1]}/maxresdefault.jpg`,
+          published: meta?.textContent.trim() || ""
+        };
+      })
+    );
+    await context.close();
+    return videos;
+  } catch (err) {
+    await context.close();
+    throw err;
+  }
+}
+// Home route
+app.get("/", (req, res) => {
+  res.json({
+    message: "Welcome to the YouTube Video Scraper API",
+    docs: "/api",
+    example: "/api/video/MrBeast"
+  });
 });
+// API landing/documentation route
+app.get("/api", (req, res) => {
+  res.json({
+    endpoints: [
+      {
+        route: "/api/video/:channelName",
+        method: "GET",
+        description: "Scrape the latest 3 videos for a given YouTube channel"
+      }
+    ]
+  });
+});
+// Dynamic video-scraping endpoint
+app.get("/api/video/:channelName", async (req, res) => {
+  try {
+    const channel = req.params.channelName;
+    const videos = await scrapeChannelVideos(channel);
+    res.json({ channel, videos, timestamp: new Date().toISOString() });
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
 });
 app.listen(PORT, () => {
+  console.log(`Server listening on port ${PORT}`);
+});