Spaces:
Running
Running
File size: 8,938 Bytes
1b44660 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 |
import { err, ok } from 'neverthrow';
import { z } from 'zod';
import type { Env } from '../index';
import { parseArticle } from './parsers';
import { tryCatchAsync } from './tryCatchAsync';
import { userAgents } from './utils';
/**
* Schema for validating responses from the Cloudflare Browser Rendering API
*/
export const articleSchema = z.object({
status: z.coerce.boolean(),
errors: z.array(z.object({ code: z.number(), message: z.string() })).optional(),
result: z.string(),
});
/**
* Fetches an article using Cloudflare's Browser Rendering API
*
* This method simulates a real browser to handle modern websites with complex
* JavaScript, cookie consent walls, paywalls, and other obstacles that might
* prevent content scraping with a regular HTTP client.
*
* @param env Application environment with Cloudflare credentials
* @param url URL of the article to fetch
* @returns Result containing either the parsed article content or an error object
*/
export async function getArticleWithBrowser(env: Env, url: string) {
const response = await tryCatchAsync(
fetch(`https://api.cloudflare.com/client/v4/accounts/${env.CLOUDFLARE_ACCOUNT_ID}/browser-rendering/content`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}`,
},
body: JSON.stringify({
url,
userAgent: userAgents[Math.floor(Math.random() * userAgents.length)],
setExtraHTTPHeaders: {
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
Connection: 'keep-alive',
DNT: '1',
'Accept-Language': 'en-US,en;q=0.5',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
},
cookies: [],
gotoOptions: {
waitUntil: 'networkidle0',
timeout: 30000,
referer: 'https://www.google.com/',
},
viewport: {
width: 390,
height: 844,
deviceScaleFactor: 3,
isMobile: true,
hasTouch: true,
isLandscape: false,
},
rejectResourceTypes: ['image', 'media', 'font', 'websocket'],
bestAttempt: true,
// all of these are very brittle, like all script tag usage
// this mostly works for now but good to revisit every once in a while
addScriptTag: [
// Ensures consistent date formatting by overriding Intl.DateTimeFormat
// to always use 'en-US' locale regardless of browser settings
// This prevents inconsistent date parsing across different environments
{
content:
"(() => { Object.defineProperty(Intl, 'DateTimeFormat', { \n writable: true, \n value: new Proxy(Intl.DateTimeFormat, { \n construct: (target, args) => new target('en-US', Object.assign({}, args[1])) \n })\n }); })();",
},
// Automatically accepts cookie consent popups by finding buttons that contain
// 'accept' and 'cookie'/'consent' text, then programmatically clicking the first match
// This bypasses cookie walls that would otherwise block content access
{
content:
"(() => { const cookieButtons = Array.from(document.querySelectorAll(\'button, a\')).filter(el => el.textContent.toLowerCase().includes(\'accept\') && (el.textContent.toLowerCase().includes(\'cookie\') || el.textContent.toLowerCase().includes(\'consent\'))); if(cookieButtons.length > 0) { cookieButtons[0].click(); } })();",
},
// Circumvents paywalls by:
// 1. Removing elements with paywall/subscribe identifiers in id/class
// 2. Removing modal overlays and fixed position barriers
// 3. Restoring normal page scroll behavior
// This targets common paywall implementations across various sites
{
content:
"(() => { const paywallElements = Array.from(document.querySelectorAll(\'div, section\')).filter(el => el.id.toLowerCase().includes(\'paywall\') || el.className.toLowerCase().includes(\'paywall\') || el.id.toLowerCase().includes(\'subscribe\') || el.className.toLowerCase().includes(\'subscribe\')); paywallElements.forEach(el => el.remove()); document.querySelectorAll(\'.modal, .modal-backdrop, body > div[style*=\"position: fixed\"]\').forEach(el => el.remove()); document.body.style.overflow = \'auto\'; })();",
},
// Cleans up the DOM by removing non-content elements that interfere with article parsing:
// - Scripts, styles, iframes that might contain tracking or ads
// - Ad containers and advertisement blocks
// - Social media widgets and sharing buttons
// - Comments sections, navbars, headers, footers (except those within articles)
// - Various UI elements not relevant to the core article content
{
content:
'(() => { document.querySelectorAll(\'script, style, iframe, .ad, .ads, .advertisement, [class*="social"], [id*="social"], .share, .comments, aside, nav, header:not(article header), footer:not(article footer), [role="complementary"], [role="banner"], [role="navigation"], form, .related, .recommended, .newsletter, .subscription\').forEach(el => el.remove()); })();',
},
// Simplifies the DOM by stripping all HTML attributes except essential ones:
// - href: preserves links
// - src: maintains images and embedded content
// - alt: keeps accessibility text for images
// - title: retains tooltip text
// This reduces noise and potential tracking parameters in the parsed content
{
content:
"(() => { const keepAttributes = [\'href\', \'src\', \'alt\', \'title\']; document.querySelectorAll(\'*\').forEach(el => { [...el.attributes].forEach(attr => { if (!keepAttributes.includes(attr.name.toLowerCase())) { el.removeAttribute(attr.name); }}); }); })();",
},
// Recursively removes empty elements to clean up the DOM structure
// Continues removing elements until no more empty ones are found
// This eliminates spacing artifacts and layout containers that serve no content purpose
{
content:
"(() => { function removeEmpty() { let removed = 0; document.querySelectorAll(\'div, span, p, section, article\').forEach(el => { if (!el.hasChildNodes() || el.textContent.trim() === \'\') { el.remove(); removed++; } }); return removed; } let pass; do { pass = removeEmpty(); } while(pass > 0); })();",
},
// Removes simple meta tags that provide minimal information value
// Meta tags with only one attribute are typically not useful for content analysis
// This helps reduce noise in the document head
{
content:
"(() => { document.querySelectorAll(\'meta\').forEach(meta => { if (meta.attributes.length <= 1) { meta.remove(); } }); })();",
},
],
waitForSelector: {
selector: 'article, .article, .content, .post, #article, main',
timeout: 5000,
},
}),
})
);
if (response.isErr()) {
return err({ type: 'FETCH_ERROR', error: response.error });
}
const parsedPageContent = articleSchema.safeParse(await response.value.json());
if (parsedPageContent.success === false) {
return err({ type: 'VALIDATION_ERROR', error: parsedPageContent.error });
}
const articleResult = parseArticle({ html: parsedPageContent.data.result });
if (articleResult.isErr()) {
return err({ type: 'PARSE_ERROR', error: articleResult.error });
}
return ok(articleResult.value);
}
/**
* Fetches an article using a simple HTTP request
*
* This is a lighter-weight alternative to browser rendering that works for
* simpler websites that don't rely heavily on client-side JavaScript for content.
*
* @param url URL of the article to fetch
* @returns Result containing either the parsed article content or an error object
*/
export async function getArticleWithFetch(url: string) {
const response = await tryCatchAsync(
fetch(url, {
method: 'GET',
headers: {
'User-Agent': userAgents[Math.floor(Math.random() * userAgents.length)],
Referer: 'https://www.google.com/',
},
})
);
if (response.isErr()) {
return err({ type: 'FETCH_ERROR', error: response.error });
}
const articleResult = parseArticle({ html: await response.value.text() });
if (articleResult.isErr()) {
return err({ type: 'PARSE_ERROR', error: articleResult.error });
}
return ok(articleResult.value);
}
|