src/tools/read.ts

import https from 'https';
import { TokenTracker } from "../utils/token-tracker";
import { ReadResponse } from '../types';
import { JINA_API_KEY } from "../config";

export function readUrl(url: string, tracker?: TokenTracker): Promise<{ response: ReadResponse }> {
  return new Promise((resolve, reject) => {
    if (!url.trim()) {
      reject(new Error('URL cannot be empty'));
      return;
    }

    const data = JSON.stringify({ url });

    const options = {
      hostname: 'r.jina.ai',
      port: 443,
      path: '/',
      method: 'POST',
      headers: {
        'Accept': 'application/json',
        'Authorization': `Bearer ${JINA_API_KEY}`,
        'Content-Type': 'application/json',
        'Content-Length': data.length,
        'X-Retain-Images': 'none',
        'X-Return-Format': 'markdown'
      }
    };

    const req = https.request(options, (res) => {
      let responseData = '';

      res.on('data', (chunk) => responseData += chunk);

      res.on('end', () => {
        // Check HTTP status code first
        if (res.statusCode && res.statusCode >= 400) {
          try {
            // Try to parse error message from response if available
            const errorResponse = JSON.parse(responseData);
            if (res.statusCode === 402) {
              reject(new Error(errorResponse.readableMessage || 'Insufficient balance'));
              return;
            }
            reject(new Error(errorResponse.readableMessage || `HTTP Error ${res.statusCode}`));
          } catch (error: unknown) {
            // If parsing fails, just return the status code
            reject(new Error(`HTTP Error ${res.statusCode}`));
          }
          return;
        }

        // Only parse JSON for successful responses
        let response: ReadResponse;
        try {
          response = JSON.parse(responseData) as ReadResponse;
        } catch (error: unknown) {
          reject(new Error(`Failed to parse response: ${error instanceof Error ? error.message : 'Unknown error'}`));
          return;
        }

        if (!response.data) {
          reject(new Error('Invalid response data'));
          return;
        }

        console.log('Read:', {
          title: response.data.title,
          url: response.data.url,
          tokens: response.data.usage?.tokens || 0
        });

        const tokens = response.data.usage?.tokens || 0;
        const tokenTracker = tracker || new TokenTracker();
        tokenTracker.trackUsage('read', {
            totalTokens: tokens,
            promptTokens: url.length,
            completionTokens: tokens
        });

        resolve({ response });
      });
    });

    // Add timeout handling
    req.setTimeout(30000, () => {
      req.destroy();
      reject(new Error('Request timed out'));
    });

    req.on('error', (error: Error) => {
      reject(new Error(`Request failed: ${error.message}`));
    });

    req.write(data);
    req.end();
  });
}

export function removeAllLineBreaks(text: string) {
  return text.replace(/(\r\n|\n|\r)/gm, " ");
}