// PRE Web GUI — Web tools (web_fetch, web_search)
// Uses Node.js native https — no curl dependency, fully cross-platform.

const https = require('https');
const http = require('http');
const fs = require('fs');
const path = require('path');
const { CONNECTIONS_FILE, ARTIFACTS_DIR } = require('../constants');
const { htmlToText } = require('../platform');
const { parseVtt, looksLikeVtt } = require('../vtt');

// pdf-parse is loaded lazily — it pulls in pdfjs-dist (~2MB) and we'd rather
// not pay that on every server boot when most fetches aren't PDFs.
let _PDFParse = null;
function _loadPdfParse() {
  if (_PDFParse) return _PDFParse;
  _PDFParse = require('pdf-parse').PDFParse;
  return _PDFParse;
}

// PDF magic bytes: every valid PDF starts with "%PDF-" (0x25 0x50 0x44 0x46 0x2D).
// We check both Content-Type and the bytes because servers sometimes mis-label
// (text/html for static .pdf assets is surprisingly common).
function _looksLikePdf(buffer, headers) {
  const ctype = (headers?.['content-type'] || '').toLowerCase();
  if (ctype.includes('application/pdf')) return true;
  if (buffer && buffer.length >= 5 && buffer.slice(0, 5).toString('ascii') === '%PDF-') return true;
  return false;
}

async function _extractPdfText(buffer) {
  const PDFParse = _loadPdfParse();
  const parser = new PDFParse({ data: new Uint8Array(buffer) });
  try {
    const result = await parser.getText();
    let info = null;
    try { info = await parser.getInfo({ parsePageInfo: false }); } catch {}
    return {
      text: (result?.text || '').trim(),
      pages: info?.total || null,
      title: info?.info?.Title || null,
      author: info?.info?.Author || null,
    };
  } finally {
    try { await parser.destroy(); } catch {}
  }
}

function getConnectionKey(name) {
  try {
    const data = JSON.parse(fs.readFileSync(CONNECTIONS_FILE, 'utf-8'));
    switch (name) {
      case 'brave_search': return data.brave_search_key || null;
      case 'github': return data.github_key || null;
      case 'wolfram': return data.wolfram_key || null;
      default: return null;
    }
  } catch {
    return null;
  }
}

/**
 * Perform an HTTP(S) GET request using Node.js native modules.
 * Follows redirects (up to 5), returns { data, headers } or throws.
 */
function httpGet(url, headers = {}, maxRedirects = 5) {
  return new Promise((resolve, reject) => {
    const mod = url.startsWith('https') ? https : http;
    const req = mod.get(url, { headers, timeout: 15000 }, (res) => {
      // Follow redirects
      if ([301, 302, 303, 307, 308].includes(res.statusCode) && res.headers.location) {
        if (maxRedirects <= 0) return reject(new Error('Too many redirects'));
        let loc = res.headers.location;
        if (loc.startsWith('/')) {
          const u = new URL(url);
          loc = `${u.protocol}//${u.host}${loc}`;
        }
        res.resume();
        return resolve(httpGet(loc, headers, maxRedirects - 1));
      }
      const chunks = [];
      res.on('data', (chunk) => chunks.push(chunk));
      res.on('end', () => resolve({ data: Buffer.concat(chunks), headers: res.headers, statusCode: res.statusCode }));
      res.on('error', reject);
    });
    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('Request timed out')); });
  });
}

/**
 * Check if a URL points to an image based on extension or explicit save_image flag.
 */
function isImageUrl(url) {
  const imageExts = /\.(png|jpg|jpeg|gif|webp|svg|bmp|ico)(\?|$)/i;
  return imageExts.test(url);
}

async function webFetch(args) {
  const url = args.url;
  if (!url) return 'Error: no url provided';

  if (!url.startsWith('http://') && !url.startsWith('https://')) {
    return 'Error: URL must start with http:// or https://';
  }

  try {
    const headers = {};

    // Add GitHub auth header if applicable
    if (url.includes('github.com') || url.includes('api.github.com') || url.includes('raw.githubusercontent.com')) {
      const ghKey = getConnectionKey('github');
      if (ghKey) {
        headers['Authorization'] = `token ${ghKey}`;
        headers['Accept'] = 'application/vnd.github+json';
      }
    }

    // If URL is an image or save_image flag is set, download as binary and save locally
    if (isImageUrl(url) || args.save_image) {
      const downloadsDir = path.join(ARTIFACTS_DIR, 'downloads');
      if (!fs.existsSync(downloadsDir)) fs.mkdirSync(downloadsDir, { recursive: true });
      const ext = (url.match(/\.(png|jpg|jpeg|gif|webp|svg)/) || [null, 'png'])[1];
      const fname = `web-${Date.now()}.${ext}`;
      const outPath = path.join(downloadsDir, fname);

      const { data } = await httpGet(url, headers);
      fs.writeFileSync(outPath, data);
      const size = data.length;
      if (size < 100) {
        fs.unlinkSync(outPath);
        return `Error: downloaded file too small (${size} bytes), likely not a valid image`;
      }
      const sizeFmt = size < 1024 ? `${size}B` : `${(size / 1024).toFixed(0)}KB`;
      return `Image downloaded and saved locally.\nPath: /artifacts/downloads/${fname}\nSize: ${sizeFmt}\nUse this path in HTML artifacts: <img src="/artifacts/downloads/${fname}">`;
    }

    // Fetch payload. We get a Buffer back from httpGet so we can sniff binary
    // formats before assuming text — PDFs in particular are byte-decoded into
    // garbage if we go straight to data.toString('utf-8').
    const { data, headers: responseHeaders } = await httpGet(url, headers);

    if (_looksLikePdf(data, responseHeaders)) {
      try {
        const { text, pages, title, author } = await _extractPdfText(data);
        if (!text) {
          return `Fetched PDF from ${url} (${data.length} bytes, ${pages || '?'} pages) but it appears to contain no extractable text (scanned/image-only). Consider using web_search for the document's contents instead.`;
        }
        let header = `[PDF] ${title || url}`;
        if (author) header += ` — ${author}`;
        if (pages) header += ` (${pages} page${pages === 1 ? '' : 's'})`;
        const MAX_PDF_TEXT = 100000; // matches the HTML truncation order of magnitude
        const body = text.length > MAX_PDF_TEXT
          ? text.slice(0, MAX_PDF_TEXT) + `\n\n[... truncated ${text.length - MAX_PDF_TEXT} chars; ask for a specific page or section to read more]`
          : text;
        return `${header}\n\n${body}`;
      } catch (pdfErr) {
        // Fall through to text decoding so the caller still sees something
        // (some malformed PDFs are recoverable, others aren't worth pursuing).
        console.warn(`[web_fetch] PDF extraction failed for ${url}: ${pdfErr.message}`);
        return `Error: detected PDF at ${url} but extraction failed: ${pdfErr.message}. The document may be encrypted, scanned, or malformed.`;
      }
    }

    const rawHtml = data.toString('utf-8').trim();
    if (!rawHtml) return `Error: no content fetched from ${url}`;

    // WebVTT auto-extract. Detection covers both Content-Type (text/vtt) and
    // the WEBVTT signature on the first line, since hosted transcripts often
    // come back as application/octet-stream or text/plain.
    const ctype = (responseHeaders?.['content-type'] || '').toLowerCase();
    if (ctype.includes('text/vtt') || /\.vtt(\?|$)/i.test(url) || looksLikeVtt(rawHtml)) {
      const { text, cueCount, speakers } = parseVtt(rawHtml);
      if (text) {
        let header = `[VTT transcript] ${url}`;
        if (speakers.length) header += ` — speakers: ${speakers.join(', ')}`;
        header += ` (${cueCount} cue${cueCount === 1 ? '' : 's'})`;
        const MAX_VTT_TEXT = 100000;
        const body = text.length > MAX_VTT_TEXT
          ? text.slice(0, MAX_VTT_TEXT) + `\n\n[... truncated ${text.length - MAX_VTT_TEXT} chars]`
          : text;
        return `${header}\n\n${body}`;
      }
      // empty parse — fall through to raw text so the caller still sees something
    }

    const output = htmlToText(rawHtml);
    return output || rawHtml.slice(0, 50000);
  } catch (err) {
    return `Error: failed to fetch ${url}: ${err.message}`;
  }
}

async function webSearch(args) {
  const query = args.query;
  if (!query) return 'Error: no query provided';

  const apiKey = getConnectionKey('brave_search');
  const count = Math.min(Math.max(args.count || 5, 1), 20);

  // Use Brave API if configured, otherwise fall back to DuckDuckGo HTML
  if (apiKey) {
    return braveSearch(query, count, apiKey);
  }
  return duckDuckGoSearch(query, count);
}

async function braveSearch(query, count, apiKey) {
  const encoded = encodeURIComponent(query);
  try {
    const url = `https://api.search.brave.com/res/v1/web/search?q=${encoded}&count=${count}`;
    const { data } = await httpGet(url, {
      'Accept': 'application/json',
      'X-Subscription-Token': apiKey,
    });

    try {
      const json = JSON.parse(data.toString('utf-8'));
      const results = json.web?.results || [];
      if (results.length === 0) return `No search results found for: ${query}`;

      let output = `Search results for: ${query}\n\n`;
      for (let i = 0; i < results.length && i < count; i++) {
        const r = results[i];
        output += `${i + 1}. ${r.title || '(no title)'}\n`;
        output += `   ${r.url || ''}\n`;
        output += `   ${r.description || ''}\n\n`;
      }
      return output;
    } catch {
      return data.toString('utf-8').slice(0, 8000);
    }
  } catch (err) {
    return `Error: search failed: ${err.message}`;
  }
}

async function duckDuckGoSearch(query, count) {
  const encoded = encodeURIComponent(query);
  try {
    const url = `https://html.duckduckgo.com/html/?q=${encoded}`;
    const { data } = await httpGet(url, {
      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    });
    const html = data.toString('utf-8');

    // Parse results from DuckDuckGo HTML response
    const results = [];
    // Each result is in a <div class="result ..."> block
    const resultBlocks = html.match(/<a class="result__a"[^>]*>[\s\S]*?<\/a>[\s\S]*?<a class="result__snippet"[^>]*>[\s\S]*?<\/a>/g)
      || html.match(/<a class="result__a"[^>]*>[\s\S]*?<\/a>[\s\S]*?<td class="result__snippet">[\s\S]*?<\/td>/g)
      || [];

    for (const block of resultBlocks) {
      if (results.length >= count) break;
      const linkMatch = block.match(/<a class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/);
      const snippetMatch = block.match(/class="result__snippet"[^>]*>([\s\S]*?)<\/(?:a|td)>/);

      if (linkMatch) {
        let resultUrl = linkMatch[1];
        const uddgMatch = resultUrl.match(/uddg=([^&]+)/);
        if (uddgMatch) resultUrl = decodeURIComponent(uddgMatch[1]);

        const title = linkMatch[2].replace(/<[^>]+>/g, '').trim();
        const snippet = snippetMatch
          ? snippetMatch[1].replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#x27;/g, "'").trim()
          : '';

        if (resultUrl && title) {
          results.push({ title, url: resultUrl, snippet });
        }
      }
    }

    if (results.length === 0) {
      // Fallback: try simpler regex for older DDG format
      const simpleLinks = html.match(/<a[^>]*class="result__url"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/g) || [];
      for (const link of simpleLinks) {
        if (results.length >= count) break;
        const m = link.match(/href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/);
        if (m) {
          let resultUrl = m[1];
          const uddgMatch = resultUrl.match(/uddg=([^&]+)/);
          if (uddgMatch) resultUrl = decodeURIComponent(uddgMatch[1]);
          results.push({ title: m[2].replace(/<[^>]+>/g, '').trim(), url: resultUrl, snippet: '' });
        }
      }
    }

    if (results.length === 0) return `No search results found for: ${query}`;

    let output = `Search results for: ${query}\n\n`;
    for (let i = 0; i < results.length; i++) {
      const r = results[i];
      output += `${i + 1}. ${r.title}\n`;
      output += `   ${r.url}\n`;
      if (r.snippet) output += `   ${r.snippet}\n`;
      output += '\n';
    }
    return output;
  } catch (err) {
    return `Error: search failed: ${err.message}`;
  }
}

module.exports = { webFetch, webSearch };
