scraper.js

const axios = require('axios');
const cheerio = require('cheerio');
const { URL } = require('url');

// Directories, directories, and social media platforms to exclude
const DIRECTORY_BLACKLIST = [
  'practo.com', 'justdial.com', 'indiamart.com', 'yelp.com', 'yellowpages.com',
  'whatclinic.com', 'wikipedia.org', 'facebook.com', 'instagram.com', 'linkedin.com',
  'youtube.com', 'twitter.com', 'x.com', 'tripadvisor.com', 'sulekha.com',
  'mouthshut.com', 'quikr.com', 'olx.in', 'pinterest.com', 'medium.com',
  'blogspot.com', 'wordpress.com', 'reddit.com', 'tumblr.com', 'amazon.com',
  'flipkart.com', 'mapquest.com', 'yellowbook.com', 'local.com', 'findlaw.com',
  'glassdoor.com', 'indeed.com', 'foursquare.com', 'groupon.com', 'bizjournals.com'
];

/**
 * Checks if a URL belongs to a directory, portal, or social network
 */
function isBlacklisted(urlStr) {
  if (!urlStr) return true;
  try {
    const urlLower = urlStr.toLowerCase();
    return DIRECTORY_BLACKLIST.some(domain => urlLower.includes(domain));
  } catch (e) {
    return true;
  }
}

/**
 * Clean Yahoo Search result title
 */
function cleanTitle(title) {
  if (!title) return '';
  let cleaned = title.trim();
  if (cleaned.includes('http')) {
    cleaned = cleaned.split('http')[0].trim();
  } else if (cleaned.includes('www.')) {
    cleaned = cleaned.split('www.')[0].trim();
  }
  cleaned = cleaned.replace(/[\s\-\.\·]+$/, '');
  return cleaned;
}

/**
 * Decodes Yahoo redirection URLs
 */
function decodeYahooUrl(rawUrl) {
  if (!rawUrl) return '';
  if (rawUrl.includes('/RU=')) {
    try {
      const parts = rawUrl.split('/RU=');
      if (parts.length > 1) {
        const encoded = parts[1].split('/RK=')[0];
        return decodeURIComponent(encoded);
      }
    } catch (e) {
      // fallback
    }
  }
  return rawUrl;
}

/**
 * Parse search query to extract Industry and City
 * @param {string} query 
 * @returns {Object}
 */
function parseQuery(query) {
  let industry = query;
  let city = 'Local';
  
  // Look for "in [City]", "near [City]", "at [City]"
  const match = query.match(/(.+?)\s+(in|near|at)\s+(.+)/i);
  if (match) {
    industry = match[1].trim();
    city = match[3].trim();
  }
  
  const titleCase = (str) => str.replace(/\b\w/g, c => c.toUpperCase());
  return {
    industry: titleCase(industry),
    city: titleCase(city)
  };
}

/**
 * Enrichment worker: Fetches homepage & contact page to extract leads data
 */
async function enrichLead(lead, industry, city) {
  const enriched = {
    title: lead.title,
    url: lead.url,
    snippet: lead.snippet,
    industry,
    city,
    contactPage: null,
    emails: [],
    phones: [],
    social: { facebook: null, instagram: null, linkedin: null, twitter: null }
  };

  try {
    // 1. Fetch homepage content
    const response = await axios.get(lead.url, {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
      },
      timeout: 4000, // strict 4s timeout for enrichment page fetch
      validateStatus: false
    });

    const html = response.data || '';
    const $ = cheerio.load(html);

    // Extract emails and phones from homepage
    const emailRegex = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
    const phoneRegex = /(\+?\d{1,4}[-.\s]?)?\(?\d{2,4}\)?[-.\s]?\d{3,4}[-.\s]?\d{4}/g;
    
    const homepageText = $('body').text();
    let emailsFound = homepageText.match(emailRegex) || [];
    let phonesFound = homepageText.match(phoneRegex) || [];

    // Extract social links from homepage
    $('a').each((i, el) => {
      const href = $(el).attr('href');
      if (href) {
        const hrefLower = href.toLowerCase();
        if (hrefLower.includes('facebook.com/') && !enriched.social.facebook) {
          enriched.social.facebook = href;
        } else if (hrefLower.includes('instagram.com/') && !enriched.social.instagram) {
          enriched.social.instagram = href;
        } else if (hrefLower.includes('linkedin.com/company/') && !enriched.social.linkedin) {
          enriched.social.linkedin = href;
        } else if ((hrefLower.includes('twitter.com/') || hrefLower.includes('x.com/')) && !enriched.social.twitter) {
          enriched.social.twitter = href;
        }

        // Search for contact page subpath
        if (!enriched.contactPage && 
            (hrefLower.includes('contact') || 
             hrefLower.includes('about') || 
             hrefLower.includes('get-in-touch') || 
             hrefLower.includes('reach-us'))) {
          try {
            enriched.contactPage = new URL(href, lead.url).href;
          } catch (e) {
            // ignore malformed URLs
          }
        }
      }
    });

    // 2. Fetch contact page content in parallel if discovered
    if (enriched.contactPage && enriched.contactPage !== lead.url) {
      try {
        const contactResponse = await axios.get(enriched.contactPage, {
          headers: {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
          },
          timeout: 3000,
          validateStatus: false
        });
        const contactHtml = contactResponse.data || '';
        const c$ = cheerio.load(contactHtml);
        const contactText = c$('body').text();

        const contactEmails = contactText.match(emailRegex) || [];
        const contactPhones = contactText.match(phoneRegex) || [];

        emailsFound = emailsFound.concat(contactEmails);
        phonesFound = phonesFound.concat(contactPhones);

        // Scan contact page for social links if missing on home page
        c$('a').each((i, el) => {
          const href = c$(el).attr('href');
          if (href) {
            const hrefLower = href.toLowerCase();
            if (hrefLower.includes('facebook.com/') && !enriched.social.facebook) enriched.social.facebook = href;
            else if (hrefLower.includes('instagram.com/') && !enriched.social.instagram) enriched.social.instagram = href;
            else if (hrefLower.includes('linkedin.com/company/') && !enriched.social.linkedin) enriched.social.linkedin = href;
            else if ((hrefLower.includes('twitter.com/') || hrefLower.includes('x.com/')) && !enriched.social.twitter) enriched.social.twitter = href;
          }
        });
      } catch (e) {
        // Silent catch for secondary page fetch failures
      }
    }

    // Clean lists (unique values, limit items)
    enriched.emails = Array.from(new Set(emailsFound)).slice(0, 3);
    enriched.phones = Array.from(new Set(phonesFound)).slice(0, 3);

  } catch (error) {
    // Graceful catch for unreachable sites
  }

  return enriched;
}

/**
 * Scrapes business website leads from Yahoo Search and enriches them
 */
async function scrapeLeads(query) {
  let searchQuery = query;
  if (!query.toLowerCase().includes('website') && !query.toLowerCase().includes('site')) {
    searchQuery = `${query} website`;
  }

  const url = `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}`;
  const rawLeads = [];
  const domainTracker = new Set();
  const { industry, city } = parseQuery(query);

  try {
    const response = await axios.get(url, {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.9'
      },
      timeout: 10000
    });

    if (response.status !== 200) {
      throw new Error(`Yahoo returned status ${response.status}`);
    }

    const $ = cheerio.load(response.data);

    $('.algo').each((i, el) => {
      const titleEl = $(el).find('.compTitle a, a').first();
      const rawTitle = titleEl.text().trim();
      const rawUrl = titleEl.attr('href');
      const snippet = $(el).find('.compText, p').first().text().trim();

      const cleanUrl = decodeYahooUrl(rawUrl);

      if (rawTitle && cleanUrl) {
        if (
          !cleanUrl.includes('yahoo.com') &&
          !cleanUrl.includes('google.com') &&
          !cleanUrl.includes('bing.com') &&
          !isBlacklisted(cleanUrl)
        ) {
          // Extract hostname to deduplicate by domain
          try {
            const domainName = new URL(cleanUrl).hostname.toLowerCase().replace('www.', '');
            if (!domainTracker.has(domainName)) {
              domainTracker.add(domainName);
              rawLeads.push({
                title: cleanTitle(rawTitle),
                url: cleanUrl,
                snippet: snippet.substring(0, 160) + (snippet.length > 160 ? '...' : '')
              });
            }
          } catch (e) {
            // skip malformed URLs
          }
        }
      }
    });

    // Enforce parallel enrichment scans on the scraped leads (limited to top 8 to stay snappy)
    const targets = rawLeads.slice(0, 8);
    const enrichmentPromises = targets.map(lead => enrichLead(lead, industry, city));
    
    // Concurrently fetch and scan all targets (Promise.allSettled guarantees completion)
    const results = await Promise.allSettled(enrichmentPromises);
    
    const enrichedLeads = results.map((res, index) => {
      if (res.status === 'fulfilled') {
        return res.value;
      } else {
        // Return baseline lead if enrichment worker failed
        const baseline = targets[index];
        return {
          title: baseline.title,
          url: baseline.url,
          snippet: baseline.snippet,
          industry,
          city,
          contactPage: null,
          emails: [],
          phones: [],
          social: { facebook: null, instagram: null, linkedin: null, twitter: null }
        };
      }
    });

    return enrichedLeads;

  } catch (error) {
    console.error(`[Scraper Error] Failed to scrape query "${query}":`, error.message);
    return [];
  }
}

module.exports = {
  scrapeLeads
};