pc_builder/app/services/part_scraper_service.ts

import * as cheerio from 'cheerio'
import { DateTime } from 'luxon'
import logger from '@adonisjs/core/services/logger'
import ComponentType from '#models/component_type'
import ScrapedPart from '#models/scraped_part'

const SOURCE = 'pc-builder.io'
const BASE_URL = 'https://fr.pc-builder.io'

/**
 * Default freshness window for the scraped_parts cache. A category is
 * re-scraped automatically the first time it is searched after this delay
 * has elapsed since its last successful scrape.
 */
const CACHE_TTL_HOURS = 24

/**
 * Maps our own ComponentType names to the corresponding pc-builder.io
 * category slug. Categories not listed here are not scraped.
 *
 * Note: pc-builder.io only has a single "storage" category covering both
 * SSDs and HDDs (no separate slugs exist), so it is only mapped to our
 * "SSD" type to avoid two types racing to own the same scraped rows
 * (source_ref is unique per source, independent of type_id).
 */
const CATEGORY_SLUGS: Record<string, string> = {
  'Processor': 'cpu',
  'Motherboard': 'motherboard',
  'Graphic Card': 'gpu',
  'Memory': 'ram',
  'SSD': 'storage',
  'Cooler': 'cpu-cooler',
  'Power Supply': 'power-supply',
  'Case': 'case',
  'Fan': 'case-fan',
  'WiFi-Card': 'wireless-network-adapter',
}

interface ScrapedRow {
  name: string
  price: number
  imageUrl: string | null
  sourceUrl: string
  sourceRef: string
}

/**
 * Fetches a pc-builder.io category page and parses the (server-rendered)
 * product table into a list of rows. Only the first ~100 products per
 * category are available without simulating the site's DataTables/CSRF
 * flow, which is an accepted limitation for this cache.
 */
async function fetchCategoryPage(slug: string): Promise<ScrapedRow[]> {
  const url = `${BASE_URL}/${slug}`
  const response = await fetch(url, {
    headers: {
      'User-Agent': 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)',
    },
  })

  if (!response.ok) {
    throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`)
  }

  const html = await response.text()
  const $ = cheerio.load(html)
  const rows: ScrapedRow[] = []

  $('#products-table tbody tr').each((_, el) => {
    const row = $(el)
    const nameLink = row.find('td.product a').first()
    const name = nameLink.text().trim()
    const sourceUrl = nameLink.attr('href')
    const img = row.find('td.product img.product-image')
    const imageUrl = img.attr('data-src') || img.attr('src') || null
    const priceText = row.find('td.add-btn .price-box p').first().text().trim()
    const addButton = row.find('a.add-to-build')
    const sourceRef = addButton.attr('data-id')

    if (!name || !sourceUrl || !sourceRef || !priceText) {
      // Skip rows without a resolved price (e.g. "search on Amazon" links
      // with no fixed offer) or missing identifiers.
      return
    }

    const price = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.'))
    if (Number.isNaN(price)) {
      return
    }

    rows.push({ name, price, imageUrl, sourceUrl, sourceRef })
  })

  return rows
}

/**
 * Scrapes a single component type's mapped category and upserts the
 * resulting rows into the scraped_parts cache table.
 */
async function scrapeComponentType(componentType: ComponentType): Promise<number> {
  const slug = CATEGORY_SLUGS[componentType.name]
  if (!slug) {
    logger.warn({ type: componentType.name }, 'No pc-builder.io category mapped, skipping')
    return 0
  }

  const rows = await fetchCategoryPage(slug)
  const scrapedAt = DateTime.now()

  for (const row of rows) {
    await ScrapedPart.updateOrCreate(
      { source: SOURCE, sourceRef: row.sourceRef },
      {
        typeId: componentType.id,
        name: row.name,
        price: row.price,
        imageUrl: row.imageUrl,
        sourceUrl: row.sourceUrl,
        scrapedAt,
      }
    )
  }

  return rows.length
}

/**
 * Scrapes every mapped ComponentType category from pc-builder.io and
 * refreshes the scraped_parts cache. Returns a per-type summary.
 */
async function scrapeAllCategories(): Promise<Record<string, number>> {
  const componentTypes = await ComponentType.all()
  const summary: Record<string, number> = {}

  for (const componentType of componentTypes) {
    if (!CATEGORY_SLUGS[componentType.name]) continue
    try {
      summary[componentType.name] = await scrapeComponentType(componentType)
    } catch (error) {
      logger.error({ type: componentType.name, err: error }, 'Failed to scrape category')
      summary[componentType.name] = 0
    }
  }

  return summary
}

/**
 * Returns true if the given ComponentType's category has never been
 * scraped, or was last scraped more than CACHE_TTL_HOURS ago.
 */
async function isCategoryStale(componentType: ComponentType): Promise<boolean> {
  const mostRecent = await ScrapedPart.query()
    .where('source', SOURCE)
    .andWhere('typeId', componentType.id)
    .orderBy('scrapedAt', 'desc')
    .first()

  if (!mostRecent) return true
  return mostRecent.scrapedAt < DateTime.now().minus({ hours: CACHE_TTL_HOURS })
}

/**
 * Ensures the cache for a given ComponentType's mapped category is fresh
 * (scraped within the last CACHE_TTL_HOURS), re-scraping it synchronously
 * if it's missing or stale. No-op for unmapped types. Scraping errors are
 * logged but not thrown, so a search still falls back to whatever is
 * already cached.
 */
async function ensureFreshCategory(componentType: ComponentType): Promise<void> {
  if (!CATEGORY_SLUGS[componentType.name]) return
  if (!(await isCategoryStale(componentType))) return

  try {
    await scrapeComponentType(componentType)
  } catch (error) {
    logger.error({ type: componentType.name, err: error }, 'Failed to refresh stale category cache')
  }
}

export {
  scrapeAllCategories,
  scrapeComponentType,
  ensureFreshCategory,
  isCategoryStale,
  CATEGORY_SLUGS,
  CACHE_TTL_HOURS,
}