pc_builder/app/services/part_scraper_service.ts

import * as cheerio from 'cheerio'
import { DateTime } from 'luxon'
import logger from '@adonisjs/core/services/logger'
import ComponentType from '#models/component_type'
import ScrapedPart from '#models/scraped_part'

const SOURCE = 'pc-builder.io'
const BASE_URL = 'https://fr.pc-builder.io'
const USER_AGENT = 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)'
const PAGE_SIZE = 100

/**
 * Default freshness window for the scraped_parts cache. A category is
 * re-scraped automatically the first time it is searched after this delay
 * has elapsed since its last successful scrape.
 */
const CACHE_TTL_HOURS = 24

/**
 * How many products to fetch when a search triggers an automatic refresh
 * (see ensureFreshCategory). Kept small since this runs synchronously
 * inside a user-facing HTTP request — a full deep scrape (thousands of
 * products for some categories) belongs in the `scrape:parts` command
 * instead, not in the request/response cycle of a search.
 */
const QUICK_REFRESH_MAX_PRODUCTS = 200

/**
 * Maps our own ComponentType names to pc-builder.io's category slug,
 * numeric category_id (required by the paginated /products/data endpoint,
 * found in each category page's inline DataTables config), and how many
 * products `node ace scrape:parts` should fetch for that category.
 *
 * maxProducts is chosen per category based on how much price variance
 * matters there and how large the catalog is: GPUs get close-to-full
 * coverage since price differences between models are large and users
 * specifically look for cheaper alternatives (see the RTX 5070 case that
 * prompted this), while smaller/less price-sensitive categories use a
 * lighter cap to keep the full scrape run in a reasonable time.
 *
 * Note: pc-builder.io only has a single "storage" category covering both
 * SSDs and HDDs (no separate slugs exist), so it is only mapped to our
 * "SSD" type to avoid two types racing to own the same scraped rows
 * (source_ref is unique per source, independent of type_id).
 */
const CATEGORY_CONFIG: Record<string, { slug: string; categoryId: string; maxProducts: number }> = {
  'Processor': { slug: 'cpu', categoryId: '1', maxProducts: 500 },
  'Motherboard': { slug: 'motherboard', categoryId: '3', maxProducts: 500 },
  'Graphic Card': { slug: 'gpu', categoryId: '6', maxProducts: 6000 },
  'Memory': { slug: 'ram', categoryId: '4', maxProducts: 800 },
  'SSD': { slug: 'storage', categoryId: '5', maxProducts: 800 },
  'Cooler': { slug: 'cpu-cooler', categoryId: '2', maxProducts: 400 },
  'Power Supply': { slug: 'power-supply', categoryId: '8', maxProducts: 400 },
  'Case': { slug: 'case', categoryId: '7', maxProducts: 400 },
  'Fan': { slug: 'case-fan', categoryId: '20', maxProducts: 300 },
  'WiFi-Card': { slug: 'wireless-network-adapter', categoryId: '13', maxProducts: 400 },
}

/** Backwards-compatible slug lookup, kept for anything relying on it. */
const CATEGORY_SLUGS: Record<string, string> = Object.fromEntries(
  Object.entries(CATEGORY_CONFIG).map(([name, config]) => [name, config.slug])
)

interface ScrapedRow {
  name: string
  /**
   * Null when pc-builder.io has no fixed offer for this part (common for
   * very recent products — it only links to a generic Amazon search,
   * e.g. "https://amazon.fr/s?k=..." instead of a priced "/dp/{ASIN}"
   * page). Such parts are still kept (not skipped) so they remain
   * searchable and importable; ScrapedPartsController#compare lets the
   * user resolve — and persist — a real price for them on demand from
   * the other supported merchants.
   */
  price: number | null
  imageUrl: string | null
  /**
   * The actual merchant page where this part can be bought (currently
   * always an Amazon.fr link, since that's the only "buy" link pc-builder.io
   * exposes on its category pages) — either a direct priced product page
   * or a generic search results page when no price is known. This is
   * what gets stored as ScrapedPart.sourceUrl and later copied into
   * Component.purchaseUrl — it intentionally is NOT the pc-builder.io
   * product page, which isn't a place a user can actually purchase
   * anything from.
   */
  purchaseUrl: string
  sourceRef: string
}

/**
 * Loads a category page once to grab the CSRF token and session cookie
 * needed to call the paginated /products/data endpoint, which is what
 * powers the client-side DataTables widget on every category page.
 */
async function getSessionAndToken(slug: string): Promise<{ cookies: string; token: string }> {
  const response = await fetch(`${BASE_URL}/${slug}`, {
    headers: { 'User-Agent': USER_AGENT },
  })
  if (!response.ok) {
    throw new Error(`Failed to load ${slug} category page: HTTP ${response.status}`)
  }
  const html = await response.text()

  const setCookieHeaders = response.headers.getSetCookie?.() ?? []
  const cookies = setCookieHeaders.map((cookie) => cookie.split(';')[0]).join('; ')

  const tokenMatch = html.match(/"_token":\s*"([^"]+)"/)
  if (!tokenMatch) {
    throw new Error(`Could not find CSRF token on ${slug} category page`)
  }

  return { cookies, token: tokenMatch[1] }
}

/**
 * Parses the "Name" / "Add to Build" cell HTML fragments returned by
 * /products/data into a single row, using the exact same markup structure
 * as the plain category page (both come from the same Blade partials), so
 * the same selectors apply.
 */
function parseRow(nameHtml: string, addToBuildHtml: string): ScrapedRow | null {
  const $name = cheerio.load(nameHtml)
  const nameLink = $name('a').first()
  const name = nameLink.text().trim()
  const productPageUrl = nameLink.attr('href')
  const img = $name('img.product-image')
  const imageUrl = img.attr('data-src') || img.attr('src') || null

  const $btn = cheerio.load(addToBuildHtml)
  const priceText = $btn('.price-box p').first().text().trim()
  const sourceRef = $btn('a.add-to-build').attr('data-id')
  const amazonUrl = $btn('a.btn-default').attr('href')

  if (!name || !productPageUrl || !sourceRef) {
    // Only truly unusable rows (missing name/link/id) are skipped. A
    // missing price is not disqualifying — see the ScrapedRow.price doc.
    return null
  }

  let price: number | null = null
  if (priceText) {
    const parsed = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.'))
    price = Number.isNaN(parsed) ? null : parsed
  }

  return { name, price, imageUrl, purchaseUrl: amazonUrl || productPageUrl, sourceRef }
}

/**
 * Fetches one page of `length` products starting at `start` from
 * pc-builder.io's DataTables-backed /products/data endpoint. This is a
 * plain HTTP POST (no headless browser needed) once the CSRF
 * token/session cookie from getSessionAndToken() are supplied — see
 * app/services/part_scraper_service.ts git history / PR description for
 * the reverse-engineered request shape (field names matter exactly:
 * columns[].data must be "Name"/"Add to Build", not numeric indices).
 */
async function fetchProductsPage(
  categoryId: string,
  cookies: string,
  token: string,
  start: number,
  length: number
): Promise<{ rows: ScrapedRow[]; recordsTotal: number }> {
  const body = new URLSearchParams({
    '_token': token,
    'draw': '1',
    'start': String(start),
    'length': String(length),
    'compatibility_filter': 'false',
    'filter_status': '',
    'category_id': categoryId,
    'compatible_with': '',
    'checkbox_filters': '[]',
    'range_filters': '[]',
    'search_query': '',
    'columns[0][data]': 'Name',
    'columns[0][name]': 'Name',
    'columns[0][searchable]': 'true',
    'columns[0][orderable]': 'false',
    'columns[0][search][value]': '',
    'columns[0][search][regex]': 'false',
    'columns[1][data]': 'Add to Build',
    'columns[1][name]': 'Add to Build',
    'columns[1][searchable]': 'true',
    'columns[1][orderable]': 'false',
    'columns[1][search][value]': '',
    'columns[1][search][regex]': 'false',
    'search[value]': '',
    'search[regex]': 'false',
  })

  const response = await fetch(`${BASE_URL}/products/data`, {
    method: 'POST',
    headers: {
      'User-Agent': USER_AGENT,
      'X-Requested-With': 'XMLHttpRequest',
      'Content-Type': 'application/x-www-form-urlencoded',
      'Cookie': cookies,
    },
    body,
  })

  if (!response.ok) {
    throw new Error(`products/data request failed: HTTP ${response.status}`)
  }

  const json = (await response.json()) as {
    recordsTotal: number
    data: { 'Name': string; 'Add to Build': string }[]
  }

  const rows = json.data
    .map((entry) => parseRow(entry.Name, entry['Add to Build']))
    .filter((row): row is ScrapedRow => row !== null)

  return { rows, recordsTotal: json.recordsTotal }
}

/**
 * Fetches up to `maxProducts` products for a category by paginating
 * through pc-builder.io's /products/data endpoint PAGE_SIZE at a time,
 * stopping early once the category's real total is exhausted. A short
 * delay between pages avoids hammering the site with a burst of requests
 * back-to-back for the larger categories (e.g. GPU's ~60 pages).
 */
async function fetchCategoryProducts(
  slug: string,
  categoryId: string,
  maxProducts: number
): Promise<ScrapedRow[]> {
  const { cookies, token } = await getSessionAndToken(slug)

  const rows: ScrapedRow[] = []
  let start = 0
  let recordsTotal = maxProducts

  while (start < maxProducts && start < recordsTotal) {
    const length = Math.min(PAGE_SIZE, maxProducts - start)
    const page = await fetchProductsPage(categoryId, cookies, token, start, length)
    rows.push(...page.rows)
    recordsTotal = page.recordsTotal
    start += length

    if (start < maxProducts && start < recordsTotal) {
      await new Promise((resolve) => setTimeout(resolve, 150))
    }
  }

  return rows
}

/**
 * Scrapes a single component type's mapped category and upserts the
 * resulting rows into the scraped_parts cache table.
 *
 * @param maxProductsOverride Caps how many products are fetched for this
 *   run, regardless of the category's configured maxProducts. Used by
 *   ensureFreshCategory to keep an on-demand refresh fast (a handful of
 *   pages) while `node ace scrape:parts` uses the full configured depth.
 */
async function scrapeComponentType(
  componentType: ComponentType,
  maxProductsOverride?: number
): Promise<number> {
  const config = CATEGORY_CONFIG[componentType.name]
  if (!config) {
    logger.warn({ type: componentType.name }, 'No pc-builder.io category mapped, skipping')
    return 0
  }

  const maxProducts = Math.min(maxProductsOverride ?? config.maxProducts, config.maxProducts)
  const rows = await fetchCategoryProducts(config.slug, config.categoryId, maxProducts)
  const scrapedAt = DateTime.now()

  for (const row of rows) {
    const existing = await ScrapedPart.query()
      .where('source', SOURCE)
      .andWhere('sourceRef', row.sourceRef)
      .first()

    // If a previous price comparison already resolved (and persisted) a
    // real price for this part from a merchant, don't blow it away just
    // because this catalog refresh came back with no price — keep
    // whichever is best (see updatePriceIfCheaper for the "best of all
    // known sources" logic used when a comparison finds a lower price).
    const hasBetterExistingPrice =
      existing?.price !== null &&
      existing?.price !== undefined &&
      (row.price === null || existing.price <= row.price)

    await ScrapedPart.updateOrCreate(
      { source: SOURCE, sourceRef: row.sourceRef },
      {
        typeId: componentType.id,
        name: row.name,
        imageUrl: row.imageUrl,
        scrapedAt,
        ...(hasBetterExistingPrice
          ? {}
          : {
              price: row.price,
              sourceUrl: row.purchaseUrl,
              priceSource: row.price !== null ? SOURCE : null,
              priceCheckedAt: row.price !== null ? scrapedAt : null,
            }),
      }
    )
  }

  return rows.length
}

/**
 * Scrapes every mapped ComponentType category from pc-builder.io and
 * refreshes the scraped_parts cache, using each category's full
 * configured depth (see CATEGORY_CONFIG). Returns a per-type summary.
 * This can take a few minutes overall (GPU alone fetches up to 6000
 * products across ~60 requests) — intended to be run manually or via a
 * scheduled job, not inside a request.
 */
async function scrapeAllCategories(): Promise<Record<string, number>> {
  const componentTypes = await ComponentType.all()
  const summary: Record<string, number> = {}

  for (const componentType of componentTypes) {
    if (!CATEGORY_CONFIG[componentType.name]) continue
    try {
      summary[componentType.name] = await scrapeComponentType(componentType)
    } catch (error) {
      logger.error({ type: componentType.name, err: error }, 'Failed to scrape category')
      summary[componentType.name] = 0
    }
  }

  return summary
}

/**
 * Returns true if the given ComponentType's category has never been
 * scraped, or was last scraped more than CACHE_TTL_HOURS ago.
 */
async function isCategoryStale(componentType: ComponentType): Promise<boolean> {
  const mostRecent = await ScrapedPart.query()
    .where('source', SOURCE)
    .andWhere('typeId', componentType.id)
    .orderBy('scrapedAt', 'desc')
    .first()

  if (!mostRecent) return true
  return mostRecent.scrapedAt < DateTime.now().minus({ hours: CACHE_TTL_HOURS })
}

/**
 * Ensures the cache for a given ComponentType's mapped category is fresh
 * (scraped within the last CACHE_TTL_HOURS), re-scraping it synchronously
 * if it's missing or stale. No-op for unmapped types. Scraping errors are
 * logged but not thrown, so a search still falls back to whatever is
 * already cached.
 *
 * Only fetches QUICK_REFRESH_MAX_PRODUCTS products (not the category's
 * full configured depth), since this runs inline in a user's search
 * request — run `node ace scrape:parts` for full-depth coverage.
 */
async function ensureFreshCategory(componentType: ComponentType): Promise<void> {
  if (!CATEGORY_CONFIG[componentType.name]) return
  if (!(await isCategoryStale(componentType))) return

  try {
    await scrapeComponentType(componentType, QUICK_REFRESH_MAX_PRODUCTS)
  } catch (error) {
    logger.error({ type: componentType.name, err: error }, 'Failed to refresh stale category cache')
  }
}

/**
 * Persists `offer` as a part's new price/source if it's cheaper than
 * whatever is currently stored (or nothing was stored yet), so the "best
 * known price" keeps improving as users compare prices (see
 * ScrapedPartsController#compare) instead of resetting to a possibly
 * stale/missing pc-builder.io price on the next catalog refresh —
 * scrapeComponentType already refuses to overwrite a price with a worse
 * one using this same "keep whichever is lower" rule.
 */
async function updatePriceIfCheaper(
  part: ScrapedPart,
  offer: { merchant: string; price: number; url: string }
): Promise<void> {
  if (part.price !== null && part.price <= offer.price) return

  part.price = offer.price
  part.sourceUrl = offer.url
  part.priceSource = offer.merchant
  part.priceCheckedAt = DateTime.now()
  await part.save()
}

export {
  scrapeAllCategories,
  scrapeComponentType,
  ensureFreshCategory,
  isCategoryStale,
  updatePriceIfCheaper,
  CATEGORY_CONFIG,
  CATEGORY_SLUGS,
  CACHE_TTL_HOURS,
  QUICK_REFRESH_MAX_PRODUCTS,
}