import * as cheerio from 'cheerio' import { DateTime } from 'luxon' import logger from '@adonisjs/core/services/logger' import ComponentType from '#models/component_type' import ScrapedPart from '#models/scraped_part' const SOURCE = 'pc-builder.io' const BASE_URL = 'https://fr.pc-builder.io' /** * Default freshness window for the scraped_parts cache. A category is * re-scraped automatically the first time it is searched after this delay * has elapsed since its last successful scrape. */ const CACHE_TTL_HOURS = 24 /** * Maps our own ComponentType names to the corresponding pc-builder.io * category slug. Categories not listed here are not scraped. * * Note: pc-builder.io only has a single "storage" category covering both * SSDs and HDDs (no separate slugs exist), so it is only mapped to our * "SSD" type to avoid two types racing to own the same scraped rows * (source_ref is unique per source, independent of type_id). */ const CATEGORY_SLUGS: Record = { 'Processor': 'cpu', 'Motherboard': 'motherboard', 'Graphic Card': 'gpu', 'Memory': 'ram', 'SSD': 'storage', 'Cooler': 'cpu-cooler', 'Power Supply': 'power-supply', 'Case': 'case', 'Fan': 'case-fan', 'WiFi-Card': 'wireless-network-adapter', } interface ScrapedRow { name: string price: number imageUrl: string | null /** * The actual merchant page where this part can be bought (currently * always an Amazon.fr link, since that's the only "buy" link pc-builder.io * exposes on its category pages). This is what gets stored as * ScrapedPart.sourceUrl and later copied into Component.purchaseUrl — it * intentionally is NOT the pc-builder.io product page, which isn't a * place a user can actually purchase anything from. */ purchaseUrl: string sourceRef: string } /** * Fetches a pc-builder.io category page and parses the (server-rendered) * product table into a list of rows. Only the first ~100 products per * category are available without simulating the site's DataTables/CSRF * flow, which is an accepted limitation for this cache. */ async function fetchCategoryPage(slug: string): Promise { const url = `${BASE_URL}/${slug}` const response = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)', }, }) if (!response.ok) { throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`) } const html = await response.text() const $ = cheerio.load(html) const rows: ScrapedRow[] = [] $('#products-table tbody tr').each((_, el) => { const row = $(el) const nameLink = row.find('td.product a').first() const name = nameLink.text().trim() const productPageUrl = nameLink.attr('href') const img = row.find('td.product img.product-image') const imageUrl = img.attr('data-src') || img.attr('src') || null const priceText = row.find('td.add-btn .price-box p').first().text().trim() const addButton = row.find('a.add-to-build') const sourceRef = addButton.attr('data-id') // Real merchant link ("View on Amazon"), either a direct product page // (https://www.amazon.fr/dp/{ASIN}?tag=...) when pc-builder.io knows the // ASIN, or a generic search results page (https://www.amazon.fr/s?k=...) // otherwise. Either way it's a genuine purchase link, unlike the // pc-builder.io product page. const amazonUrl = row.find('a.btn-default').attr('href') if (!name || !productPageUrl || !sourceRef || !priceText) { // Skip rows without a resolved price (e.g. "search on Amazon" links // with no fixed offer) or missing identifiers. return } const price = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.')) if (Number.isNaN(price)) { return } rows.push({ name, price, imageUrl, purchaseUrl: amazonUrl || productPageUrl, sourceRef }) }) return rows } /** * Scrapes a single component type's mapped category and upserts the * resulting rows into the scraped_parts cache table. */ async function scrapeComponentType(componentType: ComponentType): Promise { const slug = CATEGORY_SLUGS[componentType.name] if (!slug) { logger.warn({ type: componentType.name }, 'No pc-builder.io category mapped, skipping') return 0 } const rows = await fetchCategoryPage(slug) const scrapedAt = DateTime.now() for (const row of rows) { await ScrapedPart.updateOrCreate( { source: SOURCE, sourceRef: row.sourceRef }, { typeId: componentType.id, name: row.name, price: row.price, imageUrl: row.imageUrl, sourceUrl: row.purchaseUrl, scrapedAt, } ) } return rows.length } /** * Scrapes every mapped ComponentType category from pc-builder.io and * refreshes the scraped_parts cache. Returns a per-type summary. */ async function scrapeAllCategories(): Promise> { const componentTypes = await ComponentType.all() const summary: Record = {} for (const componentType of componentTypes) { if (!CATEGORY_SLUGS[componentType.name]) continue try { summary[componentType.name] = await scrapeComponentType(componentType) } catch (error) { logger.error({ type: componentType.name, err: error }, 'Failed to scrape category') summary[componentType.name] = 0 } } return summary } /** * Returns true if the given ComponentType's category has never been * scraped, or was last scraped more than CACHE_TTL_HOURS ago. */ async function isCategoryStale(componentType: ComponentType): Promise { const mostRecent = await ScrapedPart.query() .where('source', SOURCE) .andWhere('typeId', componentType.id) .orderBy('scrapedAt', 'desc') .first() if (!mostRecent) return true return mostRecent.scrapedAt < DateTime.now().minus({ hours: CACHE_TTL_HOURS }) } /** * Ensures the cache for a given ComponentType's mapped category is fresh * (scraped within the last CACHE_TTL_HOURS), re-scraping it synchronously * if it's missing or stale. No-op for unmapped types. Scraping errors are * logged but not thrown, so a search still falls back to whatever is * already cached. */ async function ensureFreshCategory(componentType: ComponentType): Promise { if (!CATEGORY_SLUGS[componentType.name]) return if (!(await isCategoryStale(componentType))) return try { await scrapeComponentType(componentType) } catch (error) { logger.error({ type: componentType.name, err: error }, 'Failed to refresh stale category cache') } } export { scrapeAllCategories, scrapeComponentType, ensureFreshCategory, isCategoryStale, CATEGORY_SLUGS, CACHE_TTL_HOURS, }