diff --git a/app/controllers/scraped_parts_controller.ts b/app/controllers/scraped_parts_controller.ts index b4bca15..9c0b775 100644 --- a/app/controllers/scraped_parts_controller.ts +++ b/app/controllers/scraped_parts_controller.ts @@ -2,7 +2,7 @@ import type { HttpContext } from '@adonisjs/core/http' import vine from '@vinejs/vine' import ScrapedPart from '#models/scraped_part' import ComponentType from '#models/component_type' -import { ensureFreshCategory } from '#services/part_scraper_service' +import { ensureFreshCategory, updatePriceIfCheaper } from '#services/part_scraper_service' import { findCheapestOffers } from '#services/merchant_price_service' const searchValidator = vine.compile( @@ -31,6 +31,11 @@ export default class ScrapedPartsController { * all types skip this eager refresh to avoid re-scraping every mapped * category on every keystroke; run `node ace scrape:parts` to refresh * everything at once. + * + * Results are ordered with a resolved price first (cheapest first), then + * parts with no known price yet (see ScrapedPart.price doc) last — so a + * part is never hidden just because pc-builder.io hasn't found a fixed + * Amazon offer for it, but priced parts still surface first. */ async index({ request, auth, response }: HttpContext) { await auth.authenticate() @@ -43,7 +48,9 @@ export default class ScrapedPartsController { } } - const query = ScrapedPart.query().preload('type').orderBy('name') + const query = ScrapedPart.query() + .preload('type') + .orderByRaw('price IS NULL, price ASC, name ASC') if (q) { query.whereRaw('LOWER(name) LIKE ?', [`%${q.toLowerCase()}%`]) @@ -58,10 +65,15 @@ export default class ScrapedPartsController { /** * Compares live prices for a given scraped part across Amazon, LDLC, - * Grosbill and TopAchat, and returns them sorted cheapest first. This is - * triggered on demand (one part at a time) rather than for every search - * result, since it makes several outbound HTTP requests and can take a - * few seconds. + * Grosbill, TopAchat, Materiel.net, Cdiscount and PCComponentes, and + * returns them sorted cheapest first. Triggered on demand (one part at a + * time) rather than for every search result, since it makes several + * outbound requests and can take a few seconds. + * + * If the cheapest offer found beats the part's currently stored price + * (or no price was known yet), it's persisted as the new default price + * (see updatePriceIfCheaper) — so this comparison benefits every future + * search/import for this part, not just the current user. */ async compare({ request, auth, response }: HttpContext) { await auth.authenticate() @@ -75,6 +87,10 @@ export default class ScrapedPartsController { const amazonUrl = part.sourceUrl.includes('amazon.') ? part.sourceUrl : null const offers = await findCheapestOffers(part.name, amazonUrl) + if (offers.length > 0) { + await updatePriceIfCheaper(part, offers[0]) + } + return response.json({ offers }) } } diff --git a/app/services/part_scraper_service.ts b/app/services/part_scraper_service.ts index d4765a9..b3d7b08 100644 --- a/app/services/part_scraper_service.ts +++ b/app/services/part_scraper_service.ts @@ -6,6 +6,8 @@ import ScrapedPart from '#models/scraped_part' const SOURCE = 'pc-builder.io' const BASE_URL = 'https://fr.pc-builder.io' +const USER_AGENT = 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)' +const PAGE_SIZE = 100 /** * Default freshness window for the scraped_parts cache. A category is @@ -15,95 +17,236 @@ const BASE_URL = 'https://fr.pc-builder.io' const CACHE_TTL_HOURS = 24 /** - * Maps our own ComponentType names to the corresponding pc-builder.io - * category slug. Categories not listed here are not scraped. + * How many products to fetch when a search triggers an automatic refresh + * (see ensureFreshCategory). Kept small since this runs synchronously + * inside a user-facing HTTP request — a full deep scrape (thousands of + * products for some categories) belongs in the `scrape:parts` command + * instead, not in the request/response cycle of a search. + */ +const QUICK_REFRESH_MAX_PRODUCTS = 200 + +/** + * Maps our own ComponentType names to pc-builder.io's category slug, + * numeric category_id (required by the paginated /products/data endpoint, + * found in each category page's inline DataTables config), and how many + * products `node ace scrape:parts` should fetch for that category. + * + * maxProducts is chosen per category based on how much price variance + * matters there and how large the catalog is: GPUs get close-to-full + * coverage since price differences between models are large and users + * specifically look for cheaper alternatives (see the RTX 5070 case that + * prompted this), while smaller/less price-sensitive categories use a + * lighter cap to keep the full scrape run in a reasonable time. * * Note: pc-builder.io only has a single "storage" category covering both * SSDs and HDDs (no separate slugs exist), so it is only mapped to our * "SSD" type to avoid two types racing to own the same scraped rows * (source_ref is unique per source, independent of type_id). */ -const CATEGORY_SLUGS: Record = { - 'Processor': 'cpu', - 'Motherboard': 'motherboard', - 'Graphic Card': 'gpu', - 'Memory': 'ram', - 'SSD': 'storage', - 'Cooler': 'cpu-cooler', - 'Power Supply': 'power-supply', - 'Case': 'case', - 'Fan': 'case-fan', - 'WiFi-Card': 'wireless-network-adapter', +const CATEGORY_CONFIG: Record = { + 'Processor': { slug: 'cpu', categoryId: '1', maxProducts: 500 }, + 'Motherboard': { slug: 'motherboard', categoryId: '3', maxProducts: 500 }, + 'Graphic Card': { slug: 'gpu', categoryId: '6', maxProducts: 6000 }, + 'Memory': { slug: 'ram', categoryId: '4', maxProducts: 800 }, + 'SSD': { slug: 'storage', categoryId: '5', maxProducts: 800 }, + 'Cooler': { slug: 'cpu-cooler', categoryId: '2', maxProducts: 400 }, + 'Power Supply': { slug: 'power-supply', categoryId: '8', maxProducts: 400 }, + 'Case': { slug: 'case', categoryId: '7', maxProducts: 400 }, + 'Fan': { slug: 'case-fan', categoryId: '20', maxProducts: 300 }, + 'WiFi-Card': { slug: 'wireless-network-adapter', categoryId: '13', maxProducts: 400 }, } +/** Backwards-compatible slug lookup, kept for anything relying on it. */ +const CATEGORY_SLUGS: Record = Object.fromEntries( + Object.entries(CATEGORY_CONFIG).map(([name, config]) => [name, config.slug]) +) + interface ScrapedRow { name: string - price: number + /** + * Null when pc-builder.io has no fixed offer for this part (common for + * very recent products — it only links to a generic Amazon search, + * e.g. "https://amazon.fr/s?k=..." instead of a priced "/dp/{ASIN}" + * page). Such parts are still kept (not skipped) so they remain + * searchable and importable; ScrapedPartsController#compare lets the + * user resolve — and persist — a real price for them on demand from + * the other supported merchants. + */ + price: number | null imageUrl: string | null /** * The actual merchant page where this part can be bought (currently * always an Amazon.fr link, since that's the only "buy" link pc-builder.io - * exposes on its category pages). This is what gets stored as - * ScrapedPart.sourceUrl and later copied into Component.purchaseUrl — it - * intentionally is NOT the pc-builder.io product page, which isn't a - * place a user can actually purchase anything from. + * exposes on its category pages) — either a direct priced product page + * or a generic search results page when no price is known. This is + * what gets stored as ScrapedPart.sourceUrl and later copied into + * Component.purchaseUrl — it intentionally is NOT the pc-builder.io + * product page, which isn't a place a user can actually purchase + * anything from. */ purchaseUrl: string sourceRef: string } /** - * Fetches a pc-builder.io category page and parses the (server-rendered) - * product table into a list of rows. Only the first ~100 products per - * category are available without simulating the site's DataTables/CSRF - * flow, which is an accepted limitation for this cache. + * Loads a category page once to grab the CSRF token and session cookie + * needed to call the paginated /products/data endpoint, which is what + * powers the client-side DataTables widget on every category page. */ -async function fetchCategoryPage(slug: string): Promise { - const url = `${BASE_URL}/${slug}` - const response = await fetch(url, { +async function getSessionAndToken(slug: string): Promise<{ cookies: string; token: string }> { + const response = await fetch(`${BASE_URL}/${slug}`, { + headers: { 'User-Agent': USER_AGENT }, + }) + if (!response.ok) { + throw new Error(`Failed to load ${slug} category page: HTTP ${response.status}`) + } + const html = await response.text() + + const setCookieHeaders = response.headers.getSetCookie?.() ?? [] + const cookies = setCookieHeaders.map((cookie) => cookie.split(';')[0]).join('; ') + + const tokenMatch = html.match(/"_token":\s*"([^"]+)"/) + if (!tokenMatch) { + throw new Error(`Could not find CSRF token on ${slug} category page`) + } + + return { cookies, token: tokenMatch[1] } +} + +/** + * Parses the "Name" / "Add to Build" cell HTML fragments returned by + * /products/data into a single row, using the exact same markup structure + * as the plain category page (both come from the same Blade partials), so + * the same selectors apply. + */ +function parseRow(nameHtml: string, addToBuildHtml: string): ScrapedRow | null { + const $name = cheerio.load(nameHtml) + const nameLink = $name('a').first() + const name = nameLink.text().trim() + const productPageUrl = nameLink.attr('href') + const img = $name('img.product-image') + const imageUrl = img.attr('data-src') || img.attr('src') || null + + const $btn = cheerio.load(addToBuildHtml) + const priceText = $btn('.price-box p').first().text().trim() + const sourceRef = $btn('a.add-to-build').attr('data-id') + const amazonUrl = $btn('a.btn-default').attr('href') + + if (!name || !productPageUrl || !sourceRef) { + // Only truly unusable rows (missing name/link/id) are skipped. A + // missing price is not disqualifying — see the ScrapedRow.price doc. + return null + } + + let price: number | null = null + if (priceText) { + const parsed = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.')) + price = Number.isNaN(parsed) ? null : parsed + } + + return { name, price, imageUrl, purchaseUrl: amazonUrl || productPageUrl, sourceRef } +} + +/** + * Fetches one page of `length` products starting at `start` from + * pc-builder.io's DataTables-backed /products/data endpoint. This is a + * plain HTTP POST (no headless browser needed) once the CSRF + * token/session cookie from getSessionAndToken() are supplied — see + * app/services/part_scraper_service.ts git history / PR description for + * the reverse-engineered request shape (field names matter exactly: + * columns[].data must be "Name"/"Add to Build", not numeric indices). + */ +async function fetchProductsPage( + categoryId: string, + cookies: string, + token: string, + start: number, + length: number +): Promise<{ rows: ScrapedRow[]; recordsTotal: number }> { + const body = new URLSearchParams({ + '_token': token, + 'draw': '1', + 'start': String(start), + 'length': String(length), + 'compatibility_filter': 'false', + 'filter_status': '', + 'category_id': categoryId, + 'compatible_with': '', + 'checkbox_filters': '[]', + 'range_filters': '[]', + 'search_query': '', + 'columns[0][data]': 'Name', + 'columns[0][name]': 'Name', + 'columns[0][searchable]': 'true', + 'columns[0][orderable]': 'false', + 'columns[0][search][value]': '', + 'columns[0][search][regex]': 'false', + 'columns[1][data]': 'Add to Build', + 'columns[1][name]': 'Add to Build', + 'columns[1][searchable]': 'true', + 'columns[1][orderable]': 'false', + 'columns[1][search][value]': '', + 'columns[1][search][regex]': 'false', + 'search[value]': '', + 'search[regex]': 'false', + }) + + const response = await fetch(`${BASE_URL}/products/data`, { + method: 'POST', headers: { - 'User-Agent': 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)', + 'User-Agent': USER_AGENT, + 'X-Requested-With': 'XMLHttpRequest', + 'Content-Type': 'application/x-www-form-urlencoded', + 'Cookie': cookies, }, + body, }) if (!response.ok) { - throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`) + throw new Error(`products/data request failed: HTTP ${response.status}`) } - const html = await response.text() - const $ = cheerio.load(html) + const json = (await response.json()) as { + recordsTotal: number + data: { 'Name': string; 'Add to Build': string }[] + } + + const rows = json.data + .map((entry) => parseRow(entry.Name, entry['Add to Build'])) + .filter((row): row is ScrapedRow => row !== null) + + return { rows, recordsTotal: json.recordsTotal } +} + +/** + * Fetches up to `maxProducts` products for a category by paginating + * through pc-builder.io's /products/data endpoint PAGE_SIZE at a time, + * stopping early once the category's real total is exhausted. A short + * delay between pages avoids hammering the site with a burst of requests + * back-to-back for the larger categories (e.g. GPU's ~60 pages). + */ +async function fetchCategoryProducts( + slug: string, + categoryId: string, + maxProducts: number +): Promise { + const { cookies, token } = await getSessionAndToken(slug) + const rows: ScrapedRow[] = [] + let start = 0 + let recordsTotal = maxProducts - $('#products-table tbody tr').each((_, el) => { - const row = $(el) - const nameLink = row.find('td.product a').first() - const name = nameLink.text().trim() - const productPageUrl = nameLink.attr('href') - const img = row.find('td.product img.product-image') - const imageUrl = img.attr('data-src') || img.attr('src') || null - const priceText = row.find('td.add-btn .price-box p').first().text().trim() - const addButton = row.find('a.add-to-build') - const sourceRef = addButton.attr('data-id') - // Real merchant link ("View on Amazon"), either a direct product page - // (https://www.amazon.fr/dp/{ASIN}?tag=...) when pc-builder.io knows the - // ASIN, or a generic search results page (https://www.amazon.fr/s?k=...) - // otherwise. Either way it's a genuine purchase link, unlike the - // pc-builder.io product page. - const amazonUrl = row.find('a.btn-default').attr('href') + while (start < maxProducts && start < recordsTotal) { + const length = Math.min(PAGE_SIZE, maxProducts - start) + const page = await fetchProductsPage(categoryId, cookies, token, start, length) + rows.push(...page.rows) + recordsTotal = page.recordsTotal + start += length - if (!name || !productPageUrl || !sourceRef || !priceText) { - // Skip rows without a resolved price (e.g. "search on Amazon" links - // with no fixed offer) or missing identifiers. - return + if (start < maxProducts && start < recordsTotal) { + await new Promise((resolve) => setTimeout(resolve, 150)) } - - const price = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.')) - if (Number.isNaN(price)) { - return - } - - rows.push({ name, price, imageUrl, purchaseUrl: amazonUrl || productPageUrl, sourceRef }) - }) + } return rows } @@ -111,27 +254,57 @@ async function fetchCategoryPage(slug: string): Promise { /** * Scrapes a single component type's mapped category and upserts the * resulting rows into the scraped_parts cache table. + * + * @param maxProductsOverride Caps how many products are fetched for this + * run, regardless of the category's configured maxProducts. Used by + * ensureFreshCategory to keep an on-demand refresh fast (a handful of + * pages) while `node ace scrape:parts` uses the full configured depth. */ -async function scrapeComponentType(componentType: ComponentType): Promise { - const slug = CATEGORY_SLUGS[componentType.name] - if (!slug) { +async function scrapeComponentType( + componentType: ComponentType, + maxProductsOverride?: number +): Promise { + const config = CATEGORY_CONFIG[componentType.name] + if (!config) { logger.warn({ type: componentType.name }, 'No pc-builder.io category mapped, skipping') return 0 } - const rows = await fetchCategoryPage(slug) + const maxProducts = Math.min(maxProductsOverride ?? config.maxProducts, config.maxProducts) + const rows = await fetchCategoryProducts(config.slug, config.categoryId, maxProducts) const scrapedAt = DateTime.now() for (const row of rows) { + const existing = await ScrapedPart.query() + .where('source', SOURCE) + .andWhere('sourceRef', row.sourceRef) + .first() + + // If a previous price comparison already resolved (and persisted) a + // real price for this part from a merchant, don't blow it away just + // because this catalog refresh came back with no price — keep + // whichever is best (see updatePriceIfCheaper for the "best of all + // known sources" logic used when a comparison finds a lower price). + const hasBetterExistingPrice = + existing?.price !== null && + existing?.price !== undefined && + (row.price === null || existing.price <= row.price) + await ScrapedPart.updateOrCreate( { source: SOURCE, sourceRef: row.sourceRef }, { typeId: componentType.id, name: row.name, - price: row.price, imageUrl: row.imageUrl, - sourceUrl: row.purchaseUrl, scrapedAt, + ...(hasBetterExistingPrice + ? {} + : { + price: row.price, + sourceUrl: row.purchaseUrl, + priceSource: row.price !== null ? SOURCE : null, + priceCheckedAt: row.price !== null ? scrapedAt : null, + }), } ) } @@ -141,14 +314,18 @@ async function scrapeComponentType(componentType: ComponentType): Promise> { const componentTypes = await ComponentType.all() const summary: Record = {} for (const componentType of componentTypes) { - if (!CATEGORY_SLUGS[componentType.name]) continue + if (!CATEGORY_CONFIG[componentType.name]) continue try { summary[componentType.name] = await scrapeComponentType(componentType) } catch (error) { @@ -181,23 +358,52 @@ async function isCategoryStale(componentType: ComponentType): Promise { * if it's missing or stale. No-op for unmapped types. Scraping errors are * logged but not thrown, so a search still falls back to whatever is * already cached. + * + * Only fetches QUICK_REFRESH_MAX_PRODUCTS products (not the category's + * full configured depth), since this runs inline in a user's search + * request — run `node ace scrape:parts` for full-depth coverage. */ async function ensureFreshCategory(componentType: ComponentType): Promise { - if (!CATEGORY_SLUGS[componentType.name]) return + if (!CATEGORY_CONFIG[componentType.name]) return if (!(await isCategoryStale(componentType))) return try { - await scrapeComponentType(componentType) + await scrapeComponentType(componentType, QUICK_REFRESH_MAX_PRODUCTS) } catch (error) { logger.error({ type: componentType.name, err: error }, 'Failed to refresh stale category cache') } } +/** + * Persists `offer` as a part's new price/source if it's cheaper than + * whatever is currently stored (or nothing was stored yet), so the "best + * known price" keeps improving as users compare prices (see + * ScrapedPartsController#compare) instead of resetting to a possibly + * stale/missing pc-builder.io price on the next catalog refresh — + * scrapeComponentType already refuses to overwrite a price with a worse + * one using this same "keep whichever is lower" rule. + */ +async function updatePriceIfCheaper( + part: ScrapedPart, + offer: { merchant: string; price: number; url: string } +): Promise { + if (part.price !== null && part.price <= offer.price) return + + part.price = offer.price + part.sourceUrl = offer.url + part.priceSource = offer.merchant + part.priceCheckedAt = DateTime.now() + await part.save() +} + export { scrapeAllCategories, scrapeComponentType, ensureFreshCategory, isCategoryStale, + updatePriceIfCheaper, + CATEGORY_CONFIG, CATEGORY_SLUGS, CACHE_TTL_HOURS, + QUICK_REFRESH_MAX_PRODUCTS, } diff --git a/database/migrations/1784815399002_alter_scrapeds_table.ts b/database/migrations/1784815399002_alter_scrapeds_table.ts new file mode 100644 index 0000000..f22b6cc --- /dev/null +++ b/database/migrations/1784815399002_alter_scrapeds_table.ts @@ -0,0 +1,32 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'scraped_parts' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + // Not every pc-builder.io listing has a fixed price (recent GPUs in + // particular often only have a generic "search on Amazon" link with + // no resolved offer) — these are now kept instead of being skipped, + // so price must be nullable. + table.float('price', 8, 2).nullable().alter() + // Tracks where the currently stored `price`/`source_url` came from: + // "pc-builder.io" for the original catalog price, or one of the + // merchant names from merchant_price_service.ts (Amazon, LDLC, ...) + // once a live price comparison finds (and persists) a cheaper offer. + // Null means no price has ever been resolved for this part. + table.string('price_source').nullable() + table.timestamp('price_checked_at').nullable() + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('price_source') + table.dropColumn('price_checked_at') + }) + this.schema.alterTable(this.tableName, (table) => { + table.float('price', 8, 2).notNullable().alter() + }) + } +} diff --git a/database/schema.ts b/database/schema.ts index 06b97d3..0ddcb67 100644 --- a/database/schema.ts +++ b/database/schema.ts @@ -103,7 +103,7 @@ export class ComputerSchema extends BaseModel { } export class ScrapedPartSchema extends BaseModel { - static $columns = ['createdAt', 'id', 'imageUrl', 'name', 'price', 'scrapedAt', 'source', 'sourceRef', 'sourceUrl', 'typeId', 'updatedAt'] as const + static $columns = ['createdAt', 'id', 'imageUrl', 'name', 'price', 'priceCheckedAt', 'priceSource', 'scrapedAt', 'source', 'sourceRef', 'sourceUrl', 'typeId', 'updatedAt'] as const $columns = ScrapedPartSchema.$columns @column.dateTime({ autoCreate: true }) declare createdAt: DateTime | null @@ -114,7 +114,11 @@ export class ScrapedPartSchema extends BaseModel { @column() declare name: string @column() - declare price: number + declare price: number | null + @column.dateTime() + declare priceCheckedAt: DateTime | null + @column() + declare priceSource: string | null @column.dateTime() declare scrapedAt: DateTime @column() diff --git a/inertia/pages/components/create.vue b/inertia/pages/components/create.vue index a7d45e1..6e5c3df 100644 --- a/inertia/pages/components/create.vue +++ b/inertia/pages/components/create.vue @@ -19,9 +19,11 @@ const form = reactive({ function importPart(part: ScrapedPart, cheapestOffer: MerchantOffer | null) { form.name = part.name // Prefer the live price/link found by comparing merchants (Amazon, LDLC, - // Grosbill, TopAchat) when available, since pc-builder.io's catalog price - // can be stale; fall back to the catalog price/link otherwise. - form.price = cheapestOffer?.price ?? part.price + // Grosbill, TopAchat, ...) when available, since pc-builder.io's catalog + // price can be stale (or simply unknown — see ScrapedPart.price doc); + // fall back to the catalog price/link, or leave the price field as-is + // for the user to fill in manually if neither is known. + form.price = cheapestOffer?.price ?? part.price ?? form.price form.typeId = part.type.id form.purchaseDate = new Date().toISOString().split('T')[0] form.purchaseUrl = cheapestOffer?.url ?? part.sourceUrl diff --git a/inertia/pages/types/UITypes.ts b/inertia/pages/types/UITypes.ts index 771c94d..277108e 100644 --- a/inertia/pages/types/UITypes.ts +++ b/inertia/pages/types/UITypes.ts @@ -54,7 +54,10 @@ export type Component = { export type ScrapedPart = { id: number name: string - price: number + // Null when pc-builder.io has no fixed offer for this part yet (common + // for very recent products). Comparing prices (see NewPartSearch.vue) + // can resolve and persist a real price from another merchant. + price: number | null imageUrl: string | null source: string sourceUrl: string diff --git a/inertia/pages/widgets/NewPartSearch.vue b/inertia/pages/widgets/NewPartSearch.vue index f788899..0951e9d 100644 --- a/inertia/pages/widgets/NewPartSearch.vue +++ b/inertia/pages/widgets/NewPartSearch.vue @@ -126,7 +126,11 @@ function importPart(part: ScrapedPart) { />

{{ part.name }}

-

{{ part.type.name }}

+

+ {{ part.type.name }} + · €{{ part.price.toFixed(2) }} + · No price yet +