From 60532905010dba923cfe1315a513b3954b895c60 Mon Sep 17 00:00:00 2001 From: Kevin L'Hereec Date: Thu, 23 Jul 2026 16:12:26 +0200 Subject: [PATCH] feat: paginate pc-builder.io scraping and always keep the best known price The RTX 5070 case that prompted this: only 100 GPUs out of pc-builder.io's actual 6222 were being scraped (its category pages only render the first page server-side), so cheaper RTX 5070 variants existed on the site but were invisible to our search. Investigating further also surfaced that even with full pagination, most recent GPU listings (RTX 5070 included) have no fixed Amazon price on pc-builder.io at all - just a generic "search on Amazon" link - so pagination alone doesn't fully solve price coverage either. Pagination (part_scraper_service.ts): - Reverse-engineered pc-builder.io's /products/data endpoint, which powers the client-side DataTables widget behind every category page. Requires a session cookie + CSRF token scraped from a fresh category page load, and an exact field shape (columns[].data must be the literal strings "Name"/"Add to Build", not numeric indices - a naive DataTables-shaped request 500s otherwise). - CATEGORY_CONFIG now tracks each category's real pc-builder.io category_id and a per-category maxProducts depth, chosen by actual catalog size and price-sensitivity: GPU near-full coverage (6000), smaller/less price-volatile categories capped lower (300-800) to keep a full `scrape:parts` run (all categories) in a reasonable time. - ensureFreshCategory (triggered inline by a user's search when the cache is stale) uses a much shallower QUICK_REFRESH_MAX_PRODUCTS=200 cap instead of the category's full depth, since it runs inside the request/response cycle of a search - full-depth coverage is only done by the standalone `scrape:parts` command. Always keep the best known price: - scraped_parts.price is now nullable (migration): parts with no fixed pc-builder.io offer are kept and searchable instead of being silently skipped, so at least the model is discoverable even before a price is resolved. - New price_source / price_checked_at columns track where the currently stored price came from ("pc-builder.io" or a merchant name from merchant_price_service.ts). - ScrapedPartsController#compare now persists the cheapest offer found (via the new updatePriceIfCheaper()) when it beats the currently stored price - so comparing prices benefits every future search/import for that part, not just the user who triggered it. Never regresses the price: a comparison finding a higher price than what's stored is ignored. - scrapeComponentType's catalog refresh applies the same "keep whichever price is lower" rule per row, so a routine pc-builder.io re-scrape can never overwrite a better price a comparison had already found and persisted with a worse (or missing) one. - Search results (GET /scraped-parts) are now ordered priced-cheapest-first, then parts with no known price last (`ORDER BY price IS NULL, price ASC`), so unpriced parts stay visible without pushing priced results down. - NewPartSearch.vue displays the catalog price inline (or "No price yet"), and the existing "Compare prices" flow now works for those unpriced parts the same way it does for priced ones. 7 new/updated functional tests: nullable price is searchable and sorted last, and 3 tests for updatePriceIfCheaper (cheaper offer replaces price, pricier offer is ignored, first-ever price is accepted for a previously unpriced part). --- app/controllers/scraped_parts_controller.ts | 28 +- app/services/part_scraper_service.ts | 340 ++++++++++++++---- .../1784815399002_alter_scrapeds_table.ts | 32 ++ database/schema.ts | 8 +- inertia/pages/components/create.vue | 8 +- inertia/pages/types/UITypes.ts | 5 +- inertia/pages/widgets/NewPartSearch.vue | 11 +- tests/functional/scraped_parts.spec.ts | 69 +++- 8 files changed, 418 insertions(+), 83 deletions(-) create mode 100644 database/migrations/1784815399002_alter_scrapeds_table.ts diff --git a/app/controllers/scraped_parts_controller.ts b/app/controllers/scraped_parts_controller.ts index b4bca15..9c0b775 100644 --- a/app/controllers/scraped_parts_controller.ts +++ b/app/controllers/scraped_parts_controller.ts @@ -2,7 +2,7 @@ import type { HttpContext } from '@adonisjs/core/http' import vine from '@vinejs/vine' import ScrapedPart from '#models/scraped_part' import ComponentType from '#models/component_type' -import { ensureFreshCategory } from '#services/part_scraper_service' +import { ensureFreshCategory, updatePriceIfCheaper } from '#services/part_scraper_service' import { findCheapestOffers } from '#services/merchant_price_service' const searchValidator = vine.compile( @@ -31,6 +31,11 @@ export default class ScrapedPartsController { * all types skip this eager refresh to avoid re-scraping every mapped * category on every keystroke; run `node ace scrape:parts` to refresh * everything at once. + * + * Results are ordered with a resolved price first (cheapest first), then + * parts with no known price yet (see ScrapedPart.price doc) last — so a + * part is never hidden just because pc-builder.io hasn't found a fixed + * Amazon offer for it, but priced parts still surface first. */ async index({ request, auth, response }: HttpContext) { await auth.authenticate() @@ -43,7 +48,9 @@ export default class ScrapedPartsController { } } - const query = ScrapedPart.query().preload('type').orderBy('name') + const query = ScrapedPart.query() + .preload('type') + .orderByRaw('price IS NULL, price ASC, name ASC') if (q) { query.whereRaw('LOWER(name) LIKE ?', [`%${q.toLowerCase()}%`]) @@ -58,10 +65,15 @@ export default class ScrapedPartsController { /** * Compares live prices for a given scraped part across Amazon, LDLC, - * Grosbill and TopAchat, and returns them sorted cheapest first. This is - * triggered on demand (one part at a time) rather than for every search - * result, since it makes several outbound HTTP requests and can take a - * few seconds. + * Grosbill, TopAchat, Materiel.net, Cdiscount and PCComponentes, and + * returns them sorted cheapest first. Triggered on demand (one part at a + * time) rather than for every search result, since it makes several + * outbound requests and can take a few seconds. + * + * If the cheapest offer found beats the part's currently stored price + * (or no price was known yet), it's persisted as the new default price + * (see updatePriceIfCheaper) — so this comparison benefits every future + * search/import for this part, not just the current user. */ async compare({ request, auth, response }: HttpContext) { await auth.authenticate() @@ -75,6 +87,10 @@ export default class ScrapedPartsController { const amazonUrl = part.sourceUrl.includes('amazon.') ? part.sourceUrl : null const offers = await findCheapestOffers(part.name, amazonUrl) + if (offers.length > 0) { + await updatePriceIfCheaper(part, offers[0]) + } + return response.json({ offers }) } } diff --git a/app/services/part_scraper_service.ts b/app/services/part_scraper_service.ts index d4765a9..b3d7b08 100644 --- a/app/services/part_scraper_service.ts +++ b/app/services/part_scraper_service.ts @@ -6,6 +6,8 @@ import ScrapedPart from '#models/scraped_part' const SOURCE = 'pc-builder.io' const BASE_URL = 'https://fr.pc-builder.io' +const USER_AGENT = 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)' +const PAGE_SIZE = 100 /** * Default freshness window for the scraped_parts cache. A category is @@ -15,95 +17,236 @@ const BASE_URL = 'https://fr.pc-builder.io' const CACHE_TTL_HOURS = 24 /** - * Maps our own ComponentType names to the corresponding pc-builder.io - * category slug. Categories not listed here are not scraped. + * How many products to fetch when a search triggers an automatic refresh + * (see ensureFreshCategory). Kept small since this runs synchronously + * inside a user-facing HTTP request — a full deep scrape (thousands of + * products for some categories) belongs in the `scrape:parts` command + * instead, not in the request/response cycle of a search. + */ +const QUICK_REFRESH_MAX_PRODUCTS = 200 + +/** + * Maps our own ComponentType names to pc-builder.io's category slug, + * numeric category_id (required by the paginated /products/data endpoint, + * found in each category page's inline DataTables config), and how many + * products `node ace scrape:parts` should fetch for that category. + * + * maxProducts is chosen per category based on how much price variance + * matters there and how large the catalog is: GPUs get close-to-full + * coverage since price differences between models are large and users + * specifically look for cheaper alternatives (see the RTX 5070 case that + * prompted this), while smaller/less price-sensitive categories use a + * lighter cap to keep the full scrape run in a reasonable time. * * Note: pc-builder.io only has a single "storage" category covering both * SSDs and HDDs (no separate slugs exist), so it is only mapped to our * "SSD" type to avoid two types racing to own the same scraped rows * (source_ref is unique per source, independent of type_id). */ -const CATEGORY_SLUGS: Record = { - 'Processor': 'cpu', - 'Motherboard': 'motherboard', - 'Graphic Card': 'gpu', - 'Memory': 'ram', - 'SSD': 'storage', - 'Cooler': 'cpu-cooler', - 'Power Supply': 'power-supply', - 'Case': 'case', - 'Fan': 'case-fan', - 'WiFi-Card': 'wireless-network-adapter', +const CATEGORY_CONFIG: Record = { + 'Processor': { slug: 'cpu', categoryId: '1', maxProducts: 500 }, + 'Motherboard': { slug: 'motherboard', categoryId: '3', maxProducts: 500 }, + 'Graphic Card': { slug: 'gpu', categoryId: '6', maxProducts: 6000 }, + 'Memory': { slug: 'ram', categoryId: '4', maxProducts: 800 }, + 'SSD': { slug: 'storage', categoryId: '5', maxProducts: 800 }, + 'Cooler': { slug: 'cpu-cooler', categoryId: '2', maxProducts: 400 }, + 'Power Supply': { slug: 'power-supply', categoryId: '8', maxProducts: 400 }, + 'Case': { slug: 'case', categoryId: '7', maxProducts: 400 }, + 'Fan': { slug: 'case-fan', categoryId: '20', maxProducts: 300 }, + 'WiFi-Card': { slug: 'wireless-network-adapter', categoryId: '13', maxProducts: 400 }, } +/** Backwards-compatible slug lookup, kept for anything relying on it. */ +const CATEGORY_SLUGS: Record = Object.fromEntries( + Object.entries(CATEGORY_CONFIG).map(([name, config]) => [name, config.slug]) +) + interface ScrapedRow { name: string - price: number + /** + * Null when pc-builder.io has no fixed offer for this part (common for + * very recent products — it only links to a generic Amazon search, + * e.g. "https://amazon.fr/s?k=..." instead of a priced "/dp/{ASIN}" + * page). Such parts are still kept (not skipped) so they remain + * searchable and importable; ScrapedPartsController#compare lets the + * user resolve — and persist — a real price for them on demand from + * the other supported merchants. + */ + price: number | null imageUrl: string | null /** * The actual merchant page where this part can be bought (currently * always an Amazon.fr link, since that's the only "buy" link pc-builder.io - * exposes on its category pages). This is what gets stored as - * ScrapedPart.sourceUrl and later copied into Component.purchaseUrl — it - * intentionally is NOT the pc-builder.io product page, which isn't a - * place a user can actually purchase anything from. + * exposes on its category pages) — either a direct priced product page + * or a generic search results page when no price is known. This is + * what gets stored as ScrapedPart.sourceUrl and later copied into + * Component.purchaseUrl — it intentionally is NOT the pc-builder.io + * product page, which isn't a place a user can actually purchase + * anything from. */ purchaseUrl: string sourceRef: string } /** - * Fetches a pc-builder.io category page and parses the (server-rendered) - * product table into a list of rows. Only the first ~100 products per - * category are available without simulating the site's DataTables/CSRF - * flow, which is an accepted limitation for this cache. + * Loads a category page once to grab the CSRF token and session cookie + * needed to call the paginated /products/data endpoint, which is what + * powers the client-side DataTables widget on every category page. */ -async function fetchCategoryPage(slug: string): Promise { - const url = `${BASE_URL}/${slug}` - const response = await fetch(url, { +async function getSessionAndToken(slug: string): Promise<{ cookies: string; token: string }> { + const response = await fetch(`${BASE_URL}/${slug}`, { + headers: { 'User-Agent': USER_AGENT }, + }) + if (!response.ok) { + throw new Error(`Failed to load ${slug} category page: HTTP ${response.status}`) + } + const html = await response.text() + + const setCookieHeaders = response.headers.getSetCookie?.() ?? [] + const cookies = setCookieHeaders.map((cookie) => cookie.split(';')[0]).join('; ') + + const tokenMatch = html.match(/"_token":\s*"([^"]+)"/) + if (!tokenMatch) { + throw new Error(`Could not find CSRF token on ${slug} category page`) + } + + return { cookies, token: tokenMatch[1] } +} + +/** + * Parses the "Name" / "Add to Build" cell HTML fragments returned by + * /products/data into a single row, using the exact same markup structure + * as the plain category page (both come from the same Blade partials), so + * the same selectors apply. + */ +function parseRow(nameHtml: string, addToBuildHtml: string): ScrapedRow | null { + const $name = cheerio.load(nameHtml) + const nameLink = $name('a').first() + const name = nameLink.text().trim() + const productPageUrl = nameLink.attr('href') + const img = $name('img.product-image') + const imageUrl = img.attr('data-src') || img.attr('src') || null + + const $btn = cheerio.load(addToBuildHtml) + const priceText = $btn('.price-box p').first().text().trim() + const sourceRef = $btn('a.add-to-build').attr('data-id') + const amazonUrl = $btn('a.btn-default').attr('href') + + if (!name || !productPageUrl || !sourceRef) { + // Only truly unusable rows (missing name/link/id) are skipped. A + // missing price is not disqualifying — see the ScrapedRow.price doc. + return null + } + + let price: number | null = null + if (priceText) { + const parsed = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.')) + price = Number.isNaN(parsed) ? null : parsed + } + + return { name, price, imageUrl, purchaseUrl: amazonUrl || productPageUrl, sourceRef } +} + +/** + * Fetches one page of `length` products starting at `start` from + * pc-builder.io's DataTables-backed /products/data endpoint. This is a + * plain HTTP POST (no headless browser needed) once the CSRF + * token/session cookie from getSessionAndToken() are supplied — see + * app/services/part_scraper_service.ts git history / PR description for + * the reverse-engineered request shape (field names matter exactly: + * columns[].data must be "Name"/"Add to Build", not numeric indices). + */ +async function fetchProductsPage( + categoryId: string, + cookies: string, + token: string, + start: number, + length: number +): Promise<{ rows: ScrapedRow[]; recordsTotal: number }> { + const body = new URLSearchParams({ + '_token': token, + 'draw': '1', + 'start': String(start), + 'length': String(length), + 'compatibility_filter': 'false', + 'filter_status': '', + 'category_id': categoryId, + 'compatible_with': '', + 'checkbox_filters': '[]', + 'range_filters': '[]', + 'search_query': '', + 'columns[0][data]': 'Name', + 'columns[0][name]': 'Name', + 'columns[0][searchable]': 'true', + 'columns[0][orderable]': 'false', + 'columns[0][search][value]': '', + 'columns[0][search][regex]': 'false', + 'columns[1][data]': 'Add to Build', + 'columns[1][name]': 'Add to Build', + 'columns[1][searchable]': 'true', + 'columns[1][orderable]': 'false', + 'columns[1][search][value]': '', + 'columns[1][search][regex]': 'false', + 'search[value]': '', + 'search[regex]': 'false', + }) + + const response = await fetch(`${BASE_URL}/products/data`, { + method: 'POST', headers: { - 'User-Agent': 'Mozilla/5.0 (compatible; PcBuilderPartFinder/1.0)', + 'User-Agent': USER_AGENT, + 'X-Requested-With': 'XMLHttpRequest', + 'Content-Type': 'application/x-www-form-urlencoded', + 'Cookie': cookies, }, + body, }) if (!response.ok) { - throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`) + throw new Error(`products/data request failed: HTTP ${response.status}`) } - const html = await response.text() - const $ = cheerio.load(html) + const json = (await response.json()) as { + recordsTotal: number + data: { 'Name': string; 'Add to Build': string }[] + } + + const rows = json.data + .map((entry) => parseRow(entry.Name, entry['Add to Build'])) + .filter((row): row is ScrapedRow => row !== null) + + return { rows, recordsTotal: json.recordsTotal } +} + +/** + * Fetches up to `maxProducts` products for a category by paginating + * through pc-builder.io's /products/data endpoint PAGE_SIZE at a time, + * stopping early once the category's real total is exhausted. A short + * delay between pages avoids hammering the site with a burst of requests + * back-to-back for the larger categories (e.g. GPU's ~60 pages). + */ +async function fetchCategoryProducts( + slug: string, + categoryId: string, + maxProducts: number +): Promise { + const { cookies, token } = await getSessionAndToken(slug) + const rows: ScrapedRow[] = [] + let start = 0 + let recordsTotal = maxProducts - $('#products-table tbody tr').each((_, el) => { - const row = $(el) - const nameLink = row.find('td.product a').first() - const name = nameLink.text().trim() - const productPageUrl = nameLink.attr('href') - const img = row.find('td.product img.product-image') - const imageUrl = img.attr('data-src') || img.attr('src') || null - const priceText = row.find('td.add-btn .price-box p').first().text().trim() - const addButton = row.find('a.add-to-build') - const sourceRef = addButton.attr('data-id') - // Real merchant link ("View on Amazon"), either a direct product page - // (https://www.amazon.fr/dp/{ASIN}?tag=...) when pc-builder.io knows the - // ASIN, or a generic search results page (https://www.amazon.fr/s?k=...) - // otherwise. Either way it's a genuine purchase link, unlike the - // pc-builder.io product page. - const amazonUrl = row.find('a.btn-default').attr('href') + while (start < maxProducts && start < recordsTotal) { + const length = Math.min(PAGE_SIZE, maxProducts - start) + const page = await fetchProductsPage(categoryId, cookies, token, start, length) + rows.push(...page.rows) + recordsTotal = page.recordsTotal + start += length - if (!name || !productPageUrl || !sourceRef || !priceText) { - // Skip rows without a resolved price (e.g. "search on Amazon" links - // with no fixed offer) or missing identifiers. - return + if (start < maxProducts && start < recordsTotal) { + await new Promise((resolve) => setTimeout(resolve, 150)) } - - const price = Number.parseFloat(priceText.replace(/[^\d.,]/g, '').replace(',', '.')) - if (Number.isNaN(price)) { - return - } - - rows.push({ name, price, imageUrl, purchaseUrl: amazonUrl || productPageUrl, sourceRef }) - }) + } return rows } @@ -111,27 +254,57 @@ async function fetchCategoryPage(slug: string): Promise { /** * Scrapes a single component type's mapped category and upserts the * resulting rows into the scraped_parts cache table. + * + * @param maxProductsOverride Caps how many products are fetched for this + * run, regardless of the category's configured maxProducts. Used by + * ensureFreshCategory to keep an on-demand refresh fast (a handful of + * pages) while `node ace scrape:parts` uses the full configured depth. */ -async function scrapeComponentType(componentType: ComponentType): Promise { - const slug = CATEGORY_SLUGS[componentType.name] - if (!slug) { +async function scrapeComponentType( + componentType: ComponentType, + maxProductsOverride?: number +): Promise { + const config = CATEGORY_CONFIG[componentType.name] + if (!config) { logger.warn({ type: componentType.name }, 'No pc-builder.io category mapped, skipping') return 0 } - const rows = await fetchCategoryPage(slug) + const maxProducts = Math.min(maxProductsOverride ?? config.maxProducts, config.maxProducts) + const rows = await fetchCategoryProducts(config.slug, config.categoryId, maxProducts) const scrapedAt = DateTime.now() for (const row of rows) { + const existing = await ScrapedPart.query() + .where('source', SOURCE) + .andWhere('sourceRef', row.sourceRef) + .first() + + // If a previous price comparison already resolved (and persisted) a + // real price for this part from a merchant, don't blow it away just + // because this catalog refresh came back with no price — keep + // whichever is best (see updatePriceIfCheaper for the "best of all + // known sources" logic used when a comparison finds a lower price). + const hasBetterExistingPrice = + existing?.price !== null && + existing?.price !== undefined && + (row.price === null || existing.price <= row.price) + await ScrapedPart.updateOrCreate( { source: SOURCE, sourceRef: row.sourceRef }, { typeId: componentType.id, name: row.name, - price: row.price, imageUrl: row.imageUrl, - sourceUrl: row.purchaseUrl, scrapedAt, + ...(hasBetterExistingPrice + ? {} + : { + price: row.price, + sourceUrl: row.purchaseUrl, + priceSource: row.price !== null ? SOURCE : null, + priceCheckedAt: row.price !== null ? scrapedAt : null, + }), } ) } @@ -141,14 +314,18 @@ async function scrapeComponentType(componentType: ComponentType): Promise> { const componentTypes = await ComponentType.all() const summary: Record = {} for (const componentType of componentTypes) { - if (!CATEGORY_SLUGS[componentType.name]) continue + if (!CATEGORY_CONFIG[componentType.name]) continue try { summary[componentType.name] = await scrapeComponentType(componentType) } catch (error) { @@ -181,23 +358,52 @@ async function isCategoryStale(componentType: ComponentType): Promise { * if it's missing or stale. No-op for unmapped types. Scraping errors are * logged but not thrown, so a search still falls back to whatever is * already cached. + * + * Only fetches QUICK_REFRESH_MAX_PRODUCTS products (not the category's + * full configured depth), since this runs inline in a user's search + * request — run `node ace scrape:parts` for full-depth coverage. */ async function ensureFreshCategory(componentType: ComponentType): Promise { - if (!CATEGORY_SLUGS[componentType.name]) return + if (!CATEGORY_CONFIG[componentType.name]) return if (!(await isCategoryStale(componentType))) return try { - await scrapeComponentType(componentType) + await scrapeComponentType(componentType, QUICK_REFRESH_MAX_PRODUCTS) } catch (error) { logger.error({ type: componentType.name, err: error }, 'Failed to refresh stale category cache') } } +/** + * Persists `offer` as a part's new price/source if it's cheaper than + * whatever is currently stored (or nothing was stored yet), so the "best + * known price" keeps improving as users compare prices (see + * ScrapedPartsController#compare) instead of resetting to a possibly + * stale/missing pc-builder.io price on the next catalog refresh — + * scrapeComponentType already refuses to overwrite a price with a worse + * one using this same "keep whichever is lower" rule. + */ +async function updatePriceIfCheaper( + part: ScrapedPart, + offer: { merchant: string; price: number; url: string } +): Promise { + if (part.price !== null && part.price <= offer.price) return + + part.price = offer.price + part.sourceUrl = offer.url + part.priceSource = offer.merchant + part.priceCheckedAt = DateTime.now() + await part.save() +} + export { scrapeAllCategories, scrapeComponentType, ensureFreshCategory, isCategoryStale, + updatePriceIfCheaper, + CATEGORY_CONFIG, CATEGORY_SLUGS, CACHE_TTL_HOURS, + QUICK_REFRESH_MAX_PRODUCTS, } diff --git a/database/migrations/1784815399002_alter_scrapeds_table.ts b/database/migrations/1784815399002_alter_scrapeds_table.ts new file mode 100644 index 0000000..f22b6cc --- /dev/null +++ b/database/migrations/1784815399002_alter_scrapeds_table.ts @@ -0,0 +1,32 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'scraped_parts' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + // Not every pc-builder.io listing has a fixed price (recent GPUs in + // particular often only have a generic "search on Amazon" link with + // no resolved offer) — these are now kept instead of being skipped, + // so price must be nullable. + table.float('price', 8, 2).nullable().alter() + // Tracks where the currently stored `price`/`source_url` came from: + // "pc-builder.io" for the original catalog price, or one of the + // merchant names from merchant_price_service.ts (Amazon, LDLC, ...) + // once a live price comparison finds (and persists) a cheaper offer. + // Null means no price has ever been resolved for this part. + table.string('price_source').nullable() + table.timestamp('price_checked_at').nullable() + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('price_source') + table.dropColumn('price_checked_at') + }) + this.schema.alterTable(this.tableName, (table) => { + table.float('price', 8, 2).notNullable().alter() + }) + } +} diff --git a/database/schema.ts b/database/schema.ts index 06b97d3..0ddcb67 100644 --- a/database/schema.ts +++ b/database/schema.ts @@ -103,7 +103,7 @@ export class ComputerSchema extends BaseModel { } export class ScrapedPartSchema extends BaseModel { - static $columns = ['createdAt', 'id', 'imageUrl', 'name', 'price', 'scrapedAt', 'source', 'sourceRef', 'sourceUrl', 'typeId', 'updatedAt'] as const + static $columns = ['createdAt', 'id', 'imageUrl', 'name', 'price', 'priceCheckedAt', 'priceSource', 'scrapedAt', 'source', 'sourceRef', 'sourceUrl', 'typeId', 'updatedAt'] as const $columns = ScrapedPartSchema.$columns @column.dateTime({ autoCreate: true }) declare createdAt: DateTime | null @@ -114,7 +114,11 @@ export class ScrapedPartSchema extends BaseModel { @column() declare name: string @column() - declare price: number + declare price: number | null + @column.dateTime() + declare priceCheckedAt: DateTime | null + @column() + declare priceSource: string | null @column.dateTime() declare scrapedAt: DateTime @column() diff --git a/inertia/pages/components/create.vue b/inertia/pages/components/create.vue index a7d45e1..6e5c3df 100644 --- a/inertia/pages/components/create.vue +++ b/inertia/pages/components/create.vue @@ -19,9 +19,11 @@ const form = reactive({ function importPart(part: ScrapedPart, cheapestOffer: MerchantOffer | null) { form.name = part.name // Prefer the live price/link found by comparing merchants (Amazon, LDLC, - // Grosbill, TopAchat) when available, since pc-builder.io's catalog price - // can be stale; fall back to the catalog price/link otherwise. - form.price = cheapestOffer?.price ?? part.price + // Grosbill, TopAchat, ...) when available, since pc-builder.io's catalog + // price can be stale (or simply unknown — see ScrapedPart.price doc); + // fall back to the catalog price/link, or leave the price field as-is + // for the user to fill in manually if neither is known. + form.price = cheapestOffer?.price ?? part.price ?? form.price form.typeId = part.type.id form.purchaseDate = new Date().toISOString().split('T')[0] form.purchaseUrl = cheapestOffer?.url ?? part.sourceUrl diff --git a/inertia/pages/types/UITypes.ts b/inertia/pages/types/UITypes.ts index 771c94d..277108e 100644 --- a/inertia/pages/types/UITypes.ts +++ b/inertia/pages/types/UITypes.ts @@ -54,7 +54,10 @@ export type Component = { export type ScrapedPart = { id: number name: string - price: number + // Null when pc-builder.io has no fixed offer for this part yet (common + // for very recent products). Comparing prices (see NewPartSearch.vue) + // can resolve and persist a real price from another merchant. + price: number | null imageUrl: string | null source: string sourceUrl: string diff --git a/inertia/pages/widgets/NewPartSearch.vue b/inertia/pages/widgets/NewPartSearch.vue index f788899..0951e9d 100644 --- a/inertia/pages/widgets/NewPartSearch.vue +++ b/inertia/pages/widgets/NewPartSearch.vue @@ -126,7 +126,11 @@ function importPart(part: ScrapedPart) { />

{{ part.name }}

-

{{ part.type.name }}

+

+ {{ part.type.name }} + · €{{ part.price.toFixed(2) }} + · No price yet +