From 54c5e3776612775405cc69ae7b9dddba70917ee2 Mon Sep 17 00:00:00 2001 From: Kevin L'Hereec Date: Thu, 23 Jul 2026 16:51:29 +0200 Subject: [PATCH] fix: distinguish GPU model suffixes (Ti/Super/XT) and seed RTX 5060 series RTX 5060 investigation: confirmed via pc-builder.io's own server-side search (search_query=RTX 5060 against the full 6222-product GPU catalog, 0 results) that it simply isn't in pc-builder.io's catalog yet - not a scraping bug, the source data doesn't have it. However, while checking whether the other merchants had it in stock, found a real matching bug: labelMatchesModelToken bug: extractModelToken("...RTX 5060 Ti...") only returned "5060", identical to extractModelToken("...RTX 5060...") without Ti. Every merchant lookup for either card matched on plain "5060" and could silently return the *other* card's price (confirmed live: LDLC and Grosbill returned the exact same product/price for both "RTX 5060" and "RTX 5060 Ti" queries before this fix). Fixed by: - extractModelToken now appends a trailing suffix token (Ti, Super, XT, XTX) when it immediately follows the model number, so "5060 Ti" and "5060" are extracted as distinct tokens - New labelMatchesModelToken() replaces the naive label.includes(modelToken) check everywhere: a bare token ("5060") no longer matches a label that actually has a suffix right after it ("5060 Ti"), while a suffixed token ("5060 Ti") only matches labels carrying that exact suffix - 9 new unit tests covering extraction and matching, including the regression case (RTX 5060 must not match "RTX 5060 Ti" labels) Verified fix live: LDLC/Grosbill now return correctly distinct prices for "RTX 5060" (369.95/364.99) vs "RTX 5060 Ti" (489.95/329.99). New manual_part_seeder_service.ts: since pc-builder.io's catalog lags behind real product releases, this seeds a small manually-tracked list of recent parts it doesn't carry yet (currently RTX 5060 8GB and 5060 Ti 16GB), resolving their price live from the already-supported merchants rather than hardcoding a number that would go stale. `scrape:parts` now runs this after the regular catalog scrape. Skips an entry rather than inserting a guessed price if no merchant has it in stock. Also fixes a real bug hit while testing the seeder: scraped_parts and components.purchase_url were varchar(255), which truncates/rejects Grosbill's long tracking-redirect URLs (500+ characters with encoded analytics params) - both widened to TEXT via a new migration. --- app/services/manual_part_seeder_service.ts | 82 +++++++++++++++++++ app/services/merchant_price_service.ts | 77 +++++++++++++---- commands/scrape_parts.ts | 7 ++ ...e_widen_scraped_parts_source_urls_table.ts | 27 ++++++ .../functional/merchant_price_service.spec.ts | 46 +++++++++++ 5 files changed, 225 insertions(+), 14 deletions(-) create mode 100644 app/services/manual_part_seeder_service.ts create mode 100644 database/migrations/1784817940050_create_widen_scraped_parts_source_urls_table.ts create mode 100644 tests/functional/merchant_price_service.spec.ts diff --git a/app/services/manual_part_seeder_service.ts b/app/services/manual_part_seeder_service.ts new file mode 100644 index 0000000..f4eb802 --- /dev/null +++ b/app/services/manual_part_seeder_service.ts @@ -0,0 +1,82 @@ +import { DateTime } from 'luxon' +import logger from '@adonisjs/core/services/logger' +import ComponentType from '#models/component_type' +import ScrapedPart from '#models/scraped_part' +import { findCheapestOffers } from '#services/merchant_price_service' + +const SOURCE = 'manual' + +/** + * Very recent parts pc-builder.io doesn't carry yet (its catalog appears + * to lag several months behind actual product releases — confirmed via + * its own server-side search returning 0 results for "RTX 5060"). These + * are seeded manually so they're at least searchable/importable, with + * their price resolved live from the merchants merchant_price_service.ts + * already supports (LDLC/Grosbill both carry the RTX 5060 series, for + * example) rather than a hardcoded number that would go stale immediately. + * + * sourceRef must be stable and unique per entry (used as the upsert key, + * like pc-builder.io's own product ID), and is namespaced under + * "manual-" to guarantee it can never collide with a numeric + * pc-builder.io data-id. + */ +const MANUAL_PARTS: { typeName: string; name: string; sourceRef: string }[] = [ + { + typeName: 'Graphic Card', + name: 'NVIDIA GeForce RTX 5060 8 GB Video Card', + sourceRef: 'manual-rtx-5060-8gb', + }, + { + typeName: 'Graphic Card', + name: 'NVIDIA GeForce RTX 5060 Ti 16 GB Video Card', + sourceRef: 'manual-rtx-5060-ti-16gb', + }, +] + +/** + * Upserts every entry in MANUAL_PARTS, resolving each one's price live + * from the supported merchants (Amazon/LDLC/Grosbill/...). An entry with + * no ComponentType match in the current database, or no merchant offer + * found at all, is skipped rather than inserted with a null/guessed + * price — this only fills in real, verified prices. + */ +async function seedManualParts(): Promise<{ inserted: number; skipped: number }> { + let inserted = 0 + let skipped = 0 + + for (const manualPart of MANUAL_PARTS) { + const componentType = await ComponentType.findBy('name', manualPart.typeName) + if (!componentType) { + logger.warn({ part: manualPart.name }, 'No matching ComponentType, skipping manual part') + skipped++ + continue + } + + const offers = await findCheapestOffers(manualPart.name, null) + if (offers.length === 0) { + logger.warn({ part: manualPart.name }, 'No merchant offer found, skipping manual part') + skipped++ + continue + } + + const cheapest = offers[0] + await ScrapedPart.updateOrCreate( + { source: SOURCE, sourceRef: manualPart.sourceRef }, + { + typeId: componentType.id, + name: manualPart.name, + imageUrl: null, + price: cheapest.price, + sourceUrl: cheapest.url, + priceSource: cheapest.merchant, + priceCheckedAt: DateTime.now(), + scrapedAt: DateTime.now(), + } + ) + inserted++ + } + + return { inserted, skipped } +} + +export { seedManualParts, MANUAL_PARTS } diff --git a/app/services/merchant_price_service.ts b/app/services/merchant_price_service.ts index b81d88f..b485184 100644 --- a/app/services/merchant_price_service.ts +++ b/app/services/merchant_price_service.ts @@ -112,23 +112,70 @@ function shortSearchQuery(name: string): string { return tokens.slice(0, 3).join(' ') } +/** + * Suffix tokens that modify a base GPU/CPU model number into a distinct + * SKU with a different price (e.g. "RTX 5060" vs "RTX 5060 Ti", "RTX 4070" + * vs "RTX 4070 Super"). When immediately following the model token, these + * must be treated as part of it — otherwise "5060 Ti" only matches on + * "5060" and can't be told apart from plain "5060". + */ +const MODEL_SUFFIXES = ['Ti', 'Super', 'XT', 'XTX'] + /** * Extracts the most specific "model number" token from a product name, to * disambiguate between multiple search results (e.g. "3200G" out of "AMD - * Ryzen 3 3200G ...", or "970" out of "Samsung 970 Evo Plus ..."). Prefers - * alphanumeric tokens that mix letters and digits (typical of CPU/GPU model - * numbers), falling back to a bare 3-4 digit number. + * Ryzen 3 3200G ...", "970" out of "Samsung 970 Evo Plus ...", or + * "5060 Ti" out of "NVIDIA GeForce RTX 5060 Ti 16 GB ..." — as opposed to + * plain "5060", which is a different, cheaper card). Prefers alphanumeric + * tokens that mix letters and digits (typical of CPU/GPU model numbers), + * falling back to a bare 3-4 digit number. */ function extractModelToken(name: string): string | null { const tokens = name .replace(/\([^)]*\)/g, ' ') .split(/\s+/) .filter(Boolean) - const mixed = tokens.find( + + const matchWithSuffix = (index: number, base: string): string => { + const next = tokens[index + 1] + return next && MODEL_SUFFIXES.includes(next) ? `${base} ${next}` : base + } + + const mixedIndex = tokens.findIndex( (t) => /[a-zA-Z]/.test(t) && /\d/.test(t) && !/^\d+(\.\d+)?ghz$/i.test(t) ) - if (mixed) return mixed - return tokens.find((t) => /^\d{3,4}$/.test(t)) ?? null + if (mixedIndex !== -1) return matchWithSuffix(mixedIndex, tokens[mixedIndex]) + + const bareIndex = tokens.findIndex((t) => /^\d{3,4}$/.test(t)) + if (bareIndex !== -1) return matchWithSuffix(bareIndex, tokens[bareIndex]) + + return null +} + +/** + * True if `label` matches `modelToken` as the specific SKU it represents, + * not just as a loose substring. Plain string.includes() would let "5060" + * match "RTX 5060 Ti" (wrong, more expensive/different card) and vice + * versa "5060 Ti" would still substring-match "5060" only by accident of + * word order — this instead requires the model token to appear as its own + * word boundary, and additionally rejects a bare model number (no suffix) + * when the label actually has one of MODEL_SUFFIXES right after it. + */ +function labelMatchesModelToken(label: string, modelToken: string): boolean { + const normalizedLabel = label.toLowerCase() + const normalizedToken = modelToken.toLowerCase() + + if (!normalizedLabel.includes(normalizedToken)) return false + + const hasSuffix = normalizedToken.includes(' ') + if (hasSuffix) return true + + // Bare token (e.g. "5060"): reject if the label has it immediately + // followed by a known suffix (e.g. "5060 Ti"), since that's a distinct, + // different-priced SKU we should not conflate with the plain model. + const suffixPattern = MODEL_SUFFIXES.join('|') + const followedBySuffix = new RegExp(`\\b${normalizedToken}\\s+(${suffixPattern})\\b`, 'i') + return !followedBySuffix.test(label) } /** @@ -174,7 +221,7 @@ async function fetchLdlcPrice(productName: string): Promise { const label = $(el).find('.pdt-desc a').first().text() - if (label.toLowerCase().includes(modelToken.toLowerCase())) { + if (labelMatchesModelToken(label, modelToken)) { best = $(el) return false } @@ -214,7 +261,7 @@ async function fetchGrosbillPrice(productName: string): Promise | null = null cards.each((_, el) => { const label = $(el).find('.product__label').text() - if (!modelToken || label.toLowerCase().includes(modelToken.toLowerCase())) { + if (!modelToken || labelMatchesModelToken(label, modelToken)) { best = $(el) return false } @@ -307,7 +354,7 @@ async function fetchMaterielNetPrice(productName: string): Promise '') - if (modelToken && !label.toLowerCase().includes(modelToken.toLowerCase())) continue + if (modelToken && !labelMatchesModelToken(label, modelToken)) continue const priceText = await card .locator('.o-product__price') @@ -374,7 +421,7 @@ async function fetchCdiscountPrice(productName: string): Promise '') - if (modelToken && !label.toLowerCase().includes(modelToken.toLowerCase())) continue + if (modelToken && !labelMatchesModelToken(label, modelToken)) continue const priceText = await card .locator('[data-e2e="lplr-price"]') @@ -417,7 +464,7 @@ async function fetchPcComponentesPrice(productName: string): Promise { + table.text('source_url').notNullable().alter() + }) + this.schema.alterTable('components', (table) => { + table.text('purchase_url').nullable().alter() + }) + } + + async down() { + this.schema.alterTable('scraped_parts', (table) => { + table.string('source_url').notNullable().alter() + }) + this.schema.alterTable('components', (table) => { + table.string('purchase_url').nullable().alter() + }) + } +} diff --git a/tests/functional/merchant_price_service.spec.ts b/tests/functional/merchant_price_service.spec.ts new file mode 100644 index 0000000..59c71b4 --- /dev/null +++ b/tests/functional/merchant_price_service.spec.ts @@ -0,0 +1,46 @@ +import { test } from '@japa/runner' +import { extractModelToken, labelMatchesModelToken } from '#services/merchant_price_service' + +test.group('extractModelToken', () => { + test('extracts a bare model number', ({ assert }) => { + assert.equal(extractModelToken('Samsung 970 Evo Plus 1 TB Solid State Drive'), '970') + }) + + test('extracts an alphanumeric model number', ({ assert }) => { + assert.equal(extractModelToken('AMD Ryzen 3 3200G 3.6 GHz Quad-Core Processor'), '3200G') + }) + + test('appends a known suffix immediately following the model number', ({ assert }) => { + assert.equal(extractModelToken('NVIDIA GeForce RTX 5060 Ti 16 GB Video Card'), '5060 Ti') + assert.equal(extractModelToken('NVIDIA GeForce RTX 4070 Super 12 GB Video Card'), '4070 Super') + }) + + test('does not append a suffix when the model number has none', ({ assert }) => { + assert.equal(extractModelToken('NVIDIA GeForce RTX 5060 8 GB Video Card'), '5060') + }) + + test('returns null when no model-like token is found', ({ assert }) => { + assert.isNull(extractModelToken('Some Generic Product Without Any Model Number')) + }) +}) + +test.group('labelMatchesModelToken', () => { + test('a bare model token matches a label with no suffix', ({ assert }) => { + assert.isTrue(labelMatchesModelToken('MSI VENTUS 2X OC NVIDIA GeForce RTX 5060 8G', '5060')) + }) + + test('a bare model token does NOT match a label carrying a known suffix', ({ assert }) => { + // Regression: RTX 5060 vs RTX 5060 Ti must never be conflated - they + // are different, differently-priced cards. + assert.isFalse(labelMatchesModelToken('PNY NVIDIA GeForce RTX 5060 Ti 8G OC', '5060')) + }) + + test('a suffixed model token only matches labels with that exact suffix', ({ assert }) => { + assert.isTrue(labelMatchesModelToken('PNY NVIDIA GeForce RTX 5060 Ti 8G OC', '5060 Ti')) + assert.isFalse(labelMatchesModelToken('MSI VENTUS 2X OC NVIDIA GeForce RTX 5060 8G', '5060 Ti')) + }) + + test('returns false when the token is not present at all', ({ assert }) => { + assert.isFalse(labelMatchesModelToken('AMD Radeon RX 7800 XT', '5060')) + }) +})