From 62ec796e05e77152c30817c23a9e60d68fd27c16 Mon Sep 17 00:00:00 2001 From: Kevin L'Hereec Date: Thu, 23 Jul 2026 14:58:33 +0200 Subject: [PATCH] fix: repair broken CSP and compare live merchant prices for new parts CSP fix: - The previous shield.ts config used '@viteDevUrl'/'@viteHmrUrl' CSP keywords that don't exist in this version of @adonisjs/shield (only '@viteUrl' is registered by @adonisjs/vite, and it resolves to an empty string here since Vite assets are proxied through AdonisJS on the same origin). Both placeholders were sent to the browser literally, breaking Google Fonts, the Vite HMR websocket, and pc-builder.io/favicon images. - Replaced with concrete directives: 'self' covers same-origin Vite assets, ws://localhost:{VITE_HMR_PORT|24678} is added only in dev for the HMR socket, and explicit allowlists for Google Fonts (fonts.googleapis.com/fonts.gstatic.com), pc-builder.io product images, and Google's favicon service (www.google.com) used by the merchant comparison UI Real purchase links + live price comparison: - part_scraper_service.ts: ScrapedPart.sourceUrl now stores the actual Amazon.fr purchase link scraped from pc-builder.io's category pages (https://www.amazon.fr/dp/{ASIN}?tag=... or a search fallback), instead of the pc-builder.io product page, which isn't a place anyone can buy anything from - New merchant_price_service.ts: on-demand live price lookup across Amazon, LDLC, Grosbill and TopAchat. Of 6 requested merchants, Materiel.net (search requires JS/Turbo, no results via plain GET) and Cdiscount/PCComponentes (empty body / 403) were dropped as not scrapable without a headless browser or paid API - Matching strategy: pc-builder.io's full product names (e.g. "AMD Ryzen 3 3200G 3.6 GHz Quad-Core Processor") return zero results on every merchant's search engine, so we search on a short brand+series query and then pick the specific result whose label contains the extracted model token (e.g. "3200G"). No token match means no offer returned, rather than risk quoting the price of a different, unrelated product - Fixed price parsing for LDLC/Grosbill, which render prices as nested price/sup elements for whole euros vs cents (and LDLC additionally wraps that in a duplicate outer .price div) - naive text concatenation was misreading "64,95" as "6495" - New GET /scraped-parts/compare endpoint (authenticated), called on demand per search result rather than for every row, since it makes several outbound requests and can take a few seconds - NewPartSearch.vue: "Compare prices" button per result shows the cheapest merchant with its favicon and links to the actual offer; importing a part now prefers the live cheapest price/link over pc-builder.io's potentially stale catalog price (confirmed stale in practice: pc-builder.io listed a Ryzen 3 3200G at 128 euros while Amazon sells it for 66.89 euros) - 3 new functional tests for the compare endpoint (auth required, 404 on unknown part, 422 on invalid id) --- .adonisjs/client/registry/index.ts | 6 + .adonisjs/client/registry/schema.d.ts | 12 + .adonisjs/client/registry/tree.d.ts | 1 + .adonisjs/server/routes.d.ts | 3 + app/controllers/scraped_parts_controller.ts | 29 ++ app/services/merchant_price_service.ts | 291 ++++++++++++++++++++ app/services/part_scraper_service.ts | 24 +- config/shield.ts | 26 +- inertia/pages/components/create.vue | 11 +- inertia/pages/types/UITypes.ts | 6 + inertia/pages/widgets/NewPartSearch.vue | 127 +++++++-- start/routes.ts | 5 + tests/functional/scraped_parts.spec.ts | 27 ++ 13 files changed, 532 insertions(+), 36 deletions(-) create mode 100644 app/services/merchant_price_service.ts diff --git a/.adonisjs/client/registry/index.ts b/.adonisjs/client/registry/index.ts index b46a549..f582d44 100644 --- a/.adonisjs/client/registry/index.ts +++ b/.adonisjs/client/registry/index.ts @@ -150,6 +150,12 @@ const routes = { tokens: [{"old":"/scraped-parts","type":0,"val":"scraped-parts","end":""}], types: placeholder as Registry['scraped_parts.index']['types'], }, + 'scraped_parts.compare': { + methods: ["GET","HEAD"], + pattern: '/scraped-parts/compare', + tokens: [{"old":"/scraped-parts/compare","type":0,"val":"scraped-parts","end":""},{"old":"/scraped-parts/compare","type":0,"val":"compare","end":""}], + types: placeholder as Registry['scraped_parts.compare']['types'], + }, } as const satisfies Record export { routes } diff --git a/.adonisjs/client/registry/schema.d.ts b/.adonisjs/client/registry/schema.d.ts index f08684d..a97e206 100644 --- a/.adonisjs/client/registry/schema.d.ts +++ b/.adonisjs/client/registry/schema.d.ts @@ -295,4 +295,16 @@ export interface Registry { errorResponse: ExtractErrorResponse>> } } + 'scraped_parts.compare': { + methods: ["GET","HEAD"] + pattern: '/scraped-parts/compare' + types: { + body: {} + paramsTuple: [] + params: {} + query: {} + response: ExtractResponse>> + errorResponse: ExtractErrorResponse>> + } + } } diff --git a/.adonisjs/client/registry/tree.d.ts b/.adonisjs/client/registry/tree.d.ts index d071ee3..ce87143 100644 --- a/.adonisjs/client/registry/tree.d.ts +++ b/.adonisjs/client/registry/tree.d.ts @@ -39,5 +39,6 @@ export interface ApiDefinition { } scrapedParts: { index: typeof routes['scraped_parts.index'] + compare: typeof routes['scraped_parts.compare'] } } diff --git a/.adonisjs/server/routes.d.ts b/.adonisjs/server/routes.d.ts index 8f4fb9d..e014c48 100644 --- a/.adonisjs/server/routes.d.ts +++ b/.adonisjs/server/routes.d.ts @@ -28,6 +28,7 @@ export type ScannedRoutes = { 'components.update': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'components.destroy': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'scraped_parts.index': { paramsTuple?: []; params?: {} } + 'scraped_parts.compare': { paramsTuple?: []; params?: {} } } GET: { 'home': { paramsTuple?: []; params?: {} } @@ -43,6 +44,7 @@ export type ScannedRoutes = { 'components.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'components.edit': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'scraped_parts.index': { paramsTuple?: []; params?: {} } + 'scraped_parts.compare': { paramsTuple?: []; params?: {} } } HEAD: { 'home': { paramsTuple?: []; params?: {} } @@ -58,6 +60,7 @@ export type ScannedRoutes = { 'components.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'components.edit': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } 'scraped_parts.index': { paramsTuple?: []; params?: {} } + 'scraped_parts.compare': { paramsTuple?: []; params?: {} } } POST: { 'new_account.store': { paramsTuple?: []; params?: {} } diff --git a/app/controllers/scraped_parts_controller.ts b/app/controllers/scraped_parts_controller.ts index e5f1b17..b4bca15 100644 --- a/app/controllers/scraped_parts_controller.ts +++ b/app/controllers/scraped_parts_controller.ts @@ -3,6 +3,7 @@ import vine from '@vinejs/vine' import ScrapedPart from '#models/scraped_part' import ComponentType from '#models/component_type' import { ensureFreshCategory } from '#services/part_scraper_service' +import { findCheapestOffers } from '#services/merchant_price_service' const searchValidator = vine.compile( vine.object({ @@ -11,6 +12,12 @@ const searchValidator = vine.compile( }) ) +const compareValidator = vine.compile( + vine.object({ + id: vine.number().min(1), + }) +) + export default class ScrapedPartsController { /** * Search the new-parts cache (scraped from pc-builder.io) by name and/or @@ -48,4 +55,26 @@ export default class ScrapedPartsController { const parts = await query.limit(30) return response.json(parts.map((part) => part.serialize())) } + + /** + * Compares live prices for a given scraped part across Amazon, LDLC, + * Grosbill and TopAchat, and returns them sorted cheapest first. This is + * triggered on demand (one part at a time) rather than for every search + * result, since it makes several outbound HTTP requests and can take a + * few seconds. + */ + async compare({ request, auth, response }: HttpContext) { + await auth.authenticate() + const { id } = await request.validateUsing(compareValidator) + + const part = await ScrapedPart.find(id) + if (!part) { + return response.notFound({ offers: [] }) + } + + const amazonUrl = part.sourceUrl.includes('amazon.') ? part.sourceUrl : null + const offers = await findCheapestOffers(part.name, amazonUrl) + + return response.json({ offers }) + } } diff --git a/app/services/merchant_price_service.ts b/app/services/merchant_price_service.ts new file mode 100644 index 0000000..fe20261 --- /dev/null +++ b/app/services/merchant_price_service.ts @@ -0,0 +1,291 @@ +import * as cheerio from 'cheerio' +import logger from '@adonisjs/core/services/logger' + +const USER_AGENT = + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + +export interface MerchantOffer { + merchant: 'Amazon' | 'LDLC' | 'Grosbill' | 'TopAchat' + price: number + url: string +} + +async function fetchHtml(url: string): Promise { + try { + const response = await fetch(url, { + headers: { + 'User-Agent': USER_AGENT, + 'Accept-Language': 'fr-FR,fr;q=0.9', + }, + }) + if (!response.ok) return null + return await response.text() + } catch (error) { + logger.warn({ url, err: error }, 'Failed to fetch merchant page') + return null + } +} + +/** + * Parses a price from plain text like "64,95 €" or "413.59 €*". + */ +function parsePriceText(text: string): number | null { + const normalized = text + .replace(/\u00a0/g, ' ') + .replace(/[^\d,.\s]/g, '') + .trim() + .replace(/\s/g, '') + .replace(',', '.') + const value = Number.parseFloat(normalized) + return Number.isNaN(value) ? null : value +} + +/** + * Some merchants (LDLC, Grosbill) render prices split across sibling/child + * elements instead of plain text, e.g. `
64€95
` + * renders as "64€95" once flattened to text, which parsePriceText would + * misread as 6495. This reads the integer part and the `` cents part + * separately and recombines them properly. + * + * LDLC additionally nests a duplicate `.price` wrapper + * (`
64€95
`), + * so this always resolves to the innermost element actually containing a + * ``, rather than assuming $el itself is the leaf node. + */ +function parseFragmentedPrice($el: cheerio.Cheerio): number | null { + const sup = $el.find('sup').first() + if (sup.length > 0) { + // The element that directly holds both the whole-part text node and + // the cents — not necessarily $el itself if it's a wrapper. + const leaf = sup.parent() + const cents = sup.text().replace(/\D/g, '') + const wholePart = leaf.clone().children().remove().end().text().replace(/\D/g, '') + if (wholePart) { + const value = Number.parseFloat(`${wholePart}.${cents.padEnd(2, '0').slice(0, 2)}`) + return Number.isNaN(value) ? null : value + } + } + return parsePriceText($el.text()) +} + +/** + * Builds a short, merchant-search-friendly query from a full pc-builder.io + * product name. These sites' search engines are strict/exact-match and + * choke on the long, spec-heavy names pc-builder.io uses (e.g. "AMD Ryzen 3 + * 3200G 3.6 GHz Quad-Core Processor" returns zero results everywhere, but + * "AMD Ryzen 3" does). We search on the first few tokens (brand + series), + * then use extractModelToken() to pick out the right result from the list. + */ +function shortSearchQuery(name: string): string { + const tokens = name + .replace(/\([^)]*\)/g, ' ') + .split(/\s+/) + .filter(Boolean) + return tokens.slice(0, 3).join(' ') +} + +/** + * Extracts the most specific "model number" token from a product name, to + * disambiguate between multiple search results (e.g. "3200G" out of "AMD + * Ryzen 3 3200G ...", or "970" out of "Samsung 970 Evo Plus ..."). Prefers + * alphanumeric tokens that mix letters and digits (typical of CPU/GPU model + * numbers), falling back to a bare 3-4 digit number. + */ +function extractModelToken(name: string): string | null { + const tokens = name + .replace(/\([^)]*\)/g, ' ') + .split(/\s+/) + .filter(Boolean) + const mixed = tokens.find( + (t) => /[a-zA-Z]/.test(t) && /\d/.test(t) && !/^\d+(\.\d+)?ghz$/i.test(t) + ) + if (mixed) return mixed + return tokens.find((t) => /^\d{3,4}$/.test(t)) ?? null +} + +/** + * Reads the price directly from a known Amazon.fr product page + * (https://www.amazon.fr/dp/{ASIN}?...). Only works for direct product + * links; Amazon search-result links (https://www.amazon.fr/s?k=...) can't + * be resolved to a single price this way. + */ +async function fetchAmazonPrice(amazonUrl: string): Promise { + if (!amazonUrl.includes('/dp/')) return null + + const html = await fetchHtml(amazonUrl) + if (!html) return null + + const $ = cheerio.load(html) + const priceText = $('.a-price .a-offscreen').first().text().trim() + const price = parsePriceText(priceText) + if (price === null) return null + + return { merchant: 'Amazon', price, url: amazonUrl } +} + +/** + * Searches LDLC by product name and reads the price of the result whose + * label contains the extracted model token (e.g. "3200G"). Only falls back + * to the first result when there is no model token to disambiguate with + * (rare); otherwise a search that returns unrelated products (LDLC simply + * doesn't carry this exact model) correctly yields no offer instead of a + * misleading price for the wrong product. + */ +async function fetchLdlcPrice(productName: string): Promise { + const query = shortSearchQuery(productName) + const modelToken = extractModelToken(productName) + const url = `https://www.ldlc.com/recherche/${encodeURIComponent(query)}/` + const html = await fetchHtml(url) + if (!html) return null + + const $ = cheerio.load(html) + const items = $('.listing-product li.pdt-item') + if (!items.length) return null + + let best: cheerio.Cheerio | null = modelToken ? null : items.first() + if (modelToken) { + items.each((_, el) => { + const label = $(el).find('.pdt-desc a').first().text() + if (label.toLowerCase().includes(modelToken.toLowerCase())) { + best = $(el) + return false + } + }) + } + if (!best) return null + + const link = (best as cheerio.Cheerio).find('.pdt-desc a').first().attr('href') + const priceEl = (best as cheerio.Cheerio).find('.price').first() + const price = parseFragmentedPrice(priceEl) + if (price === null || !link) return null + + return { merchant: 'LDLC', price, url: new URL(link, 'https://www.ldlc.com').toString() } +} + +/** + * Searches Grosbill by product name and reads the price of the result + * whose label contains the extracted model token. Same rationale as + * fetchLdlcPrice: without a model token match, we return null rather than + * risk quoting the price of an unrelated product. + */ +async function fetchGrosbillPrice(productName: string): Promise { + const query = shortSearchQuery(productName) + const modelToken = extractModelToken(productName) + const url = `https://www.grosbill.com/produit.aspx?q=${encodeURIComponent(query)}/` + const html = await fetchHtml(url) + if (!html) return null + + const $ = cheerio.load(html) + const items = $('[class*="la_liste_des_produits"]') + if (!items.length) return null + + let best: cheerio.Cheerio | null = modelToken ? null : items.first() + if (modelToken) { + items.each((_, el) => { + const label = $(el) + .find('.grb__liste-produit__liste__produit__information__libelle') + .text() + .replace(/\s+/g, ' ') + if (label.toLowerCase().includes(modelToken.toLowerCase())) { + best = $(el) + return false + } + }) + } + if (!best) return null + + const priceEl = (best as cheerio.Cheerio).find('.price-main').first() + const price = parseFragmentedPrice(priceEl) + if (price === null) return null + + const link = (best as cheerio.Cheerio).find('a[href]').first().attr('href') + return { merchant: 'Grosbill', price, url: link ? new URL(link, url).toString() : url } +} + +/** + * Searches TopAchat by product name. When the search resolves to a single + * matching product, TopAchat redirects straight to the product page + * (label-product / offer-price__price). List-page matching by model token + * is attempted first for queries with multiple hits. + */ +async function fetchTopAchatPrice(productName: string): Promise { + const query = shortSearchQuery(productName) + const modelToken = extractModelToken(productName) + const url = `https://www.topachat.com/search/${encodeURIComponent(query)}` + const html = await fetchHtml(url) + if (!html) return null + + const $ = cheerio.load(html) + + // Case 1: redirected straight to a single product page. + const singlePrice = $('.offer-price__price').first() + if (singlePrice.length) { + const label = $('.label-product').first().text() + if (!modelToken || label.toLowerCase().includes(modelToken.toLowerCase())) { + const priceText = singlePrice.clone().children().remove().end().text().trim() + const price = parsePriceText(priceText) + if (price !== null) { + const canonical = $('link[rel="canonical"]').attr('href') + return { merchant: 'TopAchat', price, url: canonical || url } + } + } + } + + // Case 2: a list of results (product__price / product__label cards). + const cards = $('article.product, [class*="product__"]').filter( + (_, el) => $(el).find('.product__price').length > 0 + ) + let best: cheerio.Cheerio | null = null + cards.each((_, el) => { + const label = $(el).find('.product__label').text() + if (!modelToken || label.toLowerCase().includes(modelToken.toLowerCase())) { + best = $(el) + return false + } + }) + if (!best) return null + + const priceText = (best as cheerio.Cheerio).find('.product__price').first().text().trim() + const price = parsePriceText(priceText) + if (price === null) return null + + const link = (best as cheerio.Cheerio).closest('a[href]').attr('href') + return { merchant: 'TopAchat', price, url: link ? new URL(link, url).toString() : url } +} + +/** + * Fetches offers from every supported merchant in parallel and returns + * them sorted by ascending price. Merchants that fail (blocked, no match, + * network error) are silently skipped rather than failing the whole + * comparison. + */ +async function findCheapestOffers( + productName: string, + amazonUrl: string | null +): Promise { + const lookups: Promise[] = [ + amazonUrl ? fetchAmazonPrice(amazonUrl) : Promise.resolve(null), + fetchLdlcPrice(productName), + fetchGrosbillPrice(productName), + fetchTopAchatPrice(productName), + ] + + const results = await Promise.allSettled(lookups) + const offers = results + .filter( + (result): result is PromiseFulfilledResult => + result.status === 'fulfilled' + ) + .map((result) => result.value) + .filter((offer): offer is MerchantOffer => offer !== null) + + return offers.sort((a, b) => a.price - b.price) +} + +export { + findCheapestOffers, + fetchAmazonPrice, + fetchLdlcPrice, + fetchGrosbillPrice, + fetchTopAchatPrice, +} diff --git a/app/services/part_scraper_service.ts b/app/services/part_scraper_service.ts index 9f1ace0..d4765a9 100644 --- a/app/services/part_scraper_service.ts +++ b/app/services/part_scraper_service.ts @@ -40,7 +40,15 @@ interface ScrapedRow { name: string price: number imageUrl: string | null - sourceUrl: string + /** + * The actual merchant page where this part can be bought (currently + * always an Amazon.fr link, since that's the only "buy" link pc-builder.io + * exposes on its category pages). This is what gets stored as + * ScrapedPart.sourceUrl and later copied into Component.purchaseUrl — it + * intentionally is NOT the pc-builder.io product page, which isn't a + * place a user can actually purchase anything from. + */ + purchaseUrl: string sourceRef: string } @@ -70,14 +78,20 @@ async function fetchCategoryPage(slug: string): Promise { const row = $(el) const nameLink = row.find('td.product a').first() const name = nameLink.text().trim() - const sourceUrl = nameLink.attr('href') + const productPageUrl = nameLink.attr('href') const img = row.find('td.product img.product-image') const imageUrl = img.attr('data-src') || img.attr('src') || null const priceText = row.find('td.add-btn .price-box p').first().text().trim() const addButton = row.find('a.add-to-build') const sourceRef = addButton.attr('data-id') + // Real merchant link ("View on Amazon"), either a direct product page + // (https://www.amazon.fr/dp/{ASIN}?tag=...) when pc-builder.io knows the + // ASIN, or a generic search results page (https://www.amazon.fr/s?k=...) + // otherwise. Either way it's a genuine purchase link, unlike the + // pc-builder.io product page. + const amazonUrl = row.find('a.btn-default').attr('href') - if (!name || !sourceUrl || !sourceRef || !priceText) { + if (!name || !productPageUrl || !sourceRef || !priceText) { // Skip rows without a resolved price (e.g. "search on Amazon" links // with no fixed offer) or missing identifiers. return @@ -88,7 +102,7 @@ async function fetchCategoryPage(slug: string): Promise { return } - rows.push({ name, price, imageUrl, sourceUrl, sourceRef }) + rows.push({ name, price, imageUrl, purchaseUrl: amazonUrl || productPageUrl, sourceRef }) }) return rows @@ -116,7 +130,7 @@ async function scrapeComponentType(componentType: ComponentType): Promise }>() const form = reactive({ @@ -16,12 +16,15 @@ const form = reactive({ quantity: 1, }) -function importPart(part: ScrapedPart) { +function importPart(part: ScrapedPart, cheapestOffer: MerchantOffer | null) { form.name = part.name - form.price = part.price + // Prefer the live price/link found by comparing merchants (Amazon, LDLC, + // Grosbill, TopAchat) when available, since pc-builder.io's catalog price + // can be stale; fall back to the catalog price/link otherwise. + form.price = cheapestOffer?.price ?? part.price form.typeId = part.type.id form.purchaseDate = new Date().toISOString().split('T')[0] - form.purchaseUrl = part.sourceUrl + form.purchaseUrl = cheapestOffer?.url ?? part.sourceUrl } function submitForm() { diff --git a/inertia/pages/types/UITypes.ts b/inertia/pages/types/UITypes.ts index 9931553..c0fd205 100644 --- a/inertia/pages/types/UITypes.ts +++ b/inertia/pages/types/UITypes.ts @@ -61,6 +61,12 @@ export type ScrapedPart = { type: ComponentType } +export type MerchantOffer = { + merchant: 'Amazon' | 'LDLC' | 'Grosbill' | 'TopAchat' + price: number + url: string +} + export interface ServerImage { id: number link: string diff --git a/inertia/pages/widgets/NewPartSearch.vue b/inertia/pages/widgets/NewPartSearch.vue index 4a6173d..7872dd9 100644 --- a/inertia/pages/widgets/NewPartSearch.vue +++ b/inertia/pages/widgets/NewPartSearch.vue @@ -1,17 +1,26 @@