fix: repair broken CSP and compare live merchant prices for new parts
Build & Deploy / build (push) Failing after 4s
Build & Deploy / docker (push) Skipped

CSP fix:
- The previous shield.ts config used '@viteDevUrl'/'@viteHmrUrl' CSP
  keywords that don't exist in this version of @adonisjs/shield (only
  '@viteUrl' is registered by @adonisjs/vite, and it resolves to an empty
  string here since Vite assets are proxied through AdonisJS on the same
  origin). Both placeholders were sent to the browser literally, breaking
  Google Fonts, the Vite HMR websocket, and pc-builder.io/favicon images.
- Replaced with concrete directives: 'self' covers same-origin Vite
  assets, ws://localhost:{VITE_HMR_PORT|24678} is added only in dev for
  the HMR socket, and explicit allowlists for Google Fonts
  (fonts.googleapis.com/fonts.gstatic.com), pc-builder.io product images,
  and Google's favicon service (www.google.com) used by the merchant
  comparison UI

Real purchase links + live price comparison:
- part_scraper_service.ts: ScrapedPart.sourceUrl now stores the actual
  Amazon.fr purchase link scraped from pc-builder.io's category pages
  (https://www.amazon.fr/dp/{ASIN}?tag=... or a search fallback), instead
  of the pc-builder.io product page, which isn't a place anyone can buy
  anything from
- New merchant_price_service.ts: on-demand live price lookup across
  Amazon, LDLC, Grosbill and TopAchat. Of 6 requested merchants,
  Materiel.net (search requires JS/Turbo, no results via plain GET) and
  Cdiscount/PCComponentes (empty body / 403) were dropped as not scrapable
  without a headless browser or paid API
- Matching strategy: pc-builder.io's full product names (e.g. "AMD Ryzen 3
  3200G 3.6 GHz Quad-Core Processor") return zero results on every
  merchant's search engine, so we search on a short brand+series query
  and then pick the specific result whose label contains the extracted
  model token (e.g. "3200G"). No token match means no offer returned,
  rather than risk quoting the price of a different, unrelated product
- Fixed price parsing for LDLC/Grosbill, which render prices as nested
  price/sup elements for whole euros vs cents (and LDLC additionally
  wraps that in a duplicate outer .price div) - naive text concatenation
  was misreading "64,95" as "6495"
- New GET /scraped-parts/compare endpoint (authenticated), called on
  demand per search result rather than for every row, since it makes
  several outbound requests and can take a few seconds
- NewPartSearch.vue: "Compare prices" button per result shows the
  cheapest merchant with its favicon and links to the actual offer;
  importing a part now prefers the live cheapest price/link over
  pc-builder.io's potentially stale catalog price (confirmed stale in
  practice: pc-builder.io listed a Ryzen 3 3200G at 128 euros while
  Amazon sells it for 66.89 euros)
- 3 new functional tests for the compare endpoint (auth required, 404 on
  unknown part, 422 on invalid id)
This commit is contained in:
Kevin L'Hereec committed 2026-07-23 14:58:33 +02:00
1 parent 4174d92711
commit 62ec796e05
13 files changed
+532 -36

No files matched your search

@@ -3,6 +3,7 @@ import vine from '@vinejs/vine'
import ScrapedPart from '#models/scraped_part'
import ComponentType from '#models/component_type'
import { ensureFreshCategory } from '#services/part_scraper_service'
import { findCheapestOffers } from '#services/merchant_price_service'
const searchValidator = vine.compile(
vine.object({
@@ -11,6 +12,12 @@ const searchValidator = vine.compile(
})
)
const compareValidator = vine.compile(
vine.object({
id: vine.number().min(1),
})
)
export default class ScrapedPartsController {
/**
* Search the new-parts cache (scraped from pc-builder.io) by name and/or
@@ -48,4 +55,26 @@ export default class ScrapedPartsController {
const parts = await query.limit(30)
return response.json(parts.map((part) => part.serialize()))
}
/**
* Compares live prices for a given scraped part across Amazon, LDLC,
* Grosbill and TopAchat, and returns them sorted cheapest first. This is
* triggered on demand (one part at a time) rather than for every search
* result, since it makes several outbound HTTP requests and can take a
* few seconds.
*/
async compare({ request, auth, response }: HttpContext) {
await auth.authenticate()
const { id } = await request.validateUsing(compareValidator)
const part = await ScrapedPart.find(id)
if (!part) {
return response.notFound({ offers: [] })
}
const amazonUrl = part.sourceUrl.includes('amazon.') ? part.sourceUrl : null
const offers = await findCheapestOffers(part.name, amazonUrl)
return response.json({ offers })
}
}
+291
View File
@@ -0,0 +1,291 @@
import * as cheerio from 'cheerio'
import logger from '@adonisjs/core/services/logger'
const USER_AGENT =
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
export interface MerchantOffer {
merchant: 'Amazon' | 'LDLC' | 'Grosbill' | 'TopAchat'
price: number
url: string
}
async function fetchHtml(url: string): Promise<string | null> {
try {
const response = await fetch(url, {
headers: {
'User-Agent': USER_AGENT,
'Accept-Language': 'fr-FR,fr;q=0.9',
},
})
if (!response.ok) return null
return await response.text()
} catch (error) {
logger.warn({ url, err: error }, 'Failed to fetch merchant page')
return null
}
}
/**
* Parses a price from plain text like "64,95 €" or "413.59 €*".
*/
function parsePriceText(text: string): number | null {
const normalized = text
.replace(/\u00a0/g, ' ')
.replace(/[^\d,.\s]/g, '')
.trim()
.replace(/\s/g, '')
.replace(',', '.')
const value = Number.parseFloat(normalized)
return Number.isNaN(value) ? null : value
}
/**
* Some merchants (LDLC, Grosbill) render prices split across sibling/child
* elements instead of plain text, e.g. `<div class="price">64€<sup>95</sup></div>`
* renders as "64€95" once flattened to text, which parsePriceText would
* misread as 6495. This reads the integer part and the `<sup>` cents part
* separately and recombines them properly.
*
* LDLC additionally nests a duplicate `.price` wrapper
* (`<div class="price"><div class="price">64€<sup>95</sup></div></div>`),
* so this always resolves to the innermost element actually containing a
* `<sup>`, rather than assuming $el itself is the leaf node.
*/
function parseFragmentedPrice($el: cheerio.Cheerio<any>): number | null {
const sup = $el.find('sup').first()
if (sup.length > 0) {
// The element that directly holds both the whole-part text node and
// the <sup> cents — not necessarily $el itself if it's a wrapper.
const leaf = sup.parent()
const cents = sup.text().replace(/\D/g, '')
const wholePart = leaf.clone().children().remove().end().text().replace(/\D/g, '')
if (wholePart) {
const value = Number.parseFloat(`${wholePart}.${cents.padEnd(2, '0').slice(0, 2)}`)
return Number.isNaN(value) ? null : value
}
}
return parsePriceText($el.text())
}
/**
* Builds a short, merchant-search-friendly query from a full pc-builder.io
* product name. These sites' search engines are strict/exact-match and
* choke on the long, spec-heavy names pc-builder.io uses (e.g. "AMD Ryzen 3
* 3200G 3.6 GHz Quad-Core Processor" returns zero results everywhere, but
* "AMD Ryzen 3" does). We search on the first few tokens (brand + series),
* then use extractModelToken() to pick out the right result from the list.
*/
function shortSearchQuery(name: string): string {
const tokens = name
.replace(/\([^)]*\)/g, ' ')
.split(/\s+/)
.filter(Boolean)
return tokens.slice(0, 3).join(' ')
}
/**
* Extracts the most specific "model number" token from a product name, to
* disambiguate between multiple search results (e.g. "3200G" out of "AMD
* Ryzen 3 3200G ...", or "970" out of "Samsung 970 Evo Plus ..."). Prefers
* alphanumeric tokens that mix letters and digits (typical of CPU/GPU model
* numbers), falling back to a bare 3-4 digit number.
*/
function extractModelToken(name: string): string | null {
const tokens = name
.replace(/\([^)]*\)/g, ' ')
.split(/\s+/)
.filter(Boolean)
const mixed = tokens.find(
(t) => /[a-zA-Z]/.test(t) && /\d/.test(t) && !/^\d+(\.\d+)?ghz$/i.test(t)
)
if (mixed) return mixed
return tokens.find((t) => /^\d{3,4}$/.test(t)) ?? null
}
/**
* Reads the price directly from a known Amazon.fr product page
* (https://www.amazon.fr/dp/{ASIN}?...). Only works for direct product
* links; Amazon search-result links (https://www.amazon.fr/s?k=...) can't
* be resolved to a single price this way.
*/
async function fetchAmazonPrice(amazonUrl: string): Promise<MerchantOffer | null> {
if (!amazonUrl.includes('/dp/')) return null
const html = await fetchHtml(amazonUrl)
if (!html) return null
const $ = cheerio.load(html)
const priceText = $('.a-price .a-offscreen').first().text().trim()
const price = parsePriceText(priceText)
if (price === null) return null
return { merchant: 'Amazon', price, url: amazonUrl }
}
/**
* Searches LDLC by product name and reads the price of the result whose
* label contains the extracted model token (e.g. "3200G"). Only falls back
* to the first result when there is no model token to disambiguate with
* (rare); otherwise a search that returns unrelated products (LDLC simply
* doesn't carry this exact model) correctly yields no offer instead of a
* misleading price for the wrong product.
*/
async function fetchLdlcPrice(productName: string): Promise<MerchantOffer | null> {
const query = shortSearchQuery(productName)
const modelToken = extractModelToken(productName)
const url = `https://www.ldlc.com/recherche/${encodeURIComponent(query)}/`
const html = await fetchHtml(url)
if (!html) return null
const $ = cheerio.load(html)
const items = $('.listing-product li.pdt-item')
if (!items.length) return null
let best: cheerio.Cheerio<any> | null = modelToken ? null : items.first()
if (modelToken) {
items.each((_, el) => {
const label = $(el).find('.pdt-desc a').first().text()
if (label.toLowerCase().includes(modelToken.toLowerCase())) {
best = $(el)
return false
}
})
}
if (!best) return null
const link = (best as cheerio.Cheerio<any>).find('.pdt-desc a').first().attr('href')
const priceEl = (best as cheerio.Cheerio<any>).find('.price').first()
const price = parseFragmentedPrice(priceEl)
if (price === null || !link) return null
return { merchant: 'LDLC', price, url: new URL(link, 'https://www.ldlc.com').toString() }
}
/**
* Searches Grosbill by product name and reads the price of the result
* whose label contains the extracted model token. Same rationale as
* fetchLdlcPrice: without a model token match, we return null rather than
* risk quoting the price of an unrelated product.
*/
async function fetchGrosbillPrice(productName: string): Promise<MerchantOffer | null> {
const query = shortSearchQuery(productName)
const modelToken = extractModelToken(productName)
const url = `https://www.grosbill.com/produit.aspx?q=${encodeURIComponent(query)}/`
const html = await fetchHtml(url)
if (!html) return null
const $ = cheerio.load(html)
const items = $('[class*="la_liste_des_produits"]')
if (!items.length) return null
let best: cheerio.Cheerio<any> | null = modelToken ? null : items.first()
if (modelToken) {
items.each((_, el) => {
const label = $(el)
.find('.grb__liste-produit__liste__produit__information__libelle')
.text()
.replace(/\s+/g, ' ')
if (label.toLowerCase().includes(modelToken.toLowerCase())) {
best = $(el)
return false
}
})
}
if (!best) return null
const priceEl = (best as cheerio.Cheerio<any>).find('.price-main').first()
const price = parseFragmentedPrice(priceEl)
if (price === null) return null
const link = (best as cheerio.Cheerio<any>).find('a[href]').first().attr('href')
return { merchant: 'Grosbill', price, url: link ? new URL(link, url).toString() : url }
}
/**
* Searches TopAchat by product name. When the search resolves to a single
* matching product, TopAchat redirects straight to the product page
* (label-product / offer-price__price). List-page matching by model token
* is attempted first for queries with multiple hits.
*/
async function fetchTopAchatPrice(productName: string): Promise<MerchantOffer | null> {
const query = shortSearchQuery(productName)
const modelToken = extractModelToken(productName)
const url = `https://www.topachat.com/search/${encodeURIComponent(query)}`
const html = await fetchHtml(url)
if (!html) return null
const $ = cheerio.load(html)
// Case 1: redirected straight to a single product page.
const singlePrice = $('.offer-price__price').first()
if (singlePrice.length) {
const label = $('.label-product').first().text()
if (!modelToken || label.toLowerCase().includes(modelToken.toLowerCase())) {
const priceText = singlePrice.clone().children().remove().end().text().trim()
const price = parsePriceText(priceText)
if (price !== null) {
const canonical = $('link[rel="canonical"]').attr('href')
return { merchant: 'TopAchat', price, url: canonical || url }
}
}
}
// Case 2: a list of results (product__price / product__label cards).
const cards = $('article.product, [class*="product__"]').filter(
(_, el) => $(el).find('.product__price').length > 0
)
let best: cheerio.Cheerio<any> | null = null
cards.each((_, el) => {
const label = $(el).find('.product__label').text()
if (!modelToken || label.toLowerCase().includes(modelToken.toLowerCase())) {
best = $(el)
return false
}
})
if (!best) return null
const priceText = (best as cheerio.Cheerio<any>).find('.product__price').first().text().trim()
const price = parsePriceText(priceText)
if (price === null) return null
const link = (best as cheerio.Cheerio<any>).closest('a[href]').attr('href')
return { merchant: 'TopAchat', price, url: link ? new URL(link, url).toString() : url }
}
/**
* Fetches offers from every supported merchant in parallel and returns
* them sorted by ascending price. Merchants that fail (blocked, no match,
* network error) are silently skipped rather than failing the whole
* comparison.
*/
async function findCheapestOffers(
productName: string,
amazonUrl: string | null
): Promise<MerchantOffer[]> {
const lookups: Promise<MerchantOffer | null>[] = [
amazonUrl ? fetchAmazonPrice(amazonUrl) : Promise.resolve(null),
fetchLdlcPrice(productName),
fetchGrosbillPrice(productName),
fetchTopAchatPrice(productName),
]
const results = await Promise.allSettled(lookups)
const offers = results
.filter(
(result): result is PromiseFulfilledResult<MerchantOffer | null> =>
result.status === 'fulfilled'
)
.map((result) => result.value)
.filter((offer): offer is MerchantOffer => offer !== null)
return offers.sort((a, b) => a.price - b.price)
}
export {
findCheapestOffers,
fetchAmazonPrice,
fetchLdlcPrice,
fetchGrosbillPrice,
fetchTopAchatPrice,
}
+19 -5
View File
@@ -40,7 +40,15 @@ interface ScrapedRow {
name: string
price: number
imageUrl: string | null
sourceUrl: string
/**
* The actual merchant page where this part can be bought (currently
* always an Amazon.fr link, since that's the only "buy" link pc-builder.io
* exposes on its category pages). This is what gets stored as
* ScrapedPart.sourceUrl and later copied into Component.purchaseUrl — it
* intentionally is NOT the pc-builder.io product page, which isn't a
* place a user can actually purchase anything from.
*/
purchaseUrl: string
sourceRef: string
}
@@ -70,14 +78,20 @@ async function fetchCategoryPage(slug: string): Promise<ScrapedRow[]> {
const row = $(el)
const nameLink = row.find('td.product a').first()
const name = nameLink.text().trim()
const sourceUrl = nameLink.attr('href')
const productPageUrl = nameLink.attr('href')
const img = row.find('td.product img.product-image')
const imageUrl = img.attr('data-src') || img.attr('src') || null
const priceText = row.find('td.add-btn .price-box p').first().text().trim()
const addButton = row.find('a.add-to-build')
const sourceRef = addButton.attr('data-id')
// Real merchant link ("View on Amazon"), either a direct product page
// (https://www.amazon.fr/dp/{ASIN}?tag=...) when pc-builder.io knows the
// ASIN, or a generic search results page (https://www.amazon.fr/s?k=...)
// otherwise. Either way it's a genuine purchase link, unlike the
// pc-builder.io product page.
const amazonUrl = row.find('a.btn-default').attr('href')
if (!name || !sourceUrl || !sourceRef || !priceText) {
if (!name || !productPageUrl || !sourceRef || !priceText) {
// Skip rows without a resolved price (e.g. "search on Amazon" links
// with no fixed offer) or missing identifiers.
return
@@ -88,7 +102,7 @@ async function fetchCategoryPage(slug: string): Promise<ScrapedRow[]> {
return
}
rows.push({ name, price, imageUrl, sourceUrl, sourceRef })
rows.push({ name, price, imageUrl, purchaseUrl: amazonUrl || productPageUrl, sourceRef })
})
return rows
@@ -116,7 +130,7 @@ async function scrapeComponentType(componentType: ComponentType): Promise<number
name: row.name,
price: row.price,
imageUrl: row.imageUrl,
sourceUrl: row.sourceUrl,
sourceUrl: row.purchaseUrl,
scrapedAt,
}
)